Here are the examples of the java api org.apache.hadoop.yarn.api.records.timeline.TimelineEntity taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.
87 Examples
19
Source : SystemMetricsPublisher.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
private static TimelineEnreplacedy createAppAttemptEnreplacedy(ApplicationAttemptId appAttemptId) {
TimelineEnreplacedy enreplacedy = new TimelineEnreplacedy();
enreplacedy.setEnreplacedyType(AppAttemptMetricsConstants.ENreplacedY_TYPE);
enreplacedy.setEnreplacedyId(appAttemptId.toString());
enreplacedy.addPrimaryFilter(AppAttemptMetricsConstants.PARENT_PRIMARY_FILTER, appAttemptId.getApplicationId().toString());
return enreplacedy;
}
19
Source : SystemMetricsPublisher.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
private void putEnreplacedy(TimelineEnreplacedy enreplacedy) {
try {
if (LOG.isDebugEnabled()) {
LOG.debug("Publishing the enreplacedy " + enreplacedy.getEnreplacedyId() + ", JSON-style content: " + TimelineUtils.dumpTimelineRecordtoJSON(enreplacedy));
}
client.putEnreplacedies(enreplacedy);
} catch (Exception e) {
LOG.error("Error when publishing enreplacedy [" + enreplacedy.getEnreplacedyType() + "," + enreplacedy.getEnreplacedyId() + "]", e);
}
}
19
Source : TimelineStoreTestUtils.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
/**
* Verify a single enreplacedy and its start time
*/
protected static void verifyEnreplacedyInfo(String enreplacedyId, String enreplacedyType, List<TimelineEvent> events, Map<String, Set<String>> relatedEnreplacedies, Map<String, Set<Object>> primaryFilters, Map<String, Object> otherInfo, Long startTime, TimelineEnreplacedy retrievedEnreplacedyInfo, String domainId) {
verifyEnreplacedyInfo(enreplacedyId, enreplacedyType, events, relatedEnreplacedies, primaryFilters, otherInfo, retrievedEnreplacedyInfo, domainId);
replacedertEquals(startTime, retrievedEnreplacedyInfo.getStartTime());
}
19
Source : TestTimelineACLsManager.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
@Test
public void testYarnACLsNotEnabledForEnreplacedy() throws Exception {
Configuration conf = new YarnConfiguration();
conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, false);
TimelineACLsManager timelineACLsManager = new TimelineACLsManager(conf);
timelineACLsManager.setTimelineStore(new TestTimelineStore());
TimelineEnreplacedy enreplacedy = new TimelineEnreplacedy();
enreplacedy.addPrimaryFilter(TimelineStore.SystemFilter.ENreplacedY_OWNER.toString(), "owner");
enreplacedy.setDomainId("domain_id_1");
replacedert.replacedertTrue("Always true when ACLs are not enabled", timelineACLsManager.checkAccess(UserGroupInformation.createRemoteUser("user"), ApplicationAccessType.VIEW_APP, enreplacedy));
replacedert.replacedertTrue("Always true when ACLs are not enabled", timelineACLsManager.checkAccess(UserGroupInformation.createRemoteUser("user"), ApplicationAccessType.MODIFY_APP, enreplacedy));
}
19
Source : TestTimelineACLsManager.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
@Test
public void testYarnACLsEnabledForEnreplacedy() throws Exception {
Configuration conf = new YarnConfiguration();
conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
conf.set(YarnConfiguration.YARN_ADMIN_ACL, "admin");
TimelineACLsManager timelineACLsManager = new TimelineACLsManager(conf);
timelineACLsManager.setTimelineStore(new TestTimelineStore());
TimelineEnreplacedy enreplacedy = new TimelineEnreplacedy();
enreplacedy.addPrimaryFilter(TimelineStore.SystemFilter.ENreplacedY_OWNER.toString(), "owner");
enreplacedy.setDomainId("domain_id_1");
replacedert.replacedertTrue("Owner should be allowed to view", timelineACLsManager.checkAccess(UserGroupInformation.createRemoteUser("owner"), ApplicationAccessType.VIEW_APP, enreplacedy));
replacedert.replacedertTrue("Reader should be allowed to view", timelineACLsManager.checkAccess(UserGroupInformation.createRemoteUser("reader"), ApplicationAccessType.VIEW_APP, enreplacedy));
replacedert.replacedertFalse("Other shouldn't be allowed to view", timelineACLsManager.checkAccess(UserGroupInformation.createRemoteUser("other"), ApplicationAccessType.VIEW_APP, enreplacedy));
replacedert.replacedertTrue("Admin should be allowed to view", timelineACLsManager.checkAccess(UserGroupInformation.createRemoteUser("admin"), ApplicationAccessType.VIEW_APP, enreplacedy));
replacedert.replacedertTrue("Owner should be allowed to modify", timelineACLsManager.checkAccess(UserGroupInformation.createRemoteUser("owner"), ApplicationAccessType.MODIFY_APP, enreplacedy));
replacedert.replacedertTrue("Writer should be allowed to modify", timelineACLsManager.checkAccess(UserGroupInformation.createRemoteUser("writer"), ApplicationAccessType.MODIFY_APP, enreplacedy));
replacedert.replacedertFalse("Other shouldn't be allowed to modify", timelineACLsManager.checkAccess(UserGroupInformation.createRemoteUser("other"), ApplicationAccessType.MODIFY_APP, enreplacedy));
replacedert.replacedertTrue("Admin should be allowed to modify", timelineACLsManager.checkAccess(UserGroupInformation.createRemoteUser("admin"), ApplicationAccessType.MODIFY_APP, enreplacedy));
}
19
Source : TimelineDataManager.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
private static void addDefaultDomainIdIfAbsent(TimelineEnreplacedy enreplacedy) {
// be compatible with the timeline data created before 2.6
if (enreplacedy.getDomainId() == null) {
enreplacedy.setDomainId(DEFAULT_DOMAIN_ID);
}
}
18
Source : TestSystemMetricsPublisher.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
@Test(timeout = 10000)
public void testPublishAppAttemptMetrics() throws Exception {
ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 1), 1);
RMAppAttempt appAttempt = createRMAppAttempt(appAttemptId);
metricsPublisher.appAttemptRegistered(appAttempt, Integer.MAX_VALUE + 1L);
RMApp app = mock(RMApp.clreplaced);
when(app.getFinalApplicationStatus()).thenReturn(FinalApplicationStatus.UNDEFINED);
metricsPublisher.appAttemptFinished(appAttempt, RMAppAttemptState.FINISHED, app, Integer.MAX_VALUE + 2L);
TimelineEnreplacedy enreplacedy = null;
do {
enreplacedy = store.getEnreplacedy(appAttemptId.toString(), AppAttemptMetricsConstants.ENreplacedY_TYPE, EnumSet.allOf(Field.clreplaced));
// ensure two events are both published before leaving the loop
} while (enreplacedy == null || enreplacedy.getEvents().size() < 2);
// verify all the fields
replacedert.replacedertEquals(AppAttemptMetricsConstants.ENreplacedY_TYPE, enreplacedy.getEnreplacedyType());
replacedert.replacedertEquals(appAttemptId.toString(), enreplacedy.getEnreplacedyId());
replacedert.replacedertEquals(appAttemptId.getApplicationId().toString(), enreplacedy.getPrimaryFilters().get(AppAttemptMetricsConstants.PARENT_PRIMARY_FILTER).iterator().next());
boolean hasRegisteredEvent = false;
boolean hasFinishedEvent = false;
for (TimelineEvent event : enreplacedy.getEvents()) {
if (event.getEventType().equals(AppAttemptMetricsConstants.REGISTERED_EVENT_TYPE)) {
hasRegisteredEvent = true;
replacedert.replacedertEquals(appAttempt.getHost(), event.getEventInfo().get(AppAttemptMetricsConstants.HOST_EVENT_INFO));
replacedert.replacedertEquals(appAttempt.getRpcPort(), event.getEventInfo().get(AppAttemptMetricsConstants.RPC_PORT_EVENT_INFO));
replacedert.replacedertEquals(appAttempt.getMasterContainer().getId().toString(), event.getEventInfo().get(AppAttemptMetricsConstants.MASTER_CONTAINER_EVENT_INFO));
} else if (event.getEventType().equals(AppAttemptMetricsConstants.FINISHED_EVENT_TYPE)) {
hasFinishedEvent = true;
replacedert.replacedertEquals(appAttempt.getDiagnostics(), event.getEventInfo().get(AppAttemptMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO));
replacedert.replacedertEquals(appAttempt.getTrackingUrl(), event.getEventInfo().get(AppAttemptMetricsConstants.TRACKING_URL_EVENT_INFO));
replacedert.replacedertEquals(appAttempt.getOriginalTrackingUrl(), event.getEventInfo().get(AppAttemptMetricsConstants.ORIGINAL_TRACKING_URL_EVENT_INFO));
replacedert.replacedertEquals(FinalApplicationStatus.UNDEFINED.toString(), event.getEventInfo().get(AppAttemptMetricsConstants.FINAL_STATUS_EVENT_INFO));
replacedert.replacedertEquals(YarnApplicationAttemptState.FINISHED.toString(), event.getEventInfo().get(AppAttemptMetricsConstants.STATE_EVENT_INFO));
}
}
replacedert.replacedertTrue(hasRegisteredEvent && hasFinishedEvent);
}
18
Source : SystemMetricsPublisher.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
private static TimelineEnreplacedy createContainerEnreplacedy(ContainerId containerId) {
TimelineEnreplacedy enreplacedy = new TimelineEnreplacedy();
enreplacedy.setEnreplacedyType(ContainerMetricsConstants.ENreplacedY_TYPE);
enreplacedy.setEnreplacedyId(containerId.toString());
enreplacedy.addPrimaryFilter(ContainerMetricsConstants.PARENT_PRIMARIY_FILTER, containerId.getApplicationAttemptId().toString());
return enreplacedy;
}
18
Source : TestTimelineACLsManager.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
@Test
public void testCorruptedOwnerInfoForEnreplacedy() throws Exception {
Configuration conf = new YarnConfiguration();
conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
conf.set(YarnConfiguration.YARN_ADMIN_ACL, "owner");
TimelineACLsManager timelineACLsManager = new TimelineACLsManager(conf);
timelineACLsManager.setTimelineStore(new TestTimelineStore());
TimelineEnreplacedy enreplacedy = new TimelineEnreplacedy();
try {
timelineACLsManager.checkAccess(UserGroupInformation.createRemoteUser("owner"), ApplicationAccessType.VIEW_APP, enreplacedy);
replacedert.fail("Exception is expected");
} catch (YarnException e) {
replacedert.replacedertTrue("It's not the exact expected exception", e.getMessage().contains("doesn't exist."));
}
}
18
Source : TimelineDataManager.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
/**
* Get the single timeline enreplacedy that the given user has access to. The
* meaning of each argument has been doreplacedented with
* {@link TimelineReader#getEnreplacedy}.
*
* @see TimelineReader#getEnreplacedy
*/
public TimelineEnreplacedy getEnreplacedy(String enreplacedyType, String enreplacedyId, EnumSet<Field> fields, UserGroupInformation callerUGI) throws YarnException, IOException {
TimelineEnreplacedy enreplacedy = null;
enreplacedy = store.getEnreplacedy(enreplacedyId, enreplacedyType, fields);
if (enreplacedy != null) {
addDefaultDomainIdIfAbsent(enreplacedy);
// check ACLs
if (!timelineACLsManager.checkAccess(callerUGI, ApplicationAccessType.VIEW_APP, enreplacedy)) {
enreplacedy = null;
}
}
return enreplacedy;
}
18
Source : TimelineACLsManager.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
public boolean checkAccess(UserGroupInformation callerUGI, ApplicationAccessType applicationAccessType, TimelineEnreplacedy enreplacedy) throws YarnException, IOException {
if (LOG.isDebugEnabled()) {
LOG.debug("Verifying the access of " + (callerUGI == null ? null : callerUGI.getShortUserName()) + " on the timeline enreplacedy " + new EnreplacedyIdentifier(enreplacedy.getEnreplacedyId(), enreplacedy.getEnreplacedyType()));
}
if (!adminAclsManager.areACLsEnabled()) {
return true;
}
// find domain owner and acls
AccessControlListExt aclExt = aclExts.get(enreplacedy.getDomainId());
if (aclExt == null) {
aclExt = loadDomainFromTimelineStore(enreplacedy.getDomainId());
}
if (aclExt == null) {
throw new YarnException("Domain information of the timeline enreplacedy " + new EnreplacedyIdentifier(enreplacedy.getEnreplacedyId(), enreplacedy.getEnreplacedyType()) + " doesn't exist.");
}
String owner = aclExt.owner;
AccessControlList domainACL = aclExt.acls.get(applicationAccessType);
if (domainACL == null) {
if (LOG.isDebugEnabled()) {
LOG.debug("ACL not found for access-type " + applicationAccessType + " for domain " + enreplacedy.getDomainId() + " owned by " + owner + ". Using default [" + YarnConfiguration.DEFAULT_YARN_APP_ACL + "]");
}
domainACL = new AccessControlList(YarnConfiguration.DEFAULT_YARN_APP_ACL);
}
if (callerUGI != null && (adminAclsManager.isAdmin(callerUGI) || callerUGI.getShortUserName().equals(owner) || domainACL.isUserAllowed(callerUGI))) {
return true;
}
return false;
}
18
Source : MemoryTimelineStore.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
@Override
public synchronized TimelineEnreplacedy getEnreplacedy(String enreplacedyId, String enreplacedyType, EnumSet<Field> fieldsToRetrieve) {
if (fieldsToRetrieve == null) {
fieldsToRetrieve = EnumSet.allOf(Field.clreplaced);
}
TimelineEnreplacedy enreplacedy = enreplacedies.get(new EnreplacedyIdentifier(enreplacedyId, enreplacedyType));
if (enreplacedy == null) {
return null;
} else {
return maskFields(enreplacedy, fieldsToRetrieve);
}
}
18
Source : ApplicationHistoryManagerOnTimelineStore.java
with Apache License 2.0
from naver
with Apache License 2.0
from naver
private ApplicationAttemptReport getApplicationAttempt(ApplicationAttemptId appAttemptId, boolean checkACLs) throws YarnException, IOException {
if (checkACLs) {
ApplicationReportExt app = getApplication(appAttemptId.getApplicationId(), ApplicationReportField.USER_AND_ACLS);
checkAccess(app);
}
TimelineEnreplacedy enreplacedy = timelineDataManager.getEnreplacedy(AppAttemptMetricsConstants.ENreplacedY_TYPE, appAttemptId.toString(), EnumSet.allOf(Field.clreplaced), UserGroupInformation.getLoginUser());
if (enreplacedy == null) {
throw new ApplicationAttemptNotFoundException("The enreplacedy for application attempt " + appAttemptId + " doesn't exist in the timeline store");
} else {
return convertToApplicationAttemptReport(enreplacedy);
}
}
18
Source : ApplicationHistoryManagerOnTimelineStore.java
with Apache License 2.0
from naver
with Apache License 2.0
from naver
@Override
public Map<ApplicationId, ApplicationReport> getAllApplications() throws YarnException, IOException {
TimelineEnreplacedies enreplacedies = timelineDataManager.getEnreplacedies(ApplicationMetricsConstants.ENreplacedY_TYPE, null, null, null, null, null, null, Long.MAX_VALUE, EnumSet.allOf(Field.clreplaced), UserGroupInformation.getLoginUser());
Map<ApplicationId, ApplicationReport> apps = new LinkedHashMap<ApplicationId, ApplicationReport>();
if (enreplacedies != null && enreplacedies.getEnreplacedies() != null) {
for (TimelineEnreplacedy enreplacedy : enreplacedies.getEnreplacedies()) {
try {
ApplicationReportExt app = generateApplicationReport(enreplacedy, ApplicationReportField.ALL);
apps.put(app.appReport.getApplicationId(), app.appReport);
} catch (Exception e) {
LOG.error("Error on generating application report for " + enreplacedy.getEnreplacedyId(), e);
}
}
}
return apps;
}
18
Source : ApplicationMaster.java
with Apache License 2.0
from naver
with Apache License 2.0
from naver
private static void publishApplicationAttemptEvent(final TimelineClient timelineClient, String appAttemptId, DSEvent appEvent, String domainId, UserGroupInformation ugi) {
final TimelineEnreplacedy enreplacedy = new TimelineEnreplacedy();
enreplacedy.setEnreplacedyId(appAttemptId);
enreplacedy.setEnreplacedyType(DSEnreplacedy.DS_APP_ATTEMPT.toString());
enreplacedy.setDomainId(domainId);
enreplacedy.addPrimaryFilter("user", ugi.getShortUserName());
TimelineEvent event = new TimelineEvent();
event.setEventType(appEvent.toString());
event.setTimestamp(System.currentTimeMillis());
enreplacedy.addEvent(event);
try {
timelineClient.putEnreplacedies(enreplacedy);
} catch (YarnException | IOException e) {
LOG.error("App Attempt " + (appEvent.equals(DSEvent.DS_APP_ATTEMPT_START) ? "start" : "end") + " event could not be published for " + appAttemptId.toString(), e);
}
}
18
Source : TestSystemMetricsPublisher.java
with Apache License 2.0
from lsds
with Apache License 2.0
from lsds
@Test(timeout = 10000)
public void testPublishAppAttemptMetricsForUnmanagedAM() throws Exception {
ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 1), 1);
RMAppAttempt appAttempt = createRMAppAttempt(appAttemptId, true);
metricsPublisher.appAttemptRegistered(appAttempt, Integer.MAX_VALUE + 1L);
RMApp app = mock(RMApp.clreplaced);
when(app.getFinalApplicationStatus()).thenReturn(FinalApplicationStatus.UNDEFINED);
metricsPublisher.appAttemptFinished(appAttempt, RMAppAttemptState.FINISHED, app, Integer.MAX_VALUE + 2L);
TimelineEnreplacedy enreplacedy = null;
do {
enreplacedy = store.getEnreplacedy(appAttemptId.toString(), AppAttemptMetricsConstants.ENreplacedY_TYPE, EnumSet.allOf(Field.clreplaced));
// ensure two events are both published before leaving the loop
} while (enreplacedy == null || enreplacedy.getEvents().size() < 2);
}
18
Source : TestSystemMetricsPublisher.java
with Apache License 2.0
from lsds
with Apache License 2.0
from lsds
@Test(timeout = 10000)
public void testPublishAppAttemptMetrics() throws Exception {
ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 1), 1);
RMAppAttempt appAttempt = createRMAppAttempt(appAttemptId, false);
metricsPublisher.appAttemptRegistered(appAttempt, Integer.MAX_VALUE + 1L);
RMApp app = mock(RMApp.clreplaced);
when(app.getFinalApplicationStatus()).thenReturn(FinalApplicationStatus.UNDEFINED);
metricsPublisher.appAttemptFinished(appAttempt, RMAppAttemptState.FINISHED, app, Integer.MAX_VALUE + 2L);
TimelineEnreplacedy enreplacedy = null;
do {
enreplacedy = store.getEnreplacedy(appAttemptId.toString(), AppAttemptMetricsConstants.ENreplacedY_TYPE, EnumSet.allOf(Field.clreplaced));
// ensure two events are both published before leaving the loop
} while (enreplacedy == null || enreplacedy.getEvents().size() < 2);
// verify all the fields
replacedert.replacedertEquals(AppAttemptMetricsConstants.ENreplacedY_TYPE, enreplacedy.getEnreplacedyType());
replacedert.replacedertEquals(appAttemptId.toString(), enreplacedy.getEnreplacedyId());
replacedert.replacedertEquals(appAttemptId.getApplicationId().toString(), enreplacedy.getPrimaryFilters().get(AppAttemptMetricsConstants.PARENT_PRIMARY_FILTER).iterator().next());
boolean hasRegisteredEvent = false;
boolean hasFinishedEvent = false;
for (TimelineEvent event : enreplacedy.getEvents()) {
if (event.getEventType().equals(AppAttemptMetricsConstants.REGISTERED_EVENT_TYPE)) {
hasRegisteredEvent = true;
replacedert.replacedertEquals(appAttempt.getHost(), event.getEventInfo().get(AppAttemptMetricsConstants.HOST_EVENT_INFO));
replacedert.replacedertEquals(appAttempt.getRpcPort(), event.getEventInfo().get(AppAttemptMetricsConstants.RPC_PORT_EVENT_INFO));
replacedert.replacedertEquals(appAttempt.getMasterContainer().getId().toString(), event.getEventInfo().get(AppAttemptMetricsConstants.MASTER_CONTAINER_EVENT_INFO));
} else if (event.getEventType().equals(AppAttemptMetricsConstants.FINISHED_EVENT_TYPE)) {
hasFinishedEvent = true;
replacedert.replacedertEquals(appAttempt.getDiagnostics(), event.getEventInfo().get(AppAttemptMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO));
replacedert.replacedertEquals(appAttempt.getTrackingUrl(), event.getEventInfo().get(AppAttemptMetricsConstants.TRACKING_URL_EVENT_INFO));
replacedert.replacedertEquals(appAttempt.getOriginalTrackingUrl(), event.getEventInfo().get(AppAttemptMetricsConstants.ORIGINAL_TRACKING_URL_EVENT_INFO));
replacedert.replacedertEquals(FinalApplicationStatus.UNDEFINED.toString(), event.getEventInfo().get(AppAttemptMetricsConstants.FINAL_STATUS_EVENT_INFO));
replacedert.replacedertEquals(YarnApplicationAttemptState.FINISHED.toString(), event.getEventInfo().get(AppAttemptMetricsConstants.STATE_EVENT_INFO));
}
}
replacedert.replacedertTrue(hasRegisteredEvent && hasFinishedEvent);
}
17
Source : TestSystemMetricsPublisher.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
@Test(timeout = 10000)
public void testPublishApplicationMetrics() throws Exception {
for (int i = 1; i <= 2; ++i) {
ApplicationId appId = ApplicationId.newInstance(0, i);
RMApp app = createRMApp(appId);
metricsPublisher.appCreated(app, app.getStartTime());
metricsPublisher.appFinished(app, RMAppState.FINISHED, app.getFinishTime());
if (i == 1) {
metricsPublisher.appACLsUpdated(app, "uers1,user2", 4L);
} else {
// in case user doesn't specify the ACLs
metricsPublisher.appACLsUpdated(app, null, 4L);
}
TimelineEnreplacedy enreplacedy = null;
do {
enreplacedy = store.getEnreplacedy(appId.toString(), ApplicationMetricsConstants.ENreplacedY_TYPE, EnumSet.allOf(Field.clreplaced));
// ensure three events are both published before leaving the loop
} while (enreplacedy == null || enreplacedy.getEvents().size() < 3);
// verify all the fields
replacedert.replacedertEquals(ApplicationMetricsConstants.ENreplacedY_TYPE, enreplacedy.getEnreplacedyType());
replacedert.replacedertEquals(app.getApplicationId().toString(), enreplacedy.getEnreplacedyId());
replacedert.replacedertEquals(app.getName(), enreplacedy.getOtherInfo().get(ApplicationMetricsConstants.NAME_ENreplacedY_INFO));
replacedert.replacedertEquals(app.getQueue(), enreplacedy.getOtherInfo().get(ApplicationMetricsConstants.QUEUE_ENreplacedY_INFO));
replacedert.replacedertEquals(app.getUser(), enreplacedy.getOtherInfo().get(ApplicationMetricsConstants.USER_ENreplacedY_INFO));
replacedert.replacedertEquals(app.getApplicationType(), enreplacedy.getOtherInfo().get(ApplicationMetricsConstants.TYPE_ENreplacedY_INFO));
replacedert.replacedertEquals(app.getSubmitTime(), enreplacedy.getOtherInfo().get(ApplicationMetricsConstants.SUBMITTED_TIME_ENreplacedY_INFO));
if (i == 1) {
replacedert.replacedertEquals("uers1,user2", enreplacedy.getOtherInfo().get(ApplicationMetricsConstants.APP_VIEW_ACLS_ENreplacedY_INFO));
} else {
replacedert.replacedertEquals("", enreplacedy.getOtherInfo().get(ApplicationMetricsConstants.APP_VIEW_ACLS_ENreplacedY_INFO));
}
boolean hasCreatedEvent = false;
boolean hasFinishedEvent = false;
boolean hasACLsUpdatedEvent = false;
for (TimelineEvent event : enreplacedy.getEvents()) {
if (event.getEventType().equals(ApplicationMetricsConstants.CREATED_EVENT_TYPE)) {
hasCreatedEvent = true;
replacedert.replacedertEquals(app.getStartTime(), event.getTimestamp());
} else if (event.getEventType().equals(ApplicationMetricsConstants.FINISHED_EVENT_TYPE)) {
hasFinishedEvent = true;
replacedert.replacedertEquals(app.getFinishTime(), event.getTimestamp());
replacedert.replacedertEquals(app.getDiagnostics().toString(), event.getEventInfo().get(ApplicationMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO));
replacedert.replacedertEquals(app.getFinalApplicationStatus().toString(), event.getEventInfo().get(ApplicationMetricsConstants.FINAL_STATUS_EVENT_INFO));
replacedert.replacedertEquals(YarnApplicationState.FINISHED.toString(), event.getEventInfo().get(ApplicationMetricsConstants.STATE_EVENT_INFO));
} else if (event.getEventType().equals(ApplicationMetricsConstants.ACLS_UPDATED_EVENT_TYPE)) {
hasACLsUpdatedEvent = true;
replacedert.replacedertEquals(4L, event.getTimestamp());
}
}
replacedert.replacedertTrue(hasCreatedEvent && hasFinishedEvent && hasACLsUpdatedEvent);
}
}
17
Source : SystemMetricsPublisher.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
private static TimelineEnreplacedy createApplicationEnreplacedy(ApplicationId applicationId) {
TimelineEnreplacedy enreplacedy = new TimelineEnreplacedy();
enreplacedy.setEnreplacedyType(ApplicationMetricsConstants.ENreplacedY_TYPE);
enreplacedy.setEnreplacedyId(applicationId.toString());
return enreplacedy;
}
17
Source : TestTimelineWebServices.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
private static void verifyEnreplacedies(TimelineEnreplacedies enreplacedies) {
replacedert.replacedertNotNull(enreplacedies);
replacedert.replacedertEquals(3, enreplacedies.getEnreplacedies().size());
TimelineEnreplacedy enreplacedy1 = enreplacedies.getEnreplacedies().get(0);
replacedert.replacedertNotNull(enreplacedy1);
replacedert.replacedertEquals("id_1", enreplacedy1.getEnreplacedyId());
replacedert.replacedertEquals("type_1", enreplacedy1.getEnreplacedyType());
replacedert.replacedertEquals(123l, enreplacedy1.getStartTime().longValue());
replacedert.replacedertEquals(2, enreplacedy1.getEvents().size());
replacedert.replacedertEquals(4, enreplacedy1.getPrimaryFilters().size());
replacedert.replacedertEquals(4, enreplacedy1.getOtherInfo().size());
TimelineEnreplacedy enreplacedy2 = enreplacedies.getEnreplacedies().get(1);
replacedert.replacedertNotNull(enreplacedy2);
replacedert.replacedertEquals("id_2", enreplacedy2.getEnreplacedyId());
replacedert.replacedertEquals("type_1", enreplacedy2.getEnreplacedyType());
replacedert.replacedertEquals(123l, enreplacedy2.getStartTime().longValue());
replacedert.replacedertEquals(2, enreplacedy2.getEvents().size());
replacedert.replacedertEquals(4, enreplacedy2.getPrimaryFilters().size());
replacedert.replacedertEquals(4, enreplacedy2.getOtherInfo().size());
TimelineEnreplacedy enreplacedy3 = enreplacedies.getEnreplacedies().get(2);
replacedert.replacedertNotNull(enreplacedy2);
replacedert.replacedertEquals("id_6", enreplacedy3.getEnreplacedyId());
replacedert.replacedertEquals("type_1", enreplacedy3.getEnreplacedyType());
replacedert.replacedertEquals(61l, enreplacedy3.getStartTime().longValue());
replacedert.replacedertEquals(0, enreplacedy3.getEvents().size());
replacedert.replacedertEquals(4, enreplacedy3.getPrimaryFilters().size());
replacedert.replacedertEquals(4, enreplacedy3.getOtherInfo().size());
}
17
Source : MemoryTimelineStore.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
@Override
public synchronized TimelineEvents getEnreplacedyTimelines(String enreplacedyType, SortedSet<String> enreplacedyIds, Long limit, Long windowStart, Long windowEnd, Set<String> eventTypes) {
TimelineEvents allEvents = new TimelineEvents();
if (enreplacedyIds == null) {
return allEvents;
}
if (limit == null) {
limit = DEFAULT_LIMIT;
}
if (windowStart == null) {
windowStart = Long.MIN_VALUE;
}
if (windowEnd == null) {
windowEnd = Long.MAX_VALUE;
}
for (String enreplacedyId : enreplacedyIds) {
EnreplacedyIdentifier enreplacedyID = new EnreplacedyIdentifier(enreplacedyId, enreplacedyType);
TimelineEnreplacedy enreplacedy = enreplacedies.get(enreplacedyID);
if (enreplacedy == null) {
continue;
}
EventsOfOneEnreplacedy events = new EventsOfOneEnreplacedy();
events.setEnreplacedyId(enreplacedyId);
events.setEnreplacedyType(enreplacedyType);
for (TimelineEvent event : enreplacedy.getEvents()) {
if (events.getEvents().size() >= limit) {
break;
}
if (event.getTimestamp() <= windowStart) {
continue;
}
if (event.getTimestamp() > windowEnd) {
continue;
}
if (eventTypes != null && !eventTypes.contains(event.getEventType())) {
continue;
}
events.addEvent(event);
}
allEvents.addEvent(events);
}
return allEvents;
}
17
Source : ApplicationHistoryManagerOnTimelineStore.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
@Override
public ContainerReport getContainer(ContainerId containerId) throws YarnException, IOException {
ApplicationReportExt app = getApplication(containerId.getApplicationAttemptId().getApplicationId(), ApplicationReportField.USER_AND_ACLS);
checkAccess(app);
TimelineEnreplacedy enreplacedy = timelineDataManager.getEnreplacedy(ContainerMetricsConstants.ENreplacedY_TYPE, containerId.toString(), EnumSet.allOf(Field.clreplaced), UserGroupInformation.getLoginUser());
if (enreplacedy == null) {
throw new ContainerNotFoundException("The enreplacedy for container " + containerId + " doesn't exist in the timeline store");
} else {
return convertToContainerReport(enreplacedy, serverHttpAddress, app.appReport.getUser());
}
}
17
Source : ApplicationHistoryManagerOnTimelineStore.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
@Override
public Map<ApplicationAttemptId, ApplicationAttemptReport> getApplicationAttempts(ApplicationId appId) throws YarnException, IOException {
ApplicationReportExt app = getApplication(appId, ApplicationReportField.USER_AND_ACLS);
checkAccess(app);
TimelineEnreplacedies enreplacedies = timelineDataManager.getEnreplacedies(AppAttemptMetricsConstants.ENreplacedY_TYPE, new NameValuePair(AppAttemptMetricsConstants.PARENT_PRIMARY_FILTER, appId.toString()), null, null, null, null, null, Long.MAX_VALUE, EnumSet.allOf(Field.clreplaced), UserGroupInformation.getLoginUser());
Map<ApplicationAttemptId, ApplicationAttemptReport> appAttempts = new LinkedHashMap<ApplicationAttemptId, ApplicationAttemptReport>();
for (TimelineEnreplacedy enreplacedy : enreplacedies.getEnreplacedies()) {
ApplicationAttemptReport appAttempt = convertToApplicationAttemptReport(enreplacedy);
appAttempts.put(appAttempt.getApplicationAttemptId(), appAttempt);
}
return appAttempts;
}
17
Source : ApplicationHistoryManagerOnTimelineStore.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
@Override
public Map<ContainerId, ContainerReport> getContainers(ApplicationAttemptId appAttemptId) throws YarnException, IOException {
ApplicationReportExt app = getApplication(appAttemptId.getApplicationId(), ApplicationReportField.USER_AND_ACLS);
checkAccess(app);
TimelineEnreplacedies enreplacedies = timelineDataManager.getEnreplacedies(ContainerMetricsConstants.ENreplacedY_TYPE, new NameValuePair(ContainerMetricsConstants.PARENT_PRIMARIY_FILTER, appAttemptId.toString()), null, null, null, null, null, Long.MAX_VALUE, EnumSet.allOf(Field.clreplaced), UserGroupInformation.getLoginUser());
Map<ContainerId, ContainerReport> containers = new LinkedHashMap<ContainerId, ContainerReport>();
if (enreplacedies != null && enreplacedies.getEnreplacedies() != null) {
for (TimelineEnreplacedy enreplacedy : enreplacedies.getEnreplacedies()) {
ContainerReport container = convertToContainerReport(enreplacedy, serverHttpAddress, app.appReport.getUser());
containers.put(container.getContainerId(), container);
}
}
return containers;
}
17
Source : ApplicationHistoryManagerOnTimelineStore.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
@Override
public ApplicationAttemptReport getApplicationAttempt(ApplicationAttemptId appAttemptId) throws YarnException, IOException {
ApplicationReportExt app = getApplication(appAttemptId.getApplicationId(), ApplicationReportField.USER_AND_ACLS);
checkAccess(app);
TimelineEnreplacedy enreplacedy = timelineDataManager.getEnreplacedy(AppAttemptMetricsConstants.ENreplacedY_TYPE, appAttemptId.toString(), EnumSet.allOf(Field.clreplaced), UserGroupInformation.getLoginUser());
if (enreplacedy == null) {
throw new ApplicationAttemptNotFoundException("The enreplacedy for application attempt " + appAttemptId + " doesn't exist in the timeline store");
} else {
return convertToApplicationAttemptReport(enreplacedy);
}
}
17
Source : TimelineClientImpl.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
@Override
public TimelinePutResponse putEnreplacedies(TimelineEnreplacedy... enreplacedies) throws IOException, YarnException {
TimelineEnreplacedies enreplacediesContainer = new TimelineEnreplacedies();
enreplacediesContainer.addEnreplacedies(Arrays.asList(enreplacedies));
ClientResponse resp = doPosting(enreplacediesContainer, null);
return resp.getEnreplacedy(TimelinePutResponse.clreplaced);
}
17
Source : ApplicationMaster.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
private static void publishApplicationAttemptEvent(final TimelineClient timelineClient, String appAttemptId, DSEvent appEvent, String domainId, UserGroupInformation ugi) {
final TimelineEnreplacedy enreplacedy = new TimelineEnreplacedy();
enreplacedy.setEnreplacedyId(appAttemptId);
enreplacedy.setEnreplacedyType(DSEnreplacedy.DS_APP_ATTEMPT.toString());
enreplacedy.setDomainId(domainId);
enreplacedy.addPrimaryFilter("user", ugi.getShortUserName());
TimelineEvent event = new TimelineEvent();
event.setEventType(appEvent.toString());
event.setTimestamp(System.currentTimeMillis());
enreplacedy.addEvent(event);
try {
ugi.doAs(new PrivilegedExceptionAction<TimelinePutResponse>() {
@Override
public TimelinePutResponse run() throws Exception {
return timelineClient.putEnreplacedies(enreplacedy);
}
});
} catch (Exception e) {
LOG.error("App Attempt " + (appEvent.equals(DSEvent.DS_APP_ATTEMPT_START) ? "start" : "end") + " event could not be published for " + appAttemptId.toString(), e instanceof UndeclaredThrowableException ? e.getCause() : e);
}
}
17
Source : ApplicationHistoryManagerOnTimelineStore.java
with Apache License 2.0
from naver
with Apache License 2.0
from naver
private ApplicationReportExt generateApplicationReport(TimelineEnreplacedy enreplacedy, ApplicationReportField field) throws YarnException, IOException {
ApplicationReportExt app = convertToApplicationReport(enreplacedy, field);
// If only user and acls are pulled to check attempt(s)/container(s) access
// control, we can return immediately
if (field == ApplicationReportField.USER_AND_ACLS) {
return app;
}
try {
checkAccess(app);
if (app.appReport.getCurrentApplicationAttemptId() != null) {
ApplicationAttemptReport appAttempt = getApplicationAttempt(app.appReport.getCurrentApplicationAttemptId(), false);
app.appReport.setHost(appAttempt.getHost());
app.appReport.setRpcPort(appAttempt.getRpcPort());
app.appReport.setTrackingUrl(appAttempt.getTrackingUrl());
app.appReport.setOriginalTrackingUrl(appAttempt.getOriginalTrackingUrl());
}
} catch (AuthorizationException | ApplicationAttemptNotFoundException e) {
// AuthorizationException is thrown because the user doesn't have access
// It's possible that the app is finished before the first attempt is created.
app.appReport.setDiagnostics(null);
app.appReport.setCurrentApplicationAttemptId(null);
}
if (app.appReport.getCurrentApplicationAttemptId() == null) {
app.appReport.setCurrentApplicationAttemptId(ApplicationAttemptId.newInstance(app.appReport.getApplicationId(), -1));
}
if (app.appReport.getHost() == null) {
app.appReport.setHost(UNAVAILABLE);
}
if (app.appReport.getRpcPort() < 0) {
app.appReport.setRpcPort(-1);
}
if (app.appReport.getTrackingUrl() == null) {
app.appReport.setTrackingUrl(UNAVAILABLE);
}
if (app.appReport.getOriginalTrackingUrl() == null) {
app.appReport.setOriginalTrackingUrl(UNAVAILABLE);
}
if (app.appReport.getDiagnostics() == null) {
app.appReport.setDiagnostics("");
}
return app;
}
17
Source : ApplicationHistoryManagerOnTimelineStore.java
with Apache License 2.0
from naver
with Apache License 2.0
from naver
private static ContainerReport convertToContainerReport(TimelineEnreplacedy enreplacedy, String serverHttpAddress, String user) {
int allocatedMem = 0;
int allocatedVcore = 0;
int allocatedGcore = 0;
String allocatedHost = null;
int allocatedPort = -1;
int allocatedPriority = 0;
long createdTime = 0;
long finishedTime = 0;
String diagnosticsInfo = null;
int exitStatus = ContainerExitStatus.INVALID;
ContainerState state = null;
String nodeHttpAddress = null;
Map<String, Object> enreplacedyInfo = enreplacedy.getOtherInfo();
if (enreplacedyInfo != null) {
if (enreplacedyInfo.containsKey(ContainerMetricsConstants.ALLOCATED_MEMORY_ENreplacedY_INFO)) {
allocatedMem = (Integer) enreplacedyInfo.get(ContainerMetricsConstants.ALLOCATED_MEMORY_ENreplacedY_INFO);
}
if (enreplacedyInfo.containsKey(ContainerMetricsConstants.ALLOCATED_VCORE_ENreplacedY_INFO)) {
allocatedVcore = (Integer) enreplacedyInfo.get(ContainerMetricsConstants.ALLOCATED_VCORE_ENreplacedY_INFO);
}
if (enreplacedyInfo.containsKey(ContainerMetricsConstants.ALLOCATED_GCORE_ENreplacedY_INFO)) {
allocatedGcore = (Integer) enreplacedyInfo.get(ContainerMetricsConstants.ALLOCATED_GCORE_ENreplacedY_INFO);
}
if (enreplacedyInfo.containsKey(ContainerMetricsConstants.ALLOCATED_HOST_ENreplacedY_INFO)) {
allocatedHost = enreplacedyInfo.get(ContainerMetricsConstants.ALLOCATED_HOST_ENreplacedY_INFO).toString();
}
if (enreplacedyInfo.containsKey(ContainerMetricsConstants.ALLOCATED_PORT_ENreplacedY_INFO)) {
allocatedPort = (Integer) enreplacedyInfo.get(ContainerMetricsConstants.ALLOCATED_PORT_ENreplacedY_INFO);
}
if (enreplacedyInfo.containsKey(ContainerMetricsConstants.ALLOCATED_PRIORITY_ENreplacedY_INFO)) {
allocatedPriority = (Integer) enreplacedyInfo.get(ContainerMetricsConstants.ALLOCATED_PRIORITY_ENreplacedY_INFO);
}
if (enreplacedyInfo.containsKey(ContainerMetricsConstants.ALLOCATED_HOST_HTTP_ADDRESS_ENreplacedY_INFO)) {
nodeHttpAddress = (String) enreplacedyInfo.get(ContainerMetricsConstants.ALLOCATED_HOST_HTTP_ADDRESS_ENreplacedY_INFO);
}
}
List<TimelineEvent> events = enreplacedy.getEvents();
if (events != null) {
for (TimelineEvent event : events) {
if (event.getEventType().equals(ContainerMetricsConstants.CREATED_EVENT_TYPE)) {
createdTime = event.getTimestamp();
} else if (event.getEventType().equals(ContainerMetricsConstants.FINISHED_EVENT_TYPE)) {
finishedTime = event.getTimestamp();
Map<String, Object> eventInfo = event.getEventInfo();
if (eventInfo == null) {
continue;
}
if (eventInfo.containsKey(ContainerMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO)) {
diagnosticsInfo = eventInfo.get(ContainerMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO).toString();
}
if (eventInfo.containsKey(ContainerMetricsConstants.EXIT_STATUS_EVENT_INFO)) {
exitStatus = (Integer) eventInfo.get(ContainerMetricsConstants.EXIT_STATUS_EVENT_INFO);
}
if (eventInfo.containsKey(ContainerMetricsConstants.STATE_EVENT_INFO)) {
state = ContainerState.valueOf(eventInfo.get(ContainerMetricsConstants.STATE_EVENT_INFO).toString());
}
}
}
}
NodeId allocatedNode = NodeId.newInstance(allocatedHost, allocatedPort);
ContainerId containerId = ConverterUtils.toContainerId(enreplacedy.getEnreplacedyId());
String logUrl = WebAppUtils.getAggregatedLogURL(serverHttpAddress, allocatedNode.toString(), containerId.toString(), containerId.toString(), user);
return ContainerReport.newInstance(ConverterUtils.toContainerId(enreplacedy.getEnreplacedyId()), Resource.newInstance(allocatedMem, allocatedVcore, allocatedGcore), NodeId.newInstance(allocatedHost, allocatedPort), Priority.newInstance(allocatedPriority), createdTime, finishedTime, diagnosticsInfo, logUrl, exitStatus, state, nodeHttpAddress);
}
16
Source : TestSystemMetricsPublisher.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
@Test(timeout = 10000)
public void testPublishContainerMetrics() throws Exception {
ContainerId containerId = ContainerId.newContainerId(ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 1), 1), 1);
RMContainer container = createRMContainer(containerId);
metricsPublisher.containerCreated(container, container.getCreationTime());
metricsPublisher.containerFinished(container, container.getFinishTime());
TimelineEnreplacedy enreplacedy = null;
do {
enreplacedy = store.getEnreplacedy(containerId.toString(), ContainerMetricsConstants.ENreplacedY_TYPE, EnumSet.allOf(Field.clreplaced));
// ensure two events are both published before leaving the loop
} while (enreplacedy == null || enreplacedy.getEvents().size() < 2);
// verify all the fields
replacedert.replacedertEquals(ContainerMetricsConstants.ENreplacedY_TYPE, enreplacedy.getEnreplacedyType());
replacedert.replacedertEquals(containerId.toString(), enreplacedy.getEnreplacedyId());
replacedert.replacedertEquals(containerId.getApplicationAttemptId().toString(), enreplacedy.getPrimaryFilters().get(ContainerMetricsConstants.PARENT_PRIMARIY_FILTER).iterator().next());
replacedert.replacedertEquals(container.getAllocatedNode().getHost(), enreplacedy.getOtherInfo().get(ContainerMetricsConstants.ALLOCATED_HOST_ENreplacedY_INFO));
replacedert.replacedertEquals(container.getAllocatedNode().getPort(), enreplacedy.getOtherInfo().get(ContainerMetricsConstants.ALLOCATED_PORT_ENreplacedY_INFO));
replacedert.replacedertEquals(container.getAllocatedResource().getMemory(), enreplacedy.getOtherInfo().get(ContainerMetricsConstants.ALLOCATED_MEMORY_ENreplacedY_INFO));
replacedert.replacedertEquals(container.getAllocatedResource().getVirtualCores(), enreplacedy.getOtherInfo().get(ContainerMetricsConstants.ALLOCATED_VCORE_ENreplacedY_INFO));
replacedert.replacedertEquals(container.getAllocatedPriority().getPriority(), enreplacedy.getOtherInfo().get(ContainerMetricsConstants.ALLOCATED_PRIORITY_ENreplacedY_INFO));
boolean hasCreatedEvent = false;
boolean hasFinishedEvent = false;
for (TimelineEvent event : enreplacedy.getEvents()) {
if (event.getEventType().equals(ContainerMetricsConstants.CREATED_EVENT_TYPE)) {
hasCreatedEvent = true;
replacedert.replacedertEquals(container.getCreationTime(), event.getTimestamp());
} else if (event.getEventType().equals(ContainerMetricsConstants.FINISHED_EVENT_TYPE)) {
hasFinishedEvent = true;
replacedert.replacedertEquals(container.getFinishTime(), event.getTimestamp());
replacedert.replacedertEquals(container.getDiagnosticsInfo(), event.getEventInfo().get(ContainerMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO));
replacedert.replacedertEquals(container.getContainerExitStatus(), event.getEventInfo().get(ContainerMetricsConstants.EXIT_STATUS_EVENT_INFO));
replacedert.replacedertEquals(container.getContainerState().toString(), event.getEventInfo().get(ContainerMetricsConstants.STATE_EVENT_INFO));
}
}
replacedert.replacedertTrue(hasCreatedEvent && hasFinishedEvent);
}
16
Source : TimelineStoreTestUtils.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
/**
* Create a test enreplacedy
*/
protected static TimelineEnreplacedy createEnreplacedy(String enreplacedyId, String enreplacedyType, Long startTime, List<TimelineEvent> events, Map<String, Set<String>> relatedEnreplacedies, Map<String, Set<Object>> primaryFilters, Map<String, Object> otherInfo, String domainId) {
TimelineEnreplacedy enreplacedy = new TimelineEnreplacedy();
enreplacedy.setEnreplacedyId(enreplacedyId);
enreplacedy.setEnreplacedyType(enreplacedyType);
enreplacedy.setStartTime(startTime);
enreplacedy.setEvents(events);
if (relatedEnreplacedies != null) {
for (Entry<String, Set<String>> e : relatedEnreplacedies.entrySet()) {
for (String v : e.getValue()) {
enreplacedy.addRelatedEnreplacedy(e.getKey(), v);
}
}
} else {
enreplacedy.setRelatedEnreplacedies(null);
}
enreplacedy.setPrimaryFilters(primaryFilters);
enreplacedy.setOtherInfo(otherInfo);
enreplacedy.setDomainId(domainId);
return enreplacedy;
}
16
Source : TimelineStoreTestUtils.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
/**
* Verify a single enreplacedy
*/
protected static void verifyEnreplacedyInfo(String enreplacedyId, String enreplacedyType, List<TimelineEvent> events, Map<String, Set<String>> relatedEnreplacedies, Map<String, Set<Object>> primaryFilters, Map<String, Object> otherInfo, TimelineEnreplacedy retrievedEnreplacedyInfo, String domainId) {
if (enreplacedyId == null) {
replacedertNull(retrievedEnreplacedyInfo);
return;
}
replacedertEquals(enreplacedyId, retrievedEnreplacedyInfo.getEnreplacedyId());
replacedertEquals(enreplacedyType, retrievedEnreplacedyInfo.getEnreplacedyType());
replacedertEquals(domainId, retrievedEnreplacedyInfo.getDomainId());
if (events == null) {
replacedertNull(retrievedEnreplacedyInfo.getEvents());
} else {
replacedertEquals(events, retrievedEnreplacedyInfo.getEvents());
}
if (relatedEnreplacedies == null) {
replacedertNull(retrievedEnreplacedyInfo.getRelatedEnreplacedies());
} else {
replacedertEquals(relatedEnreplacedies, retrievedEnreplacedyInfo.getRelatedEnreplacedies());
}
if (primaryFilters == null) {
replacedertNull(retrievedEnreplacedyInfo.getPrimaryFilters());
} else {
replacedertTrue(primaryFilters.equals(retrievedEnreplacedyInfo.getPrimaryFilters()));
}
if (otherInfo == null) {
replacedertNull(retrievedEnreplacedyInfo.getOtherInfo());
} else {
replacedertTrue(otherInfo.equals(retrievedEnreplacedyInfo.getOtherInfo()));
}
}
16
Source : TestTimelineDataManager.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
@Test
public void testGetOldEnreplacedyWithOutDomainId() throws Exception {
TimelineEnreplacedy enreplacedy = dataManaer.getEnreplacedy("OLD_ENreplacedY_TYPE_1", "OLD_ENreplacedY_ID_1", null, UserGroupInformation.getCurrentUser());
replacedert.replacedertNotNull(enreplacedy);
replacedert.replacedertEquals("OLD_ENreplacedY_ID_1", enreplacedy.getEnreplacedyId());
replacedert.replacedertEquals("OLD_ENreplacedY_TYPE_1", enreplacedy.getEnreplacedyType());
replacedert.replacedertEquals(TimelineDataManager.DEFAULT_DOMAIN_ID, enreplacedy.getDomainId());
}
16
Source : TestApplicationHistoryManagerOnTimelineStore.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
private static TimelineEnreplacedy createAppAttemptTimelineEnreplacedy(ApplicationAttemptId appAttemptId) {
TimelineEnreplacedy enreplacedy = new TimelineEnreplacedy();
enreplacedy.setEnreplacedyType(AppAttemptMetricsConstants.ENreplacedY_TYPE);
enreplacedy.setEnreplacedyId(appAttemptId.toString());
enreplacedy.setDomainId(TimelineDataManager.DEFAULT_DOMAIN_ID);
enreplacedy.addPrimaryFilter(AppAttemptMetricsConstants.PARENT_PRIMARY_FILTER, appAttemptId.getApplicationId().toString());
enreplacedy.addPrimaryFilter(TimelineStore.SystemFilter.ENreplacedY_OWNER.toString(), "yarn");
TimelineEvent tEvent = new TimelineEvent();
tEvent.setEventType(AppAttemptMetricsConstants.REGISTERED_EVENT_TYPE);
tEvent.setTimestamp(Integer.MAX_VALUE + 1L);
Map<String, Object> eventInfo = new HashMap<String, Object>();
eventInfo.put(AppAttemptMetricsConstants.TRACKING_URL_EVENT_INFO, "test tracking url");
eventInfo.put(AppAttemptMetricsConstants.ORIGINAL_TRACKING_URL_EVENT_INFO, "test original tracking url");
eventInfo.put(AppAttemptMetricsConstants.HOST_EVENT_INFO, "test host");
eventInfo.put(AppAttemptMetricsConstants.RPC_PORT_EVENT_INFO, 100);
eventInfo.put(AppAttemptMetricsConstants.MASTER_CONTAINER_EVENT_INFO, ContainerId.newContainerId(appAttemptId, 1));
tEvent.setEventInfo(eventInfo);
enreplacedy.addEvent(tEvent);
tEvent = new TimelineEvent();
tEvent.setEventType(AppAttemptMetricsConstants.FINISHED_EVENT_TYPE);
tEvent.setTimestamp(Integer.MAX_VALUE + 2L);
eventInfo = new HashMap<String, Object>();
eventInfo.put(AppAttemptMetricsConstants.TRACKING_URL_EVENT_INFO, "test tracking url");
eventInfo.put(AppAttemptMetricsConstants.ORIGINAL_TRACKING_URL_EVENT_INFO, "test original tracking url");
eventInfo.put(AppAttemptMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO, "test diagnostics info");
eventInfo.put(AppAttemptMetricsConstants.FINAL_STATUS_EVENT_INFO, FinalApplicationStatus.UNDEFINED.toString());
eventInfo.put(AppAttemptMetricsConstants.STATE_EVENT_INFO, YarnApplicationAttemptState.FINISHED.toString());
tEvent.setEventInfo(eventInfo);
enreplacedy.addEvent(tEvent);
return enreplacedy;
}
16
Source : TimelineWebServices.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
/**
* Return a single enreplacedy of the given enreplacedy type and Id.
*/
@GET
@Path("/{enreplacedyType}/{enreplacedyId}")
@Produces({ MediaType.APPLICATION_JSON /* , MediaType.APPLICATION_XML */
})
public TimelineEnreplacedy getEnreplacedy(@Context HttpServletRequest req, @Context HttpServletResponse res, @PathParam("enreplacedyType") String enreplacedyType, @PathParam("enreplacedyId") String enreplacedyId, @QueryParam("fields") String fields) {
init(res);
TimelineEnreplacedy enreplacedy = null;
try {
enreplacedy = timelineDataManager.getEnreplacedy(parseStr(enreplacedyType), parseStr(enreplacedyId), parseFieldsStr(fields, ","), getUser(req));
} catch (IllegalArgumentException e) {
throw new BadRequestException("requested invalid field.");
} catch (Exception e) {
LOG.error("Error getting enreplacedy", e);
throw new WebApplicationException(e, Response.Status.INTERNAL_SERVER_ERROR);
}
if (enreplacedy == null) {
throw new NotFoundException("Timeline enreplacedy " + new EnreplacedyIdentifier(parseStr(enreplacedyId), parseStr(enreplacedyType)) + " is not found");
}
return enreplacedy;
}
16
Source : TimelineDataManager.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
/**
* Get the events whose enreplacedies the given user has access to. The meaning of
* each argument has been doreplacedented with
* {@link TimelineReader#getEnreplacedyTimelines}.
*
* @see TimelineReader#getEnreplacedyTimelines
*/
public TimelineEvents getEvents(String enreplacedyType, SortedSet<String> enreplacedyIds, SortedSet<String> eventTypes, Long windowStart, Long windowEnd, Long limit, UserGroupInformation callerUGI) throws YarnException, IOException {
TimelineEvents events = null;
events = store.getEnreplacedyTimelines(enreplacedyType, enreplacedyIds, limit, windowStart, windowEnd, eventTypes);
if (events != null) {
Iterator<TimelineEvents.EventsOfOneEnreplacedy> eventsItr = events.getAllEvents().iterator();
while (eventsItr.hasNext()) {
TimelineEvents.EventsOfOneEnreplacedy eventsOfOneEnreplacedy = eventsItr.next();
try {
TimelineEnreplacedy enreplacedy = store.getEnreplacedy(eventsOfOneEnreplacedy.getEnreplacedyId(), eventsOfOneEnreplacedy.getEnreplacedyType(), EnumSet.of(Field.PRIMARY_FILTERS));
addDefaultDomainIdIfAbsent(enreplacedy);
// check ACLs
if (!timelineACLsManager.checkAccess(callerUGI, ApplicationAccessType.VIEW_APP, enreplacedy)) {
eventsItr.remove();
}
} catch (Exception e) {
LOG.error("Error when verifying access for user " + callerUGI + " on the events of the timeline enreplacedy " + new EnreplacedyIdentifier(eventsOfOneEnreplacedy.getEnreplacedyId(), eventsOfOneEnreplacedy.getEnreplacedyType()), e);
eventsItr.remove();
}
}
}
if (events == null) {
return new TimelineEvents();
}
return events;
}
16
Source : MemoryTimelineStore.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
@Override
public synchronized TimelineEnreplacedies getEnreplacedies(String enreplacedyType, Long limit, Long windowStart, Long windowEnd, String fromId, Long fromTs, NameValuePair primaryFilter, Collection<NameValuePair> secondaryFilters, EnumSet<Field> fields, CheckAcl checkAcl) throws IOException {
if (limit == null) {
limit = DEFAULT_LIMIT;
}
if (windowStart == null) {
windowStart = Long.MIN_VALUE;
}
if (windowEnd == null) {
windowEnd = Long.MAX_VALUE;
}
if (fields == null) {
fields = EnumSet.allOf(Field.clreplaced);
}
Iterator<TimelineEnreplacedy> enreplacedyIterator = null;
if (fromId != null) {
TimelineEnreplacedy firstEnreplacedy = enreplacedies.get(new EnreplacedyIdentifier(fromId, enreplacedyType));
if (firstEnreplacedy == null) {
return new TimelineEnreplacedies();
} else {
enreplacedyIterator = new TreeSet<TimelineEnreplacedy>(enreplacedies.values()).tailSet(firstEnreplacedy, true).iterator();
}
}
if (enreplacedyIterator == null) {
enreplacedyIterator = new PriorityQueue<TimelineEnreplacedy>(enreplacedies.values()).iterator();
}
List<TimelineEnreplacedy> enreplacediesSelected = new ArrayList<TimelineEnreplacedy>();
while (enreplacedyIterator.hasNext()) {
TimelineEnreplacedy enreplacedy = enreplacedyIterator.next();
if (enreplacediesSelected.size() >= limit) {
break;
}
if (!enreplacedy.getEnreplacedyType().equals(enreplacedyType)) {
continue;
}
if (enreplacedy.getStartTime() <= windowStart) {
continue;
}
if (enreplacedy.getStartTime() > windowEnd) {
continue;
}
if (fromTs != null && enreplacedyInsertTimes.get(new EnreplacedyIdentifier(enreplacedy.getEnreplacedyId(), enreplacedy.getEnreplacedyType())) > fromTs) {
continue;
}
if (primaryFilter != null && !matchPrimaryFilter(enreplacedy.getPrimaryFilters(), primaryFilter)) {
continue;
}
if (secondaryFilters != null) {
// AND logic
boolean flag = true;
for (NameValuePair secondaryFilter : secondaryFilters) {
if (secondaryFilter != null && !matchPrimaryFilter(enreplacedy.getPrimaryFilters(), secondaryFilter) && !matchFilter(enreplacedy.getOtherInfo(), secondaryFilter)) {
flag = false;
break;
}
}
if (!flag) {
continue;
}
}
if (enreplacedy.getDomainId() == null) {
enreplacedy.setDomainId(DEFAULT_DOMAIN_ID);
}
if (checkAcl == null || checkAcl.check(enreplacedy)) {
enreplacediesSelected.add(enreplacedy);
}
}
List<TimelineEnreplacedy> enreplacediesToReturn = new ArrayList<TimelineEnreplacedy>();
for (TimelineEnreplacedy enreplacedySelected : enreplacediesSelected) {
enreplacediesToReturn.add(maskFields(enreplacedySelected, fields));
}
Collections.sort(enreplacediesToReturn);
TimelineEnreplacedies enreplacediesWrapper = new TimelineEnreplacedies();
enreplacediesWrapper.setEnreplacedies(enreplacediesToReturn);
return enreplacediesWrapper;
}
16
Source : ApplicationHistoryManagerOnTimelineStore.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
@Override
public Map<ApplicationId, ApplicationReport> getApplications(long appsNum) throws YarnException, IOException {
TimelineEnreplacedies enreplacedies = timelineDataManager.getEnreplacedies(ApplicationMetricsConstants.ENreplacedY_TYPE, null, null, null, null, null, null, appsNum == Long.MAX_VALUE ? this.maxLoadedApplications : appsNum, EnumSet.allOf(Field.clreplaced), UserGroupInformation.getLoginUser());
Map<ApplicationId, ApplicationReport> apps = new LinkedHashMap<ApplicationId, ApplicationReport>();
if (enreplacedies != null && enreplacedies.getEnreplacedies() != null) {
for (TimelineEnreplacedy enreplacedy : enreplacedies.getEnreplacedies()) {
ApplicationReportExt app = generateApplicationReport(enreplacedy, ApplicationReportField.ALL);
apps.put(app.appReport.getApplicationId(), app.appReport);
}
}
return apps;
}
16
Source : ApplicationHistoryManagerOnTimelineStore.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
private ApplicationReportExt generateApplicationReport(TimelineEnreplacedy enreplacedy, ApplicationReportField field) throws YarnException, IOException {
ApplicationReportExt app = convertToApplicationReport(enreplacedy, field);
// If only user and acls are pulled to check attempt(s)/container(s) access
// control, we can return immediately
if (field == ApplicationReportField.USER_AND_ACLS) {
return app;
}
try {
checkAccess(app);
if (app.appReport.getCurrentApplicationAttemptId() != null) {
ApplicationAttemptReport appAttempt = getApplicationAttempt(app.appReport.getCurrentApplicationAttemptId());
app.appReport.setHost(appAttempt.getHost());
app.appReport.setRpcPort(appAttempt.getRpcPort());
app.appReport.setTrackingUrl(appAttempt.getTrackingUrl());
app.appReport.setOriginalTrackingUrl(appAttempt.getOriginalTrackingUrl());
}
} catch (AuthorizationException e) {
// AuthorizationException is thrown because the user doesn't have access
app.appReport.setDiagnostics(null);
app.appReport.setCurrentApplicationAttemptId(null);
} catch (ApplicationAttemptNotFoundException e) {
// It's possible that the app is finished before the first attempt is created.
app.appReport.setDiagnostics(null);
app.appReport.setCurrentApplicationAttemptId(null);
}
if (app.appReport.getCurrentApplicationAttemptId() == null) {
app.appReport.setCurrentApplicationAttemptId(ApplicationAttemptId.newInstance(app.appReport.getApplicationId(), -1));
}
if (app.appReport.getHost() == null) {
app.appReport.setHost(UNAVAILABLE);
}
if (app.appReport.getRpcPort() < 0) {
app.appReport.setRpcPort(-1);
}
if (app.appReport.getTrackingUrl() == null) {
app.appReport.setTrackingUrl(UNAVAILABLE);
}
if (app.appReport.getOriginalTrackingUrl() == null) {
app.appReport.setOriginalTrackingUrl(UNAVAILABLE);
}
if (app.appReport.getDiagnostics() == null) {
app.appReport.setDiagnostics("");
}
return app;
}
16
Source : ApplicationHistoryManagerOnTimelineStore.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
private ApplicationReportExt getApplication(ApplicationId appId, ApplicationReportField field) throws YarnException, IOException {
TimelineEnreplacedy enreplacedy = timelineDataManager.getEnreplacedy(ApplicationMetricsConstants.ENreplacedY_TYPE, appId.toString(), EnumSet.allOf(Field.clreplaced), UserGroupInformation.getLoginUser());
if (enreplacedy == null) {
throw new ApplicationNotFoundException("The enreplacedy for application " + appId + " doesn't exist in the timeline store");
} else {
return generateApplicationReport(enreplacedy, field);
}
}
16
Source : TestSystemMetricsPublisher.java
with Apache License 2.0
from naver
with Apache License 2.0
from naver
@Test(timeout = 10000)
public void testPublishApplicationMetrics() throws Exception {
for (int i = 1; i <= 2; ++i) {
ApplicationId appId = ApplicationId.newInstance(0, i);
RMApp app = createRMApp(appId);
metricsPublisher.appCreated(app, app.getStartTime());
metricsPublisher.appFinished(app, RMAppState.FINISHED, app.getFinishTime());
if (i == 1) {
metricsPublisher.appACLsUpdated(app, "uers1,user2", 4L);
} else {
// in case user doesn't specify the ACLs
metricsPublisher.appACLsUpdated(app, null, 4L);
}
TimelineEnreplacedy enreplacedy = null;
do {
enreplacedy = store.getEnreplacedy(appId.toString(), ApplicationMetricsConstants.ENreplacedY_TYPE, EnumSet.allOf(Field.clreplaced));
// ensure three events are both published before leaving the loop
} while (enreplacedy == null || enreplacedy.getEvents().size() < 3);
// verify all the fields
replacedert.replacedertEquals(ApplicationMetricsConstants.ENreplacedY_TYPE, enreplacedy.getEnreplacedyType());
replacedert.replacedertEquals(app.getApplicationId().toString(), enreplacedy.getEnreplacedyId());
replacedert.replacedertEquals(app.getName(), enreplacedy.getOtherInfo().get(ApplicationMetricsConstants.NAME_ENreplacedY_INFO));
replacedert.replacedertEquals(app.getQueue(), enreplacedy.getOtherInfo().get(ApplicationMetricsConstants.QUEUE_ENreplacedY_INFO));
replacedert.replacedertEquals(app.getUser(), enreplacedy.getOtherInfo().get(ApplicationMetricsConstants.USER_ENreplacedY_INFO));
replacedert.replacedertEquals(app.getApplicationType(), enreplacedy.getOtherInfo().get(ApplicationMetricsConstants.TYPE_ENreplacedY_INFO));
replacedert.replacedertEquals(app.getSubmitTime(), enreplacedy.getOtherInfo().get(ApplicationMetricsConstants.SUBMITTED_TIME_ENreplacedY_INFO));
if (i == 1) {
replacedert.replacedertEquals("uers1,user2", enreplacedy.getOtherInfo().get(ApplicationMetricsConstants.APP_VIEW_ACLS_ENreplacedY_INFO));
} else {
replacedert.replacedertEquals("", enreplacedy.getOtherInfo().get(ApplicationMetricsConstants.APP_VIEW_ACLS_ENreplacedY_INFO));
replacedert.replacedertEquals(app.getRMAppMetrics().getMemorySeconds(), Long.parseLong(enreplacedy.getOtherInfo().get(ApplicationMetricsConstants.APP_MEM_METRICS).toString()));
replacedert.replacedertEquals(app.getRMAppMetrics().getVcoreSeconds(), Long.parseLong(enreplacedy.getOtherInfo().get(ApplicationMetricsConstants.APP_CPU_METRICS).toString()));
replacedert.replacedertEquals(app.getRMAppMetrics().getGcoreSeconds(), Long.parseLong(enreplacedy.getOtherInfo().get(ApplicationMetricsConstants.APP_GPU_METRICS).toString()));
}
boolean hasCreatedEvent = false;
boolean hasFinishedEvent = false;
boolean hasACLsUpdatedEvent = false;
for (TimelineEvent event : enreplacedy.getEvents()) {
if (event.getEventType().equals(ApplicationMetricsConstants.CREATED_EVENT_TYPE)) {
hasCreatedEvent = true;
replacedert.replacedertEquals(app.getStartTime(), event.getTimestamp());
} else if (event.getEventType().equals(ApplicationMetricsConstants.FINISHED_EVENT_TYPE)) {
hasFinishedEvent = true;
replacedert.replacedertEquals(app.getFinishTime(), event.getTimestamp());
replacedert.replacedertEquals(app.getDiagnostics().toString(), event.getEventInfo().get(ApplicationMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO));
replacedert.replacedertEquals(app.getFinalApplicationStatus().toString(), event.getEventInfo().get(ApplicationMetricsConstants.FINAL_STATUS_EVENT_INFO));
replacedert.replacedertEquals(YarnApplicationState.FINISHED.toString(), event.getEventInfo().get(ApplicationMetricsConstants.STATE_EVENT_INFO));
} else if (event.getEventType().equals(ApplicationMetricsConstants.ACLS_UPDATED_EVENT_TYPE)) {
hasACLsUpdatedEvent = true;
replacedert.replacedertEquals(4L, event.getTimestamp());
}
}
replacedert.replacedertTrue(hasCreatedEvent && hasFinishedEvent && hasACLsUpdatedEvent);
}
}
16
Source : TestSystemMetricsPublisher.java
with Apache License 2.0
from naver
with Apache License 2.0
from naver
@Test(timeout = 10000)
public void testPublishContainerMetrics() throws Exception {
ContainerId containerId = ContainerId.newContainerId(ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 1), 1), 1);
RMContainer container = createRMContainer(containerId);
metricsPublisher.containerCreated(container, container.getCreationTime());
metricsPublisher.containerFinished(container, container.getFinishTime());
TimelineEnreplacedy enreplacedy = null;
do {
enreplacedy = store.getEnreplacedy(containerId.toString(), ContainerMetricsConstants.ENreplacedY_TYPE, EnumSet.allOf(Field.clreplaced));
// ensure two events are both published before leaving the loop
} while (enreplacedy == null || enreplacedy.getEvents().size() < 2);
// verify all the fields
replacedert.replacedertEquals(ContainerMetricsConstants.ENreplacedY_TYPE, enreplacedy.getEnreplacedyType());
replacedert.replacedertEquals(containerId.toString(), enreplacedy.getEnreplacedyId());
replacedert.replacedertEquals(containerId.getApplicationAttemptId().toString(), enreplacedy.getPrimaryFilters().get(ContainerMetricsConstants.PARENT_PRIMARIY_FILTER).iterator().next());
replacedert.replacedertEquals(container.getAllocatedNode().getHost(), enreplacedy.getOtherInfo().get(ContainerMetricsConstants.ALLOCATED_HOST_ENreplacedY_INFO));
replacedert.replacedertEquals(container.getAllocatedNode().getPort(), enreplacedy.getOtherInfo().get(ContainerMetricsConstants.ALLOCATED_PORT_ENreplacedY_INFO));
replacedert.replacedertEquals(container.getAllocatedResource().getMemory(), enreplacedy.getOtherInfo().get(ContainerMetricsConstants.ALLOCATED_MEMORY_ENreplacedY_INFO));
replacedert.replacedertEquals(container.getAllocatedResource().getVirtualCores(), enreplacedy.getOtherInfo().get(ContainerMetricsConstants.ALLOCATED_VCORE_ENreplacedY_INFO));
replacedert.replacedertEquals(container.getAllocatedResource().getGpuCores(), enreplacedy.getOtherInfo().get(ContainerMetricsConstants.ALLOCATED_GCORE_ENreplacedY_INFO));
replacedert.replacedertEquals(container.getAllocatedPriority().getPriority(), enreplacedy.getOtherInfo().get(ContainerMetricsConstants.ALLOCATED_PRIORITY_ENreplacedY_INFO));
boolean hasCreatedEvent = false;
boolean hasFinishedEvent = false;
for (TimelineEvent event : enreplacedy.getEvents()) {
if (event.getEventType().equals(ContainerMetricsConstants.CREATED_EVENT_TYPE)) {
hasCreatedEvent = true;
replacedert.replacedertEquals(container.getCreationTime(), event.getTimestamp());
} else if (event.getEventType().equals(ContainerMetricsConstants.FINISHED_EVENT_TYPE)) {
hasFinishedEvent = true;
replacedert.replacedertEquals(container.getFinishTime(), event.getTimestamp());
replacedert.replacedertEquals(container.getDiagnosticsInfo(), event.getEventInfo().get(ContainerMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO));
replacedert.replacedertEquals(container.getContainerExitStatus(), event.getEventInfo().get(ContainerMetricsConstants.EXIT_STATUS_EVENT_INFO));
replacedert.replacedertEquals(container.getContainerState().toString(), event.getEventInfo().get(ContainerMetricsConstants.STATE_EVENT_INFO));
}
}
replacedert.replacedertTrue(hasCreatedEvent && hasFinishedEvent);
}
16
Source : ApplicationHistoryManagerOnTimelineStore.java
with Apache License 2.0
from naver
with Apache License 2.0
from naver
private static ApplicationReportExt convertToApplicationReport(TimelineEnreplacedy enreplacedy, ApplicationReportField field) {
String user = null;
String queue = null;
String name = null;
String type = null;
long createdTime = 0;
long finishedTime = 0;
ApplicationAttemptId latestApplicationAttemptId = null;
String diagnosticsInfo = null;
FinalApplicationStatus finalStatus = FinalApplicationStatus.UNDEFINED;
YarnApplicationState state = null;
ApplicationResourceUsageReport appResources = null;
Map<ApplicationAccessType, String> appViewACLs = new HashMap<ApplicationAccessType, String>();
Map<String, Object> enreplacedyInfo = enreplacedy.getOtherInfo();
if (enreplacedyInfo != null) {
if (enreplacedyInfo.containsKey(ApplicationMetricsConstants.USER_ENreplacedY_INFO)) {
user = enreplacedyInfo.get(ApplicationMetricsConstants.USER_ENreplacedY_INFO).toString();
}
if (enreplacedyInfo.containsKey(ApplicationMetricsConstants.APP_VIEW_ACLS_ENreplacedY_INFO)) {
String appViewACLsStr = enreplacedyInfo.get(ApplicationMetricsConstants.APP_VIEW_ACLS_ENreplacedY_INFO).toString();
if (appViewACLsStr.length() > 0) {
appViewACLs.put(ApplicationAccessType.VIEW_APP, appViewACLsStr);
}
}
if (field == ApplicationReportField.USER_AND_ACLS) {
return new ApplicationReportExt(ApplicationReport.newInstance(ConverterUtils.toApplicationId(enreplacedy.getEnreplacedyId()), latestApplicationAttemptId, user, queue, name, null, -1, null, state, diagnosticsInfo, null, createdTime, finishedTime, finalStatus, null, null, 1.0F, type, null), appViewACLs);
}
if (enreplacedyInfo.containsKey(ApplicationMetricsConstants.QUEUE_ENreplacedY_INFO)) {
queue = enreplacedyInfo.get(ApplicationMetricsConstants.QUEUE_ENreplacedY_INFO).toString();
}
if (enreplacedyInfo.containsKey(ApplicationMetricsConstants.NAME_ENreplacedY_INFO)) {
name = enreplacedyInfo.get(ApplicationMetricsConstants.NAME_ENreplacedY_INFO).toString();
}
if (enreplacedyInfo.containsKey(ApplicationMetricsConstants.TYPE_ENreplacedY_INFO)) {
type = enreplacedyInfo.get(ApplicationMetricsConstants.TYPE_ENreplacedY_INFO).toString();
}
if (enreplacedyInfo.containsKey(ApplicationMetricsConstants.APP_CPU_METRICS)) {
long vcoreSeconds = Long.parseLong(enreplacedyInfo.get(ApplicationMetricsConstants.APP_CPU_METRICS).toString());
long memorySeconds = Long.parseLong(enreplacedyInfo.get(ApplicationMetricsConstants.APP_MEM_METRICS).toString());
long gcoreSeconds = Long.parseLong(enreplacedyInfo.get(ApplicationMetricsConstants.APP_GPU_METRICS).toString());
appResources = ApplicationResourceUsageReport.newInstance(0, 0, null, null, null, memorySeconds, vcoreSeconds, gcoreSeconds);
}
}
List<TimelineEvent> events = enreplacedy.getEvents();
if (events != null) {
for (TimelineEvent event : events) {
if (event.getEventType().equals(ApplicationMetricsConstants.CREATED_EVENT_TYPE)) {
createdTime = event.getTimestamp();
} else if (event.getEventType().equals(ApplicationMetricsConstants.FINISHED_EVENT_TYPE)) {
finishedTime = event.getTimestamp();
Map<String, Object> eventInfo = event.getEventInfo();
if (eventInfo == null) {
continue;
}
if (eventInfo.containsKey(ApplicationMetricsConstants.LATEST_APP_ATTEMPT_EVENT_INFO)) {
latestApplicationAttemptId = ConverterUtils.toApplicationAttemptId(eventInfo.get(ApplicationMetricsConstants.LATEST_APP_ATTEMPT_EVENT_INFO).toString());
}
if (eventInfo.containsKey(ApplicationMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO)) {
diagnosticsInfo = eventInfo.get(ApplicationMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO).toString();
}
if (eventInfo.containsKey(ApplicationMetricsConstants.FINAL_STATUS_EVENT_INFO)) {
finalStatus = FinalApplicationStatus.valueOf(eventInfo.get(ApplicationMetricsConstants.FINAL_STATUS_EVENT_INFO).toString());
}
if (eventInfo.containsKey(ApplicationMetricsConstants.STATE_EVENT_INFO)) {
state = YarnApplicationState.valueOf(eventInfo.get(ApplicationMetricsConstants.STATE_EVENT_INFO).toString());
}
}
}
}
return new ApplicationReportExt(ApplicationReport.newInstance(ConverterUtils.toApplicationId(enreplacedy.getEnreplacedyId()), latestApplicationAttemptId, user, queue, name, null, -1, null, state, diagnosticsInfo, null, createdTime, finishedTime, finalStatus, appResources, null, 1.0F, type, null), appViewACLs);
}
15
Source : SystemMetricsPublisher.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
private void publishAppAttemptRegisteredEvent(AppAttemptRegisteredEvent event) {
TimelineEnreplacedy enreplacedy = createAppAttemptEnreplacedy(event.getApplicationAttemptId());
TimelineEvent tEvent = new TimelineEvent();
tEvent.setEventType(AppAttemptMetricsConstants.REGISTERED_EVENT_TYPE);
tEvent.setTimestamp(event.getTimestamp());
Map<String, Object> eventInfo = new HashMap<String, Object>();
eventInfo.put(AppAttemptMetricsConstants.TRACKING_URL_EVENT_INFO, event.getTrackingUrl());
eventInfo.put(AppAttemptMetricsConstants.ORIGINAL_TRACKING_URL_EVENT_INFO, event.getOriginalTrackingURL());
eventInfo.put(AppAttemptMetricsConstants.HOST_EVENT_INFO, event.getHost());
eventInfo.put(AppAttemptMetricsConstants.RPC_PORT_EVENT_INFO, event.getRpcPort());
eventInfo.put(AppAttemptMetricsConstants.MASTER_CONTAINER_EVENT_INFO, event.getMasterContainerId().toString());
tEvent.setEventInfo(eventInfo);
enreplacedy.addEvent(tEvent);
putEnreplacedy(enreplacedy);
}
15
Source : SystemMetricsPublisher.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
private void publishContainerCreatedEvent(ContainerCreatedEvent event) {
TimelineEnreplacedy enreplacedy = createContainerEnreplacedy(event.getContainerId());
Map<String, Object> enreplacedyInfo = new HashMap<String, Object>();
enreplacedyInfo.put(ContainerMetricsConstants.ALLOCATED_MEMORY_ENreplacedY_INFO, event.getAllocatedResource().getMemory());
enreplacedyInfo.put(ContainerMetricsConstants.ALLOCATED_VCORE_ENreplacedY_INFO, event.getAllocatedResource().getVirtualCores());
enreplacedyInfo.put(ContainerMetricsConstants.ALLOCATED_HOST_ENreplacedY_INFO, event.getAllocatedNode().getHost());
enreplacedyInfo.put(ContainerMetricsConstants.ALLOCATED_PORT_ENreplacedY_INFO, event.getAllocatedNode().getPort());
enreplacedyInfo.put(ContainerMetricsConstants.ALLOCATED_PRIORITY_ENreplacedY_INFO, event.getAllocatedPriority().getPriority());
enreplacedyInfo.put(ContainerMetricsConstants.ALLOCATED_HOST_HTTP_ADDRESS_ENreplacedY_INFO, event.getNodeHttpAddress());
enreplacedy.setOtherInfo(enreplacedyInfo);
TimelineEvent tEvent = new TimelineEvent();
tEvent.setEventType(ContainerMetricsConstants.CREATED_EVENT_TYPE);
tEvent.setTimestamp(event.getTimestamp());
enreplacedy.addEvent(tEvent);
putEnreplacedy(enreplacedy);
}
15
Source : SystemMetricsPublisher.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
private void publishApplicationCreatedEvent(ApplicationCreatedEvent event) {
TimelineEnreplacedy enreplacedy = createApplicationEnreplacedy(event.getApplicationId());
Map<String, Object> enreplacedyInfo = new HashMap<String, Object>();
enreplacedyInfo.put(ApplicationMetricsConstants.NAME_ENreplacedY_INFO, event.getApplicationName());
enreplacedyInfo.put(ApplicationMetricsConstants.TYPE_ENreplacedY_INFO, event.getApplicationType());
enreplacedyInfo.put(ApplicationMetricsConstants.USER_ENreplacedY_INFO, event.getUser());
enreplacedyInfo.put(ApplicationMetricsConstants.QUEUE_ENreplacedY_INFO, event.getQueue());
enreplacedyInfo.put(ApplicationMetricsConstants.SUBMITTED_TIME_ENreplacedY_INFO, event.getSubmittedTime());
enreplacedy.setOtherInfo(enreplacedyInfo);
TimelineEvent tEvent = new TimelineEvent();
tEvent.setEventType(ApplicationMetricsConstants.CREATED_EVENT_TYPE);
tEvent.setTimestamp(event.getTimestamp());
enreplacedy.addEvent(tEvent);
putEnreplacedy(enreplacedy);
}
15
Source : TestLeveldbTimelineStore.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
@Test
public void testRelatingToNonExistingEnreplacedy() throws IOException {
TimelineEnreplacedy enreplacedyToStore = new TimelineEnreplacedy();
enreplacedyToStore.setEnreplacedyType("TEST_ENreplacedY_TYPE_1");
enreplacedyToStore.setEnreplacedyId("TEST_ENreplacedY_ID_1");
enreplacedyToStore.setDomainId(TimelineDataManager.DEFAULT_DOMAIN_ID);
enreplacedyToStore.addRelatedEnreplacedy("TEST_ENreplacedY_TYPE_2", "TEST_ENreplacedY_ID_2");
TimelineEnreplacedies enreplacedies = new TimelineEnreplacedies();
enreplacedies.addEnreplacedy(enreplacedyToStore);
store.put(enreplacedies);
TimelineEnreplacedy enreplacedyToGet = store.getEnreplacedy("TEST_ENreplacedY_ID_2", "TEST_ENreplacedY_TYPE_2", null);
replacedert.replacedertNotNull(enreplacedyToGet);
replacedert.replacedertEquals("DEFAULT", enreplacedyToGet.getDomainId());
replacedert.replacedertEquals("TEST_ENreplacedY_TYPE_1", enreplacedyToGet.getRelatedEnreplacedies().keySet().iterator().next());
replacedert.replacedertEquals("TEST_ENreplacedY_ID_1", enreplacedyToGet.getRelatedEnreplacedies().values().iterator().next().iterator().next());
}
15
Source : TestApplicationHistoryManagerOnTimelineStore.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
private static TimelineEnreplacedy createApplicationTimelineEnreplacedy(ApplicationId appId, boolean emptyACLs, boolean noAttempt) {
TimelineEnreplacedy enreplacedy = new TimelineEnreplacedy();
enreplacedy.setEnreplacedyType(ApplicationMetricsConstants.ENreplacedY_TYPE);
enreplacedy.setEnreplacedyId(appId.toString());
enreplacedy.setDomainId(TimelineDataManager.DEFAULT_DOMAIN_ID);
enreplacedy.addPrimaryFilter(TimelineStore.SystemFilter.ENreplacedY_OWNER.toString(), "yarn");
Map<String, Object> enreplacedyInfo = new HashMap<String, Object>();
enreplacedyInfo.put(ApplicationMetricsConstants.NAME_ENreplacedY_INFO, "test app");
enreplacedyInfo.put(ApplicationMetricsConstants.TYPE_ENreplacedY_INFO, "test app type");
enreplacedyInfo.put(ApplicationMetricsConstants.USER_ENreplacedY_INFO, "user1");
enreplacedyInfo.put(ApplicationMetricsConstants.QUEUE_ENreplacedY_INFO, "test queue");
enreplacedyInfo.put(ApplicationMetricsConstants.SUBMITTED_TIME_ENreplacedY_INFO, Integer.MAX_VALUE + 1L);
if (emptyACLs) {
enreplacedyInfo.put(ApplicationMetricsConstants.APP_VIEW_ACLS_ENreplacedY_INFO, "");
} else {
enreplacedyInfo.put(ApplicationMetricsConstants.APP_VIEW_ACLS_ENreplacedY_INFO, "user2");
}
enreplacedy.setOtherInfo(enreplacedyInfo);
TimelineEvent tEvent = new TimelineEvent();
tEvent.setEventType(ApplicationMetricsConstants.CREATED_EVENT_TYPE);
tEvent.setTimestamp(Integer.MAX_VALUE + 2L + appId.getId());
enreplacedy.addEvent(tEvent);
tEvent = new TimelineEvent();
tEvent.setEventType(ApplicationMetricsConstants.FINISHED_EVENT_TYPE);
tEvent.setTimestamp(Integer.MAX_VALUE + 3L + appId.getId());
Map<String, Object> eventInfo = new HashMap<String, Object>();
eventInfo.put(ApplicationMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO, "test diagnostics info");
eventInfo.put(ApplicationMetricsConstants.FINAL_STATUS_EVENT_INFO, FinalApplicationStatus.UNDEFINED.toString());
eventInfo.put(ApplicationMetricsConstants.STATE_EVENT_INFO, YarnApplicationState.FINISHED.toString());
if (noAttempt) {
eventInfo.put(ApplicationMetricsConstants.LATEST_APP_ATTEMPT_EVENT_INFO, ApplicationAttemptId.newInstance(appId, 1));
}
tEvent.setEventInfo(eventInfo);
enreplacedy.addEvent(tEvent);
return enreplacedy;
}
15
Source : TestApplicationHistoryManagerOnTimelineStore.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
private static TimelineEnreplacedy createContainerEnreplacedy(ContainerId containerId) {
TimelineEnreplacedy enreplacedy = new TimelineEnreplacedy();
enreplacedy.setEnreplacedyType(ContainerMetricsConstants.ENreplacedY_TYPE);
enreplacedy.setEnreplacedyId(containerId.toString());
enreplacedy.setDomainId(TimelineDataManager.DEFAULT_DOMAIN_ID);
enreplacedy.addPrimaryFilter(ContainerMetricsConstants.PARENT_PRIMARIY_FILTER, containerId.getApplicationAttemptId().toString());
enreplacedy.addPrimaryFilter(TimelineStore.SystemFilter.ENreplacedY_OWNER.toString(), "yarn");
Map<String, Object> enreplacedyInfo = new HashMap<String, Object>();
enreplacedyInfo.put(ContainerMetricsConstants.ALLOCATED_MEMORY_ENreplacedY_INFO, -1);
enreplacedyInfo.put(ContainerMetricsConstants.ALLOCATED_VCORE_ENreplacedY_INFO, -1);
enreplacedyInfo.put(ContainerMetricsConstants.ALLOCATED_HOST_ENreplacedY_INFO, "test host");
enreplacedyInfo.put(ContainerMetricsConstants.ALLOCATED_PORT_ENreplacedY_INFO, 100);
enreplacedyInfo.put(ContainerMetricsConstants.ALLOCATED_PRIORITY_ENreplacedY_INFO, -1);
enreplacedy.setOtherInfo(enreplacedyInfo);
TimelineEvent tEvent = new TimelineEvent();
tEvent.setEventType(ContainerMetricsConstants.CREATED_EVENT_TYPE);
tEvent.setTimestamp(Integer.MAX_VALUE + 1L);
enreplacedy.addEvent(tEvent);
;
tEvent = new TimelineEvent();
tEvent.setEventType(ContainerMetricsConstants.FINISHED_EVENT_TYPE);
tEvent.setTimestamp(Integer.MAX_VALUE + 2L);
Map<String, Object> eventInfo = new HashMap<String, Object>();
eventInfo.put(ContainerMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO, "test diagnostics info");
eventInfo.put(ContainerMetricsConstants.EXIT_STATUS_EVENT_INFO, -1);
eventInfo.put(ContainerMetricsConstants.STATE_EVENT_INFO, ContainerState.COMPLETE.toString());
tEvent.setEventInfo(eventInfo);
enreplacedy.addEvent(tEvent);
return enreplacedy;
}
15
Source : ApplicationMaster.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
private static void publishContainerStartEvent(final TimelineClient timelineClient, Container container, String domainId, UserGroupInformation ugi) {
final TimelineEnreplacedy enreplacedy = new TimelineEnreplacedy();
enreplacedy.setEnreplacedyId(container.getId().toString());
enreplacedy.setEnreplacedyType(DSEnreplacedy.DS_CONTAINER.toString());
enreplacedy.setDomainId(domainId);
enreplacedy.addPrimaryFilter("user", ugi.getShortUserName());
TimelineEvent event = new TimelineEvent();
event.setTimestamp(System.currentTimeMillis());
event.setEventType(DSEvent.DS_CONTAINER_START.toString());
event.addEventInfo("Node", container.getNodeId().toString());
event.addEventInfo("Resources", container.getResource().toString());
enreplacedy.addEvent(event);
try {
ugi.doAs(new PrivilegedExceptionAction<TimelinePutResponse>() {
@Override
public TimelinePutResponse run() throws Exception {
return timelineClient.putEnreplacedies(enreplacedy);
}
});
} catch (Exception e) {
LOG.error("Container start event could not be published for " + container.getId().toString(), e instanceof UndeclaredThrowableException ? e.getCause() : e);
}
}
See More Examples