Here are the examples of the java api org.apache.hadoop.security.Credentials taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.
234 Examples
19
Source : DrillFileSystem.java
with Apache License 2.0
from zpochen
with Apache License 2.0
from zpochen
@Override
@LimitedPrivate({ "HDFS", "MapReduce" })
public Token<?>[] addDelegationTokens(String renewer, Credentials credentials) throws IOException {
return underlyingFs.addDelegationTokens(renewer, credentials);
}
19
Source : MockRM.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
public RMApp submitApp(int masterMemory, String name, String user, Map<ApplicationAccessType, String> acls, boolean unmanaged, String queue, int maxAppAttempts, Credentials ts, String appType) throws Exception {
return submitApp(masterMemory, name, user, acls, unmanaged, queue, maxAppAttempts, ts, appType, true);
}
19
Source : MockRM.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
public RMApp submitApp(int masterMemory, String name, String user, Map<ApplicationAccessType, String> acls, boolean unmanaged, String queue, int maxAppAttempts, Credentials ts, String appType, boolean waitForAccepted) throws Exception {
return submitApp(masterMemory, name, user, acls, unmanaged, queue, maxAppAttempts, ts, appType, waitForAccepted, false);
}
19
Source : MockRM.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
public RMApp submitApp(int masterMemory, String name, String user, Map<ApplicationAccessType, String> acls, boolean unmanaged, String queue, int maxAppAttempts, Credentials ts) throws Exception {
return submitApp(masterMemory, name, user, acls, unmanaged, queue, maxAppAttempts, ts, null);
}
19
Source : MockRM.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
public RMApp submitApp(int masterMemory, String name, String user, Map<ApplicationAccessType, String> acls, boolean unmanaged, String queue, int maxAppAttempts, Credentials ts, String appType, boolean waitForAccepted, boolean keepContainers, boolean isAppIdProvided, ApplicationId applicationId) throws Exception {
return submitApp(masterMemory, name, user, acls, unmanaged, queue, maxAppAttempts, ts, appType, waitForAccepted, keepContainers, isAppIdProvided, applicationId, 0, null, true);
}
19
Source : MockRM.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
public RMApp submitApp(int masterMemory, String name, String user, Map<ApplicationAccessType, String> acls, boolean unmanaged, String queue, int maxAppAttempts, Credentials ts, String appType, boolean waitForAccepted, boolean keepContainers) throws Exception {
return submitApp(masterMemory, name, user, acls, unmanaged, queue, maxAppAttempts, ts, appType, waitForAccepted, keepContainers, false, null, 0, null, true);
}
19
Source : DelegationTokenRenewer.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
/**
* Synchronously renew delegation tokens.
* @param user user
*/
public void addApplicationSync(ApplicationId applicationId, Credentials ts, boolean shouldCancelAtEnd, String user) throws IOException, InterruptedException {
handleAppSubmitEvent(new DelegationTokenRenewerAppSubmitEvent(applicationId, ts, shouldCancelAtEnd, user));
}
19
Source : DelegationTokenRenewer.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
/**
* Asynchronously add application tokens for renewal.
* @param applicationId added application
* @param ts tokens
* @param shouldCancelAtEnd true if tokens should be canceled when the app is
* done else false.
* @param user user
* @throws IOException
*/
public void addApplicationAsync(ApplicationId applicationId, Credentials ts, boolean shouldCancelAtEnd, String user) {
processDelegationTokenRenewerEvent(new DelegationTokenRenewerAppSubmitEvent(applicationId, ts, shouldCancelAtEnd, user));
}
19
Source : LogAggregationService.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
@SuppressWarnings("unchecked")
private void initApp(final ApplicationId appId, String user, Credentials credentials, ContainerLogsRetentionPolicy logRetentionPolicy, Map<ApplicationAccessType, String> appAcls, LogAggregationContext logAggregationContext) {
ApplicationEvent eventResponse;
try {
verifyAndCreateRemoteLogDir(getConfig());
initAppAggregator(appId, user, credentials, logRetentionPolicy, appAcls, logAggregationContext);
eventResponse = new ApplicationEvent(appId, ApplicationEventType.APPLICATION_LOG_HANDLING_INITED);
} catch (YarnRuntimeException e) {
LOG.warn("Application failed to init aggregation", e);
eventResponse = new ApplicationEvent(appId, ApplicationEventType.APPLICATION_LOG_HANDLING_FAILED);
}
this.dispatcher.getEventHandler().handle(eventResponse);
}
19
Source : ContainerManagerImpl.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
private Credentials parseCredentials(ContainerLaunchContext launchContext) throws IOException {
Credentials credentials = new Credentials();
// //////////// Parse credentials
ByteBuffer tokens = launchContext.getTokens();
if (tokens != null) {
DataInputByteBuffer buf = new DataInputByteBuffer();
tokens.rewind();
buf.reset(tokens);
credentials.readTokenStorageStream(buf);
if (LOG.isDebugEnabled()) {
for (Token<? extends TokenIdentifier> tk : credentials.getAllTokens()) {
LOG.debug(tk.getService() + " = " + tk.toString());
}
}
}
// //////////// End of parsing credentials
return credentials;
}
19
Source : YarnClientImpl.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
private void addTimelineDelegationToken(ContainerLaunchContext clc) throws YarnException, IOException {
Credentials credentials = new Credentials();
DataInputByteBuffer dibb = new DataInputByteBuffer();
ByteBuffer tokens = clc.getTokens();
if (tokens != null) {
dibb.reset(tokens);
credentials.readTokenStorageStream(dibb);
tokens.rewind();
}
// If the timeline delegation token is already in the CLC, no need to add
// one more
for (org.apache.hadoop.security.token.Token<? extends TokenIdentifier> token : credentials.getAllTokens()) {
if (token.getKind().equals(TimelineDelegationTokenIdentifier.KIND_NAME)) {
return;
}
}
org.apache.hadoop.security.token.Token<TimelineDelegationTokenIdentifier> timelineDelegationToken = getTimelineDelegationToken();
if (timelineDelegationToken == null) {
return;
}
credentials.addToken(timelineService, timelineDelegationToken);
if (LOG.isDebugEnabled()) {
LOG.debug("Add timline delegation token into credentials: " + timelineDelegationToken);
}
DataOutputBuffer dob = new DataOutputBuffer();
credentials.writeTokenStorageToStream(dob);
tokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
clc.setTokens(tokens);
}
19
Source : CopyListing.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
/**
* The CopyListing abstraction is responsible for how the list of
* sources and targets is constructed, for DistCp's copy function.
* The copy-listing should be a SequenceFile<Text, CopyListingFileStatus>,
* located at the path specified to buildListing(),
* each entry being a pair of (Source relative path, source file status),
* all the paths being fully qualified.
*/
public abstract clreplaced CopyListing extends Configured {
private Credentials credentials;
/**
* Build listing function creates the input listing that distcp uses to
* perform the copy.
*
* The build listing is a sequence file that has relative path of a file in the key
* and the file status information of the source file in the value
*
* For instance if the source path is /tmp/data and the traversed path is
* /tmp/data/dir1/dir2/file1, then the sequence file would contain
*
* key: /dir1/dir2/file1 and value: FileStatus(/tmp/data/dir1/dir2/file1)
*
* File would also contain directory entries. Meaning, if /tmp/data/dir1/dir2/file1
* is the only file under /tmp/data, the resulting sequence file would contain the
* following entries
*
* key: /dir1 and value: FileStatus(/tmp/data/dir1)
* key: /dir1/dir2 and value: FileStatus(/tmp/data/dir1/dir2)
* key: /dir1/dir2/file1 and value: FileStatus(/tmp/data/dir1/dir2/file1)
*
* Cases requiring special handling:
* If source path is a file (/tmp/file1), contents of the file will be as follows
*
* TARGET DOES NOT EXIST: Key-"", Value-FileStatus(/tmp/file1)
* TARGET IS FILE : Key-"", Value-FileStatus(/tmp/file1)
* TARGET IS DIR : Key-"/file1", Value-FileStatus(/tmp/file1)
*
* @param pathToListFile - Output file where the listing would be stored
* @param options - Input options to distcp
* @throws IOException - Exception if any
*/
public final void buildListing(Path pathToListFile, DistCpOptions options) throws IOException {
validatePaths(options);
doBuildListing(pathToListFile, options);
Configuration config = getConf();
config.set(DistCpConstants.CONF_LABEL_LISTING_FILE_PATH, pathToListFile.toString());
config.setLong(DistCpConstants.CONF_LABEL_TOTAL_BYTES_TO_BE_COPIED, getBytesToCopy());
config.setLong(DistCpConstants.CONF_LABEL_TOTAL_NUMBER_OF_RECORDS, getNumberOfPaths());
validateFinalListing(pathToListFile, options);
}
/**
* Validate input and output paths
*
* @param options - Input options
* @throws InvalidInputException: If inputs are invalid
* @throws IOException: any Exception with FS
*/
protected abstract void validatePaths(DistCpOptions options) throws IOException, InvalidInputException;
/**
* The interface to be implemented by sub-clreplacedes, to create the source/target file listing.
* @param pathToListFile Path on HDFS where the listing file is written.
* @param options Input Options for DistCp (indicating source/target paths.)
* @throws IOException: Thrown on failure to create the listing file.
*/
protected abstract void doBuildListing(Path pathToListFile, DistCpOptions options) throws IOException;
/**
* Return the total bytes that distCp should copy for the source paths
* This doesn't consider whether file is same should be skipped during copy
*
* @return total bytes to copy
*/
protected abstract long getBytesToCopy();
/**
* Return the total number of paths to distcp, includes directories as well
* This doesn't consider whether file/dir is already present and should be skipped during copy
*
* @return Total number of paths to distcp
*/
protected abstract long getNumberOfPaths();
/**
* Validate the final resulting path listing. Checks if there are duplicate
* entries. If preserving ACLs, checks that file system can support ACLs.
* If preserving XAttrs, checks that file system can support XAttrs.
*
* @param pathToListFile - path listing build by doBuildListing
* @param options - Input options to distcp
* @throws IOException - Any issues while checking for duplicates and throws
* @throws DuplicateFileException - if there are duplicates
*/
private void validateFinalListing(Path pathToListFile, DistCpOptions options) throws DuplicateFileException, IOException {
Configuration config = getConf();
FileSystem fs = pathToListFile.getFileSystem(config);
Path sortedList = DistCpUtils.sortListing(fs, config, pathToListFile);
SequenceFile.Reader reader = new SequenceFile.Reader(config, SequenceFile.Reader.file(sortedList));
try {
// source relative path can never hold *
Text lastKey = new Text("*");
CopyListingFileStatus lastFileStatus = new CopyListingFileStatus();
Text currentKey = new Text();
Set<URI> aclSupportCheckFsSet = Sets.newHashSet();
Set<URI> xAttrSupportCheckFsSet = Sets.newHashSet();
while (reader.next(currentKey)) {
if (currentKey.equals(lastKey)) {
CopyListingFileStatus currentFileStatus = new CopyListingFileStatus();
reader.getCurrentValue(currentFileStatus);
throw new DuplicateFileException("File " + lastFileStatus.getPath() + " and " + currentFileStatus.getPath() + " would cause duplicates. Aborting");
}
reader.getCurrentValue(lastFileStatus);
if (options.shouldPreserve(DistCpOptions.FileAttribute.ACL)) {
FileSystem lastFs = lastFileStatus.getPath().getFileSystem(config);
URI lastFsUri = lastFs.getUri();
if (!aclSupportCheckFsSet.contains(lastFsUri)) {
DistCpUtils.checkFileSystemAclSupport(lastFs);
aclSupportCheckFsSet.add(lastFsUri);
}
}
if (options.shouldPreserve(DistCpOptions.FileAttribute.XATTR)) {
FileSystem lastFs = lastFileStatus.getPath().getFileSystem(config);
URI lastFsUri = lastFs.getUri();
if (!xAttrSupportCheckFsSet.contains(lastFsUri)) {
DistCpUtils.checkFileSystemXAttrSupport(lastFs);
xAttrSupportCheckFsSet.add(lastFsUri);
}
}
lastKey.set(currentKey);
}
} finally {
IOUtils.closeStream(reader);
}
}
/**
* Protected constructor, to initialize configuration.
* @param configuration The input configuration,
* with which the source/target FileSystems may be accessed.
* @param credentials - Credentials object on which the FS delegation tokens are cached.If null
* delegation token caching is skipped
*/
protected CopyListing(Configuration configuration, Credentials credentials) {
setConf(configuration);
setCredentials(credentials);
}
/**
* set Credentials store, on which FS delegatin token will be cached
* @param credentials - Credentials object
*/
protected void setCredentials(Credentials credentials) {
this.credentials = credentials;
}
/**
* get credentials to update the delegation tokens for accessed FS objects
* @return Credentials object
*/
protected Credentials getCredentials() {
return credentials;
}
/**
* Public Factory method with which the appropriate CopyListing implementation may be retrieved.
* @param configuration The input configuration.
* @param credentials Credentials object on which the FS delegation tokens are cached
* @param options The input Options, to help choose the appropriate CopyListing Implementation.
* @return An instance of the appropriate CopyListing implementation.
* @throws java.io.IOException - Exception if any
*/
public static CopyListing getCopyListing(Configuration configuration, Credentials credentials, DistCpOptions options) throws IOException {
String copyListingClreplacedName = configuration.get(DistCpConstants.CONF_LABEL_COPY_LISTING_CLreplaced, "");
Clreplaced<? extends CopyListing> copyListingClreplaced;
try {
if (!copyListingClreplacedName.isEmpty()) {
copyListingClreplaced = configuration.getClreplaced(DistCpConstants.CONF_LABEL_COPY_LISTING_CLreplaced, GlobbedCopyListing.clreplaced, CopyListing.clreplaced);
} else {
if (options.getSourceFileListing() == null) {
copyListingClreplaced = GlobbedCopyListing.clreplaced;
} else {
copyListingClreplaced = FileBasedCopyListing.clreplaced;
}
}
copyListingClreplacedName = copyListingClreplaced.getName();
Constructor<? extends CopyListing> constructor = copyListingClreplaced.getDeclaredConstructor(Configuration.clreplaced, Credentials.clreplaced);
return constructor.newInstance(configuration, credentials);
} catch (Exception e) {
throw new IOException("Unable to instantiate " + copyListingClreplacedName, e);
}
}
static clreplaced DuplicateFileException extends RuntimeException {
public DuplicateFileException(String message) {
super(message);
}
}
static clreplaced InvalidInputException extends RuntimeException {
public InvalidInputException(String message) {
super(message);
}
}
public static clreplaced AclsNotSupportedException extends RuntimeException {
public AclsNotSupportedException(String message) {
super(message);
}
}
public static clreplaced XAttrsNotSupportedException extends RuntimeException {
public XAttrsNotSupportedException(String message) {
super(message);
}
}
}
19
Source : CopyListing.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
/**
* set Credentials store, on which FS delegatin token will be cached
* @param credentials - Credentials object
*/
protected void setCredentials(Credentials credentials) {
this.credentials = credentials;
}
19
Source : TestMerger.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
@Test
public void testEncryptedMerger() throws Throwable {
jobConf.setBoolean(MRJobConfig.MR_ENCRYPTED_INTERMEDIATE_DATA, true);
conf.setBoolean(MRJobConfig.MR_ENCRYPTED_INTERMEDIATE_DATA, true);
Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials();
TokenCache.setShuffleSecretKey(new byte[16], credentials);
UserGroupInformation.getCurrentUser().addCredentials(credentials);
testInMemoryAndOnDiskMerger();
}
19
Source : JobSubmitter.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
private void printTokens(JobID jobId, Credentials credentials) throws IOException {
LOG.info("Submitting tokens for job: " + jobId);
for (Token<?> token : credentials.getAllTokens()) {
LOG.info(token);
}
}
19
Source : TestDelegationToken.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
@Test
public void testDelegationTokenWithDoAs() throws Exception {
final DistributedFileSystem dfs = cluster.getFileSystem();
final Credentials creds = new Credentials();
final Token<?>[] tokens = dfs.addDelegationTokens("JobTracker", creds);
replacedert.replacedertEquals(1, tokens.length);
@SuppressWarnings("unchecked")
final Token<DelegationTokenIdentifier> token = (Token<DelegationTokenIdentifier>) tokens[0];
final UserGroupInformation longUgi = UserGroupInformation.createRemoteUser("JobTracker/[email protected]");
final UserGroupInformation shortUgi = UserGroupInformation.createRemoteUser("JobTracker");
longUgi.doAs(new PrivilegedExceptionAction<Object>() {
@Override
public Object run() throws IOException {
try {
token.renew(config);
} catch (Exception e) {
replacedert.fail("Could not renew delegation token for user " + longUgi);
}
return null;
}
});
shortUgi.doAs(new PrivilegedExceptionAction<Object>() {
@Override
public Object run() throws Exception {
token.renew(config);
return null;
}
});
longUgi.doAs(new PrivilegedExceptionAction<Object>() {
@Override
public Object run() throws IOException {
try {
token.cancel(config);
} catch (Exception e) {
replacedert.fail("Could not cancel delegation token for user " + longUgi);
}
return null;
}
});
}
19
Source : TestDelegationToken.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
@Test
public void testAddDelegationTokensDFSApi() throws Exception {
UserGroupInformation ugi = UserGroupInformation.createRemoteUser("JobTracker");
DistributedFileSystem dfs = cluster.getFileSystem();
Credentials creds = new Credentials();
final Token<?>[] tokens = dfs.addDelegationTokens("JobTracker", creds);
replacedert.replacedertEquals(1, tokens.length);
replacedert.replacedertEquals(1, creds.numberOfTokens());
checkTokenIdentifier(ugi, tokens[0]);
final Token<?>[] tokens2 = dfs.addDelegationTokens("JobTracker", creds);
// already have token
replacedert.replacedertEquals(0, tokens2.length);
replacedert.replacedertEquals(1, creds.numberOfTokens());
}
19
Source : KeyProviderDelegationTokenExtension.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
/**
* Preplacedes the renewer and Credentials object to the underlying
* {@link DelegationTokenExtension}
* @param renewer the user allowed to renew the delegation tokens
* @param credentials cache in which to add new delegation tokens
* @return list of new delegation tokens
* @throws IOException thrown if IOException if an IO error occurs.
*/
public Token<?>[] addDelegationTokens(final String renewer, Credentials credentials) throws IOException {
return getExtension().addDelegationTokens(renewer, credentials);
}
19
Source : TestDelegationTokenRenewer.java
with Apache License 2.0
from naver
with Apache License 2.0
from naver
// Test FileSystem memory leak in obtainSystemTokensForUser.
@Test
public void testFSLeakInObtainSystemTokensForUser() throws Exception {
Credentials credentials = new Credentials();
String user = "test";
int oldCounter = MyFS.getInstanceCounter();
delegationTokenRenewer.obtainSystemTokensForUser(user, credentials);
delegationTokenRenewer.obtainSystemTokensForUser(user, credentials);
delegationTokenRenewer.obtainSystemTokensForUser(user, credentials);
replacedert.replacedertEquals(oldCounter, MyFS.getInstanceCounter());
}
19
Source : DelegationTokenRenewer.java
with Apache License 2.0
from naver
with Apache License 2.0
from naver
/**
* Asynchronously add application tokens for renewal.
* @param applicationId added application
* @param ts tokens
* @param shouldCancelAtEnd true if tokens should be canceled when the app is
* done else false.
* @param user user
*/
public void addApplicationAsync(ApplicationId applicationId, Credentials ts, boolean shouldCancelAtEnd, String user) {
processDelegationTokenRenewerEvent(new DelegationTokenRenewerAppSubmitEvent(applicationId, ts, shouldCancelAtEnd, user));
}
19
Source : CopyListing.java
with Apache License 2.0
from naver
with Apache License 2.0
from naver
/**
* The CopyListing abstraction is responsible for how the list of
* sources and targets is constructed, for DistCp's copy function.
* The copy-listing should be a
* SequenceFile<Text, CopyListingFileStatus>, located at the path
* specified to buildListing(), each entry being a pair of (Source relative
* path, source file status), all the paths being fully qualified.
*/
public abstract clreplaced CopyListing extends Configured {
private Credentials credentials;
/**
* Build listing function creates the input listing that distcp uses to
* perform the copy.
*
* The build listing is a sequence file that has relative path of a file in the key
* and the file status information of the source file in the value
*
* For instance if the source path is /tmp/data and the traversed path is
* /tmp/data/dir1/dir2/file1, then the sequence file would contain
*
* key: /dir1/dir2/file1 and value: FileStatus(/tmp/data/dir1/dir2/file1)
*
* File would also contain directory entries. Meaning, if /tmp/data/dir1/dir2/file1
* is the only file under /tmp/data, the resulting sequence file would contain the
* following entries
*
* key: /dir1 and value: FileStatus(/tmp/data/dir1)
* key: /dir1/dir2 and value: FileStatus(/tmp/data/dir1/dir2)
* key: /dir1/dir2/file1 and value: FileStatus(/tmp/data/dir1/dir2/file1)
*
* Cases requiring special handling:
* If source path is a file (/tmp/file1), contents of the file will be as follows
*
* TARGET DOES NOT EXIST: Key-"", Value-FileStatus(/tmp/file1)
* TARGET IS FILE : Key-"", Value-FileStatus(/tmp/file1)
* TARGET IS DIR : Key-"/file1", Value-FileStatus(/tmp/file1)
*
* @param pathToListFile - Output file where the listing would be stored
* @param options - Input options to distcp
* @throws IOException - Exception if any
*/
public final void buildListing(Path pathToListFile, DistCpOptions options) throws IOException {
validatePaths(options);
doBuildListing(pathToListFile, options);
Configuration config = getConf();
config.set(DistCpConstants.CONF_LABEL_LISTING_FILE_PATH, pathToListFile.toString());
config.setLong(DistCpConstants.CONF_LABEL_TOTAL_BYTES_TO_BE_COPIED, getBytesToCopy());
config.setLong(DistCpConstants.CONF_LABEL_TOTAL_NUMBER_OF_RECORDS, getNumberOfPaths());
validateFinalListing(pathToListFile, options);
}
/**
* Validate input and output paths
*
* @param options - Input options
* @throws InvalidInputException If inputs are invalid
* @throws IOException any Exception with FS
*/
protected abstract void validatePaths(DistCpOptions options) throws IOException, InvalidInputException;
/**
* The interface to be implemented by sub-clreplacedes, to create the source/target file listing.
* @param pathToListFile Path on HDFS where the listing file is written.
* @param options Input Options for DistCp (indicating source/target paths.)
* @throws IOException Thrown on failure to create the listing file.
*/
protected abstract void doBuildListing(Path pathToListFile, DistCpOptions options) throws IOException;
/**
* Return the total bytes that distCp should copy for the source paths
* This doesn't consider whether file is same should be skipped during copy
*
* @return total bytes to copy
*/
protected abstract long getBytesToCopy();
/**
* Return the total number of paths to distcp, includes directories as well
* This doesn't consider whether file/dir is already present and should be skipped during copy
*
* @return Total number of paths to distcp
*/
protected abstract long getNumberOfPaths();
/**
* Validate the final resulting path listing. Checks if there are duplicate
* entries. If preserving ACLs, checks that file system can support ACLs.
* If preserving XAttrs, checks that file system can support XAttrs.
*
* @param pathToListFile - path listing build by doBuildListing
* @param options - Input options to distcp
* @throws IOException - Any issues while checking for duplicates and throws
* @throws DuplicateFileException - if there are duplicates
*/
private void validateFinalListing(Path pathToListFile, DistCpOptions options) throws DuplicateFileException, IOException {
Configuration config = getConf();
FileSystem fs = pathToListFile.getFileSystem(config);
Path sortedList = DistCpUtils.sortListing(fs, config, pathToListFile);
SequenceFile.Reader reader = new SequenceFile.Reader(config, SequenceFile.Reader.file(sortedList));
try {
// source relative path can never hold *
Text lastKey = new Text("*");
CopyListingFileStatus lastFileStatus = new CopyListingFileStatus();
Text currentKey = new Text();
Set<URI> aclSupportCheckFsSet = Sets.newHashSet();
Set<URI> xAttrSupportCheckFsSet = Sets.newHashSet();
while (reader.next(currentKey)) {
if (currentKey.equals(lastKey)) {
CopyListingFileStatus currentFileStatus = new CopyListingFileStatus();
reader.getCurrentValue(currentFileStatus);
throw new DuplicateFileException("File " + lastFileStatus.getPath() + " and " + currentFileStatus.getPath() + " would cause duplicates. Aborting");
}
reader.getCurrentValue(lastFileStatus);
if (options.shouldPreserve(DistCpOptions.FileAttribute.ACL)) {
FileSystem lastFs = lastFileStatus.getPath().getFileSystem(config);
URI lastFsUri = lastFs.getUri();
if (!aclSupportCheckFsSet.contains(lastFsUri)) {
DistCpUtils.checkFileSystemAclSupport(lastFs);
aclSupportCheckFsSet.add(lastFsUri);
}
}
if (options.shouldPreserve(DistCpOptions.FileAttribute.XATTR)) {
FileSystem lastFs = lastFileStatus.getPath().getFileSystem(config);
URI lastFsUri = lastFs.getUri();
if (!xAttrSupportCheckFsSet.contains(lastFsUri)) {
DistCpUtils.checkFileSystemXAttrSupport(lastFs);
xAttrSupportCheckFsSet.add(lastFsUri);
}
}
lastKey.set(currentKey);
}
} finally {
IOUtils.closeStream(reader);
}
}
/**
* Protected constructor, to initialize configuration.
* @param configuration The input configuration,
* with which the source/target FileSystems may be accessed.
* @param credentials - Credentials object on which the FS delegation tokens are cached.If null
* delegation token caching is skipped
*/
protected CopyListing(Configuration configuration, Credentials credentials) {
setConf(configuration);
setCredentials(credentials);
}
/**
* set Credentials store, on which FS delegatin token will be cached
* @param credentials - Credentials object
*/
protected void setCredentials(Credentials credentials) {
this.credentials = credentials;
}
/**
* get credentials to update the delegation tokens for accessed FS objects
* @return Credentials object
*/
protected Credentials getCredentials() {
return credentials;
}
/**
* Public Factory method with which the appropriate CopyListing implementation may be retrieved.
* @param configuration The input configuration.
* @param credentials Credentials object on which the FS delegation tokens are cached
* @param options The input Options, to help choose the appropriate CopyListing Implementation.
* @return An instance of the appropriate CopyListing implementation.
* @throws java.io.IOException - Exception if any
*/
public static CopyListing getCopyListing(Configuration configuration, Credentials credentials, DistCpOptions options) throws IOException {
if (options.shouldUseDiff()) {
return new GlobbedCopyListing(configuration, credentials);
}
String copyListingClreplacedName = configuration.get(DistCpConstants.CONF_LABEL_COPY_LISTING_CLreplaced, "");
Clreplaced<? extends CopyListing> copyListingClreplaced;
try {
if (!copyListingClreplacedName.isEmpty()) {
copyListingClreplaced = configuration.getClreplaced(DistCpConstants.CONF_LABEL_COPY_LISTING_CLreplaced, GlobbedCopyListing.clreplaced, CopyListing.clreplaced);
} else {
if (options.getSourceFileListing() == null) {
copyListingClreplaced = GlobbedCopyListing.clreplaced;
} else {
copyListingClreplaced = FileBasedCopyListing.clreplaced;
}
}
copyListingClreplacedName = copyListingClreplaced.getName();
Constructor<? extends CopyListing> constructor = copyListingClreplaced.getDeclaredConstructor(Configuration.clreplaced, Credentials.clreplaced);
return constructor.newInstance(configuration, credentials);
} catch (Exception e) {
throw new IOException("Unable to instantiate " + copyListingClreplacedName, e);
}
}
static clreplaced DuplicateFileException extends RuntimeException {
public DuplicateFileException(String message) {
super(message);
}
}
static clreplaced InvalidInputException extends RuntimeException {
public InvalidInputException(String message) {
super(message);
}
}
public static clreplaced AclsNotSupportedException extends RuntimeException {
public AclsNotSupportedException(String message) {
super(message);
}
}
public static clreplaced XAttrsNotSupportedException extends RuntimeException {
public XAttrsNotSupportedException(String message) {
super(message);
}
}
}
19
Source : TestMerger.java
with Apache License 2.0
from naver
with Apache License 2.0
from naver
@Test
public void testEncryptedMerger() throws Throwable {
jobConf.setBoolean(MRJobConfig.MR_ENCRYPTED_INTERMEDIATE_DATA, true);
conf.setBoolean(MRJobConfig.MR_ENCRYPTED_INTERMEDIATE_DATA, true);
Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials();
TokenCache.setEncryptedSpillKey(new byte[16], credentials);
UserGroupInformation.getCurrentUser().addCredentials(credentials);
testInMemoryAndOnDiskMerger();
}
19
Source : YarnTestBase.java
with Apache License 2.0
from ljygz
with Apache License 2.0
from ljygz
public static boolean verifyTokenKindInContainerCredentials(final Collection<String> tokens, final String containerId) throws IOException {
File cwd = new File("target/" + YARN_CONFIGURATION.get(TEST_CLUSTER_NAME_KEY));
if (!cwd.exists() || !cwd.isDirectory()) {
return false;
}
File containerTokens = findFile(cwd.getAbsolutePath(), new FilenameFilter() {
@Override
public boolean accept(File dir, String name) {
return name.equals(containerId + ".tokens");
}
});
if (containerTokens != null) {
LOG.info("Verifying tokens in {}", containerTokens.getAbsolutePath());
Credentials tmCredentials = Credentials.readTokenStorageFile(containerTokens, new Configuration());
Collection<Token<? extends TokenIdentifier>> userTokens = tmCredentials.getAllTokens();
Set<String> tokenKinds = new HashSet<>(4);
for (Token<? extends TokenIdentifier> token : userTokens) {
tokenKinds.add(token.getKind().toString());
}
return tokenKinds.containsAll(tokens);
} else {
LOG.warn("Unable to find credential file for container {}", containerId);
return false;
}
}
19
Source : HadoopOutputFormatCommonBase.java
with Apache License 2.0
from ljygz
with Apache License 2.0
from ljygz
/**
* A common base for both "mapred" and "mapreduce" Hadoop output formats.
*
* <p>The base is taking care of handling (serializing) security credentials.
*/
@Internal
public abstract clreplaced HadoopOutputFormatCommonBase<T> extends RichOutputFormat<T> {
protected transient Credentials credentials;
protected HadoopOutputFormatCommonBase(Credentials creds) {
this.credentials = creds;
}
protected void write(ObjectOutputStream out) throws IOException {
this.credentials.write(out);
}
public void read(ObjectInputStream in) throws IOException {
this.credentials = new Credentials();
credentials.readFields(in);
}
}
19
Source : HadoopInputFormatCommonBase.java
with Apache License 2.0
from ljygz
with Apache License 2.0
from ljygz
/**
* A common base for both "mapred" and "mapreduce" Hadoop input formats.
*/
@Internal
public abstract clreplaced HadoopInputFormatCommonBase<T, SPITTYPE extends InputSplit> extends RichInputFormat<T, SPITTYPE> {
protected transient Credentials credentials;
protected HadoopInputFormatCommonBase(Credentials creds) {
this.credentials = creds;
}
protected void write(ObjectOutputStream out) throws IOException {
this.credentials.write(out);
}
public void read(ObjectInputStream in) throws IOException {
this.credentials = new Credentials();
credentials.readFields(in);
}
/**
* This method only exists because there is no UserGroupInformation.getCredentials() method
* in Hadoop 1.x
*
* <p>Note that this method returns "null" in Hadoop 1.x environments.
*
* @param ugi The user information
* @return new credentials object from the user information. MAY RETURN NULL!
*/
public static Credentials getCredentialsFromUGI(UserGroupInformation ugi) {
Method getCredentialsMethod = null;
for (Method m : ugi.getClreplaced().getMethods()) {
if (m.getName().equals("getCredentials")) {
getCredentialsMethod = m;
break;
}
}
if (getCredentialsMethod == null) {
return null;
} else {
try {
return (Credentials) getCredentialsMethod.invoke(ugi);
} catch (InvocationTargetException | IllegalAccessException e) {
throw new RuntimeException("Unable to get credentials from UserGroupInformation. This is only supported by Hadoop 2.2.0+");
}
}
}
}
19
Source : CopyListing.java
with Apache License 2.0
from HotelsDotCom
with Apache License 2.0
from HotelsDotCom
/**
* The CopyListing abstraction is responsible for how the list of sources and targets is constructed for DistCp's copy
* function. The copy-listing should be a SequenceFile<Text, CopyListingFileStatus> located at the path specified to
* buildListing(), each entry being a pair of (Source relative path, source file status) and all the paths being fully
* qualified.
*/
public abstract clreplaced CopyListing extends Configured {
private Credentials credentials;
/**
* Build listing function creates the input listing that S3MapReduceCp uses to perform the copy. The build listing is
* a sequence file that has the relative path of a file in the key and the file status information of the source file
* in the value For instance if the source path is /tmp/data and the traversed path is /tmp/data/dir1/dir2/file1, then
* the sequence file would contain the key: /dir1/dir2/file1 and the value: FileStatus(/tmp/data/dir1/dir2/file1). The
* file would also contain directory entries i.e if /tmp/data/dir1/dir2/file1 is the only file under /tmp/data, the
* resulting sequence file would contain the following entries key: /dir1 and value: FileStatus(/tmp/data/dir1) key:
* /dir1/dir2 and value: FileStatus(/tmp/data/dir1/dir2) key: /dir1/dir2/file1 and value:
* FileStatus(/tmp/data/dir1/dir2/file1) Cases requiring special handling: If source path is a file (/tmp/file1),
* contents of the file will be as follows TARGET DOES NOT EXIST: Key-"", Value-FileStatus(/tmp/file1) TARGET IS FILE
* : Key-"", Value-FileStatus(/tmp/file1) TARGET IS DIR : Key-"/file1", Value-FileStatus(/tmp/file1)
*
* @param pathToListFile Output file where the listing would be stored
* @param options Input options to S3MapReduceCp
* @throws IOException Exception if any
*/
public final void buildListing(Path pathToListFile, S3MapReduceCpOptions options) throws IOException {
validatePath(options);
doBuildListing(pathToListFile, options);
Configuration config = getConf();
config.set(S3MapReduceCpConstants.CONF_LABEL_LISTING_FILE_PATH, pathToListFile.toString());
config.setLong(S3MapReduceCpConstants.CONF_LABEL_TOTAL_BYTES_TO_BE_COPIED, getBytesToCopy());
config.setLong(S3MapReduceCpConstants.CONF_LABEL_TOTAL_NUMBER_OF_RECORDS, getNumberOfPaths());
validateFinalListing(pathToListFile, options);
}
/**
* Validate input and output paths
*
* @param options Input options
* @throws InvalidInputException: If inputs are invalid
* @throws IOException any Exception with FS
*/
protected abstract void validatePath(S3MapReduceCpOptions options) throws IOException, InvalidInputException;
/**
* The interface to be implemented by sub-clreplacedes, to create the source/target file listing.
*
* @param pathToListFile Path on HDFS where the listing file is written.
* @param options Input Options for S3MapReduceCp (indicating source/target paths.)
* @throws IOException Thrown on failure to create the listing file.
*/
protected abstract void doBuildListing(Path pathToListFile, S3MapReduceCpOptions options) throws IOException;
/**
* Return the total bytes that S3MapReduceCp should copy for the source paths This doesn't consider whether file is
* same should be skipped during copy
*
* @return total bytes to copy
*/
protected abstract long getBytesToCopy();
/**
* Return the total number of paths to S3MapReduceCp, includes directories as well This doesn't consider whether
* file/dir is already present and should be skipped during copy
*
* @return Total number of paths to S3MapReduceCp
*/
protected abstract long getNumberOfPaths();
/**
* Validate the final resulting path listing. Checks if there are duplicate entries. If preserving ACLs, checks that
* file system can support ACLs. If preserving XAttrs, checks that file system can support XAttrs.
*
* @param pathToListFile path listing build by doBuildListing
* @param options Input options to S3MapReduceCp
* @throws IOException Any issues while checking for duplicates and throws
* @throws DuplicateFileException if there are duplicates
*/
private void validateFinalListing(Path pathToListFile, S3MapReduceCpOptions options) throws DuplicateFileException, IOException {
Configuration config = getConf();
FileSystem fs = pathToListFile.getFileSystem(config);
Path sortedList = sortListing(fs, config, pathToListFile);
SequenceFile.Reader reader = new SequenceFile.Reader(config, SequenceFile.Reader.file(sortedList));
try {
// source relative path can never hold *
Text lastKey = new Text("*");
CopyListingFileStatus lastFileStatus = new CopyListingFileStatus();
Text currentKey = new Text();
while (reader.next(currentKey)) {
if (currentKey.equals(lastKey)) {
CopyListingFileStatus currentFileStatus = new CopyListingFileStatus();
reader.getCurrentValue(currentFileStatus);
throw new DuplicateFileException("File " + lastFileStatus.getPath() + " and " + currentFileStatus.getPath() + " would cause duplicates. Aborting");
}
reader.getCurrentValue(lastFileStatus);
lastKey.set(currentKey);
}
} finally {
IOUtils.closeStream(reader);
}
}
/**
* Sort sequence file containing FileStatus and Text as key and value respecitvely
*
* @param fs File System
* @param conf Configuration
* @param sourceListing Source listing file
* @return Path of the sorted file. Is source file with _sorted appended to the name
* @throws IOException Any exception during sort.
*/
private static Path sortListing(FileSystem fs, Configuration conf, Path sourceListing) throws IOException {
SequenceFile.Sorter sorter = new SequenceFile.Sorter(fs, Text.clreplaced, CopyListingFileStatus.clreplaced, conf);
Path output = new Path(sourceListing.toString() + "_sorted");
if (fs.exists(output)) {
fs.delete(output, false);
}
sorter.sort(sourceListing, output);
return output;
}
/**
* Protected constructor, to initialize configuration.
*
* @param configuration The input configuration, with which the source/target FileSystems may be accessed.
* @param credentials - Credentials object on which the FS delegation tokens are cached.If null delegation token
* caching is skipped
*/
protected CopyListing(Configuration configuration, Credentials credentials) {
setConf(configuration);
setCredentials(credentials);
}
/**
* set Credentials store, on which FS delegatin token will be cached
*
* @param credentials Credentials object
*/
protected void setCredentials(Credentials credentials) {
this.credentials = credentials;
}
/**
* get credentials to update the delegation tokens for accessed FS objects
*
* @return Credentials object
*/
protected Credentials getCredentials() {
return credentials;
}
/**
* Public Factory method with which the appropriate CopyListing implementation may be retrieved.
*
* @param configuration The input configuration.
* @param credentials Credentials object on which the FS delegation tokens are cached
* @param options The input Options, to help choose the appropriate CopyListing Implementation.
* @return An instance of the appropriate CopyListing implementation.
* @throws java.io.IOException Exception if any
*/
public static CopyListing getCopyListing(Configuration configuration, Credentials credentials, S3MapReduceCpOptions options) throws IOException {
String copyListingClreplacedName = configuration.get(S3MapReduceCpConstants.CONF_LABEL_COPY_LISTING_CLreplaced, "");
Clreplaced<? extends CopyListing> copyListingClreplaced;
try {
if (!copyListingClreplacedName.isEmpty()) {
copyListingClreplaced = configuration.getClreplaced(S3MapReduceCpConstants.CONF_LABEL_COPY_LISTING_CLreplaced, SimpleCopyListing.clreplaced, CopyListing.clreplaced);
} else {
copyListingClreplaced = SimpleCopyListing.clreplaced;
}
copyListingClreplacedName = copyListingClreplaced.getName();
Constructor<? extends CopyListing> constructor = copyListingClreplaced.getDeclaredConstructor(Configuration.clreplaced, Credentials.clreplaced);
return constructor.newInstance(configuration, credentials);
} catch (NoSuchMethodException | SecurityException | InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException e) {
throw new IOException("Unable to instantiate " + copyListingClreplacedName, e);
}
}
static clreplaced DuplicateFileException extends RuntimeException {
private static final long serialVersionUID = 1L;
public DuplicateFileException(String message) {
super(message);
}
}
static clreplaced InvalidInputException extends RuntimeException {
private static final long serialVersionUID = 1L;
public InvalidInputException(String message) {
super(message);
}
}
}
19
Source : CopyListing.java
with Apache License 2.0
from HotelsDotCom
with Apache License 2.0
from HotelsDotCom
/**
* set Credentials store, on which FS delegatin token will be cached
*
* @param credentials Credentials object
*/
protected void setCredentials(Credentials credentials) {
this.credentials = credentials;
}
19
Source : HadoopFileSystemWrapper.java
with Apache License 2.0
from dremio
with Apache License 2.0
from dremio
@Override
@LimitedPrivate({ "HDFS", "MapReduce" })
public Token<?>[] addDelegationTokens(String renewer, Credentials credentials) throws IOException {
try (WaitRecorder recorder = OperatorStats.getWaitRecorder(operatorStats)) {
return underlyingFs.addDelegationTokens(renewer, credentials);
} catch (FSError e) {
throw propagateFSError(e);
}
}
19
Source : HoodieWrapperFileSystem.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
@Override
public Token<?>[] addDelegationTokens(String renewer, Credentials credentials) throws IOException {
return fileSystem.addDelegationTokens(renewer, credentials);
}
19
Source : YarnTwillPreparer.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
/**
* Creates an {@link Credentials} by copying the {@link Credentials} of the current user.
*/
private Credentials createCredentials() {
Credentials credentials = new Credentials();
try {
credentials.addAll(UserGroupInformation.getCurrentUser().getCredentials());
} catch (IOException e) {
LOG.warn("Failed to get current user UGI. Current user credentials not added.", e);
}
return credentials;
}
19
Source : YarnSecureStore.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
/**
* A {@link SecureStore} for hadoop credentials.
*/
public final clreplaced YarnSecureStore implements SecureStore {
private final Credentials credentials;
public static SecureStore create(Credentials credentials) {
return new YarnSecureStore(credentials);
}
private YarnSecureStore(Credentials credentials) {
this.credentials = credentials;
}
@Override
public Credentials getStore() {
return credentials;
}
}
19
Source : YarnSecureStore.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
public static SecureStore create(Credentials credentials) {
return new YarnSecureStore(credentials);
}
18
Source : StoreDiag.java
with Apache License 2.0
from steveloughran
with Apache License 2.0
from steveloughran
/**
* Dump all the user's tokens.
* @throws IOException failure.
*/
public void dumpUserTokens() throws IOException {
UserGroupInformation currentUser = UserGroupInformation.getCurrentUser();
Credentials credentials = currentUser.getCredentials();
Collection<Token<? extends TokenIdentifier>> allTokens = credentials.getAllTokens();
for (Token<? extends TokenIdentifier> token : allTokens) {
println("%s", token);
}
}
18
Source : TestDelegationTokenRenewer.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
/**
* Basic idea of the test:
* 1. register a token for 2 seconds with no cancel at the end
* 2. cancel it immediately
* 3. Sleep and check that the 2 seconds renew didn't happen
* (totally 5 renewals)
* 4. check cancellation
* @throws IOException
* @throws URISyntaxException
*/
@Test(timeout = 60000)
public void testDTRenewalWithNoCancel() throws Exception {
MyFS dfs = (MyFS) FileSystem.get(conf);
LOG.info("dfs=" + (Object) dfs.hashCode() + ";conf=" + conf.hashCode());
Credentials ts = new Credentials();
MyToken token1 = dfs.getDelegationToken("user1");
// to cause this one to be set for renew in 2 secs
Renewer.tokenToRenewIn2Sec = token1;
LOG.info("token=" + token1 + " should be renewed for 2 secs");
String nn1 = DelegationTokenRenewer.SCHEME + "://host1:0";
ts.addToken(new Text(nn1), token1);
ApplicationId applicationId_1 = BuilderUtils.newApplicationId(0, 1);
delegationTokenRenewer.addApplicationAsync(applicationId_1, ts, false, "user");
waitForEventsToGetProcessed(delegationTokenRenewer);
delegationTokenRenewer.applicationFinished(applicationId_1);
waitForEventsToGetProcessed(delegationTokenRenewer);
// number of renewals so far
int numberOfExpectedRenewals = Renewer.counter;
try {
// sleep 6 seconds, so it has time to renew
Thread.sleep(6 * 1000);
} catch (InterruptedException e) {
}
LOG.info("Counter = " + Renewer.counter + ";t=" + Renewer.lastRenewed);
// counter and the token should still be the old ones
replacedertEquals("renew wasn't called as many times as expected", numberOfExpectedRenewals, Renewer.counter);
// also renewing of the canceled token should not fail, because it has not
// been canceled
token1.renew(conf);
}
18
Source : TestDelegationTokenRenewer.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
/**
* Basic idea of the test:
* 1. create tokens.
* 2. Mark one of them to be renewed in 2 seconds (instead of
* 24 hours)
* 3. register them for renewal
* 4. sleep for 3 seconds
* 5. count number of renewals (should 3 initial ones + one extra)
* 6. register another token for 2 seconds
* 7. cancel it immediately
* 8. Sleep and check that the 2 seconds renew didn't happen
* (totally 5 renewals)
* 9. check cancellation
* @throws IOException
* @throws URISyntaxException
*/
@Test(timeout = 60000)
public void testDTRenewal() throws Exception {
MyFS dfs = (MyFS) FileSystem.get(conf);
LOG.info("dfs=" + (Object) dfs.hashCode() + ";conf=" + conf.hashCode());
// Test 1. - add three tokens - make sure exactly one get's renewed
// get the delegation tokens
MyToken token1, token2, token3;
token1 = dfs.getDelegationToken("user1");
token2 = dfs.getDelegationToken("user2");
token3 = dfs.getDelegationToken("user3");
// to cause this one to be set for renew in 2 secs
Renewer.tokenToRenewIn2Sec = token1;
LOG.info("token=" + token1 + " should be renewed for 2 secs");
// three distinct Namenodes
String nn1 = DelegationTokenRenewer.SCHEME + "://host1:0";
String nn2 = DelegationTokenRenewer.SCHEME + "://host2:0";
String nn3 = DelegationTokenRenewer.SCHEME + "://host3:0";
Credentials ts = new Credentials();
// register the token for renewal
ts.addToken(new Text(nn1), token1);
ts.addToken(new Text(nn2), token2);
ts.addToken(new Text(nn3), token3);
// register the tokens for renewal
ApplicationId applicationId_0 = BuilderUtils.newApplicationId(0, 0);
delegationTokenRenewer.addApplicationAsync(applicationId_0, ts, true, "user");
waitForEventsToGetProcessed(delegationTokenRenewer);
// first 3 initial renewals + 1 real
int numberOfExpectedRenewals = 3 + 1;
int attempts = 10;
while (attempts-- > 0) {
try {
// sleep 3 seconds, so it has time to renew
Thread.sleep(3 * 1000);
} catch (InterruptedException e) {
}
// since we cannot guarantee timely execution - let's give few chances
if (Renewer.counter == numberOfExpectedRenewals)
break;
}
LOG.info("dfs=" + dfs.hashCode() + ";Counter = " + Renewer.counter + ";t=" + Renewer.lastRenewed);
replacedertEquals("renew wasn't called as many times as expected(4):", numberOfExpectedRenewals, Renewer.counter);
replacedertEquals("most recently renewed token mismatch", Renewer.lastRenewed, token1);
// Test 2.
// add another token ( that expires in 2 secs). Then remove it, before
// time is up.
// Wait for 3 secs , and make sure no renews were called
ts = new Credentials();
MyToken token4 = dfs.getDelegationToken("user4");
// to cause this one to be set for renew in 2 secs
Renewer.tokenToRenewIn2Sec = token4;
LOG.info("token=" + token4 + " should be renewed for 2 secs");
String nn4 = DelegationTokenRenewer.SCHEME + "://host4:0";
ts.addToken(new Text(nn4), token4);
ApplicationId applicationId_1 = BuilderUtils.newApplicationId(0, 1);
delegationTokenRenewer.addApplicationAsync(applicationId_1, ts, true, "user");
waitForEventsToGetProcessed(delegationTokenRenewer);
delegationTokenRenewer.applicationFinished(applicationId_1);
waitForEventsToGetProcessed(delegationTokenRenewer);
// number of renewals so far
numberOfExpectedRenewals = Renewer.counter;
try {
// sleep 6 seconds, so it has time to renew
Thread.sleep(6 * 1000);
} catch (InterruptedException e) {
}
LOG.info("Counter = " + Renewer.counter + ";t=" + Renewer.lastRenewed);
// counter and the token should stil be the old ones
replacedertEquals("renew wasn't called as many times as expected", numberOfExpectedRenewals, Renewer.counter);
// also renewing of the cancelled token should fail
try {
token4.renew(conf);
fail("Renewal of cancelled token should have failed");
} catch (InvalidToken ite) {
// expected
}
}
18
Source : TestRMAppTransitions.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
@Test(timeout = 30000)
public void testAppRecoverPath() throws IOException {
LOG.info("--- START: testAppRecoverPath ---");
ApplicationSubmissionContext sub = Records.newRecord(ApplicationSubmissionContext.clreplaced);
ContainerLaunchContext clc = Records.newRecord(ContainerLaunchContext.clreplaced);
Credentials credentials = new Credentials();
DataOutputBuffer dob = new DataOutputBuffer();
credentials.writeTokenStorageToStream(dob);
ByteBuffer securityTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
clc.setTokens(securityTokens);
sub.setAMContainerSpec(clc);
testCreateAppSubmittedRecovery(sub);
}
18
Source : DelegationTokenRenewer.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
protected Token<?>[] obtainSystemTokensForUser(String user, final Credentials credentials) throws IOException, InterruptedException {
// Get new hdfs tokens on behalf of this user
UserGroupInformation proxyUser = UserGroupInformation.createProxyUser(user, UserGroupInformation.getLoginUser());
Token<?>[] newTokens = proxyUser.doAs(new PrivilegedExceptionAction<Token<?>[]>() {
@Override
public Token<?>[] run() throws Exception {
return FileSystem.get(getConfig()).addDelegationTokens(UserGroupInformation.getLoginUser().getUserName(), credentials);
}
});
return newTokens;
}
18
Source : LogAggregationService.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
protected void initAppAggregator(final ApplicationId appId, String user, Credentials credentials, ContainerLogsRetentionPolicy logRetentionPolicy, Map<ApplicationAccessType, String> appAcls, LogAggregationContext logAggregationContext) {
// Get user's FileSystem credentials
final UserGroupInformation userUgi = UserGroupInformation.createRemoteUser(user);
if (credentials != null) {
userUgi.addCredentials(credentials);
}
// New application
final AppLogAggregator appLogAggregator = new AppLogAggregatorImpl(this.dispatcher, this.deletionService, getConfig(), appId, userUgi, this.nodeId, dirsHandler, getRemoteNodeLogFileForApp(appId, user), logRetentionPolicy, appAcls, logAggregationContext, this.context, getLocalFileContext(getConfig()));
if (this.appLogAggregators.putIfAbsent(appId, appLogAggregator) != null) {
throw new YarnRuntimeException("Duplicate initApp for " + appId);
}
// wait until check for existing aggregator to create dirs
try {
// Create the app dir
createAppDir(user, appId, userUgi);
} catch (Exception e) {
appLogAggregators.remove(appId);
closeFileSystems(userUgi);
if (!(e instanceof YarnRuntimeException)) {
e = new YarnRuntimeException(e);
}
throw (YarnRuntimeException) e;
}
// TODO Get the user configuration for the list of containers that need log
// aggregation.
// Schedule the aggregator.
Runnable aggregatorWrapper = new Runnable() {
public void run() {
try {
appLogAggregator.run();
} finally {
appLogAggregators.remove(appId);
closeFileSystems(userUgi);
}
}
};
this.threadPool.execute(aggregatorWrapper);
}
18
Source : ContainerManagerImpl.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
private ContainerManagerApplicationProto buildAppProto(ApplicationId appId, String user, Credentials credentials, Map<ApplicationAccessType, String> appAcls, LogAggregationContext logAggregationContext) {
ContainerManagerApplicationProto.Builder builder = ContainerManagerApplicationProto.newBuilder();
builder.setId(((ApplicationIdPBImpl) appId).getProto());
builder.setUser(user);
if (logAggregationContext != null) {
builder.setLogAggregationContext(((LogAggregationContextPBImpl) logAggregationContext).getProto());
}
builder.clearCredentials();
if (credentials != null) {
DataOutputBuffer dob = new DataOutputBuffer();
try {
credentials.writeTokenStorageToStream(dob);
builder.setCredentials(ByteString.copyFrom(dob.getData()));
} catch (IOException e) {
// should not occur
LOG.error("Cannot serialize credentials", e);
}
}
builder.clearAcls();
if (appAcls != null) {
for (Map.Entry<ApplicationAccessType, String> acl : appAcls.entrySet()) {
ApplicationACLMapProto p = ApplicationACLMapProto.newBuilder().setAccessType(ProtoUtils.convertToProtoFormat(acl.getKey())).setAcl(acl.getValue()).build();
builder.addAcls(p);
}
}
return builder.build();
}
18
Source : TestAMRMClient.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
@SuppressWarnings("unchecked")
private org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> getAMRMToken() throws IOException {
Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials();
Iterator<org.apache.hadoop.security.token.Token<?>> iter = credentials.getAllTokens().iterator();
org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> result = null;
while (iter.hasNext()) {
org.apache.hadoop.security.token.Token<?> token = iter.next();
if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) {
if (result != null) {
replacedert.fail("credentials has more than one AMRM token." + " token1: " + result + " token2: " + token);
}
result = (org.apache.hadoop.security.token.Token<AMRMTokenIdentifier>) token;
}
}
return result;
}
18
Source : CredentialsTestJob.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
private static void checkSecrets(Credentials ts) {
if (ts == null) {
throw new RuntimeException("The credentials are not available");
// fail the test
}
for (int i = 0; i < NUM_OF_KEYS; i++) {
String secretName = "alias" + i;
// get token storage and a key
byte[] secretValue = ts.getSecretKey(new Text(secretName));
System.out.println(secretValue);
if (secretValue == null) {
throw new RuntimeException("The key " + secretName + " is not available. ");
// fail the test
}
String secretValueStr = new String(secretValue);
if (!("preplacedword" + i).equals(secretValueStr)) {
throw new RuntimeException("The key " + secretName + " is not correct. Expected value is " + ("preplacedword" + i) + ". Actual value is " + // fail the test
secretValueStr);
}
}
}
18
Source : TestTokenCache.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
private void checkToken(Credentials creds, Token<?>... tokens) {
replacedertEquals(tokens.length, creds.getAllTokens().size());
for (Token<?> token : tokens) {
Token<?> credsToken = creds.getToken(token.getService());
replacedertTrue(credsToken != null);
replacedertEquals(token, credsToken);
}
}
18
Source : TestTokenCache.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
@Test
public void testObtainTokens() throws Exception {
Credentials credentials = new Credentials();
FileSystem fs = mock(FileSystem.clreplaced);
TokenCache.obtainTokensForNamenodesInternal(fs, credentials, conf);
verify(fs).addDelegationTokens(eq(renewer), eq(credentials));
}
18
Source : TokenCache.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
/**
* store job token
* @param t
*/
@InterfaceAudience.Private
public static void setJobToken(Token<? extends TokenIdentifier> t, Credentials credentials) {
credentials.addToken(JOB_TOKEN, t);
}
18
Source : TokenCache.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
/**
* auxiliary method to get user's secret keys..
* @param alias
* @return secret key from the storage
*/
public static byte[] getSecretKey(Credentials credentials, Text alias) {
if (credentials == null)
return null;
return credentials.getSecretKey(alias);
}
18
Source : TokenCache.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
@InterfaceAudience.Private
public static byte[] getShuffleSecretKey(Credentials credentials) {
return getSecretKey(credentials, SHUFFLE_TOKEN);
}
18
Source : TestCredentials.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
@Test
public void testAddTokensToUGI() {
UserGroupInformation ugi = UserGroupInformation.createRemoteUser("someone");
Credentials creds = new Credentials();
for (int i = 0; i < service.length; i++) {
creds.addToken(service[i], token[i]);
}
ugi.addCredentials(creds);
creds = ugi.getCredentials();
for (int i = 0; i < service.length; i++) {
replacedertSame(token[i], creds.getToken(service[i]));
}
replacedertEquals(service.length, creds.numberOfTokens());
}
18
Source : DelegationTokenRenewer.java
with Apache License 2.0
from naver
with Apache License 2.0
from naver
@VisibleForTesting
protected Token<?>[] obtainSystemTokensForUser(String user, final Credentials credentials) throws IOException, InterruptedException {
// Get new hdfs tokens on behalf of this user
UserGroupInformation proxyUser = UserGroupInformation.createProxyUser(user, UserGroupInformation.getLoginUser());
Token<?>[] newTokens = proxyUser.doAs(new PrivilegedExceptionAction<Token<?>[]>() {
@Override
public Token<?>[] run() throws Exception {
FileSystem fs = FileSystem.get(getConfig());
try {
return fs.addDelegationTokens(UserGroupInformation.getLoginUser().getUserName(), credentials);
} finally {
// Close the FileSystem created by the new proxy user,
// So that we don't leave an entry in the FileSystem cache
fs.close();
}
}
});
return newTokens;
}
18
Source : ApplicationAttemptStateDataPBImpl.java
with Apache License 2.0
from naver
with Apache License 2.0
from naver
@Override
public void setAppAttemptTokens(Credentials attemptTokens) {
maybeInitBuilder();
if (attemptTokens == null) {
builder.clearAppAttemptTokens();
return;
}
this.appAttemptTokens = convertCredentialsToByteBuffer(attemptTokens);
}
18
Source : ApplicationAttemptStateDataPBImpl.java
with Apache License 2.0
from naver
with Apache License 2.0
from naver
private static ByteBuffer convertCredentialsToByteBuffer(Credentials credentials) {
ByteBuffer appAttemptTokens = null;
DataOutputBuffer dob = new DataOutputBuffer();
try {
if (credentials != null) {
credentials.writeTokenStorageToStream(dob);
appAttemptTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
}
return appAttemptTokens;
} catch (IOException e) {
LOG.error("Failed to convert Credentials to ByteBuffer.");
replacedert false;
return null;
} finally {
IOUtils.closeStream(dob);
}
}
See More Examples