org.apache.hadoop.security.Credentials

Here are the examples of the java api org.apache.hadoop.security.Credentials taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.

234 Examples 7

19 Source : DrillFileSystem.java
with Apache License 2.0
from zpochen

@Override
@LimitedPrivate({ "HDFS", "MapReduce" })
public Token<?>[] addDelegationTokens(String renewer, Credentials credentials) throws IOException {
    return underlyingFs.addDelegationTokens(renewer, credentials);
}

19 Source : MockRM.java
with Apache License 2.0
from NJUJYB

public RMApp submitApp(int masterMemory, String name, String user, Map<ApplicationAccessType, String> acls, boolean unmanaged, String queue, int maxAppAttempts, Credentials ts, String appType) throws Exception {
    return submitApp(masterMemory, name, user, acls, unmanaged, queue, maxAppAttempts, ts, appType, true);
}

19 Source : MockRM.java
with Apache License 2.0
from NJUJYB

public RMApp submitApp(int masterMemory, String name, String user, Map<ApplicationAccessType, String> acls, boolean unmanaged, String queue, int maxAppAttempts, Credentials ts, String appType, boolean waitForAccepted) throws Exception {
    return submitApp(masterMemory, name, user, acls, unmanaged, queue, maxAppAttempts, ts, appType, waitForAccepted, false);
}

19 Source : MockRM.java
with Apache License 2.0
from NJUJYB

public RMApp submitApp(int masterMemory, String name, String user, Map<ApplicationAccessType, String> acls, boolean unmanaged, String queue, int maxAppAttempts, Credentials ts) throws Exception {
    return submitApp(masterMemory, name, user, acls, unmanaged, queue, maxAppAttempts, ts, null);
}

19 Source : MockRM.java
with Apache License 2.0
from NJUJYB

public RMApp submitApp(int masterMemory, String name, String user, Map<ApplicationAccessType, String> acls, boolean unmanaged, String queue, int maxAppAttempts, Credentials ts, String appType, boolean waitForAccepted, boolean keepContainers, boolean isAppIdProvided, ApplicationId applicationId) throws Exception {
    return submitApp(masterMemory, name, user, acls, unmanaged, queue, maxAppAttempts, ts, appType, waitForAccepted, keepContainers, isAppIdProvided, applicationId, 0, null, true);
}

19 Source : MockRM.java
with Apache License 2.0
from NJUJYB

public RMApp submitApp(int masterMemory, String name, String user, Map<ApplicationAccessType, String> acls, boolean unmanaged, String queue, int maxAppAttempts, Credentials ts, String appType, boolean waitForAccepted, boolean keepContainers) throws Exception {
    return submitApp(masterMemory, name, user, acls, unmanaged, queue, maxAppAttempts, ts, appType, waitForAccepted, keepContainers, false, null, 0, null, true);
}

19 Source : DelegationTokenRenewer.java
with Apache License 2.0
from NJUJYB

/**
 * Synchronously renew delegation tokens.
 * @param user user
 */
public void addApplicationSync(ApplicationId applicationId, Credentials ts, boolean shouldCancelAtEnd, String user) throws IOException, InterruptedException {
    handleAppSubmitEvent(new DelegationTokenRenewerAppSubmitEvent(applicationId, ts, shouldCancelAtEnd, user));
}

19 Source : DelegationTokenRenewer.java
with Apache License 2.0
from NJUJYB

/**
 * Asynchronously add application tokens for renewal.
 * @param applicationId added application
 * @param ts tokens
 * @param shouldCancelAtEnd true if tokens should be canceled when the app is
 * done else false.
 * @param user user
 * @throws IOException
 */
public void addApplicationAsync(ApplicationId applicationId, Credentials ts, boolean shouldCancelAtEnd, String user) {
    processDelegationTokenRenewerEvent(new DelegationTokenRenewerAppSubmitEvent(applicationId, ts, shouldCancelAtEnd, user));
}

19 Source : LogAggregationService.java
with Apache License 2.0
from NJUJYB

@SuppressWarnings("unchecked")
private void initApp(final ApplicationId appId, String user, Credentials credentials, ContainerLogsRetentionPolicy logRetentionPolicy, Map<ApplicationAccessType, String> appAcls, LogAggregationContext logAggregationContext) {
    ApplicationEvent eventResponse;
    try {
        verifyAndCreateRemoteLogDir(getConfig());
        initAppAggregator(appId, user, credentials, logRetentionPolicy, appAcls, logAggregationContext);
        eventResponse = new ApplicationEvent(appId, ApplicationEventType.APPLICATION_LOG_HANDLING_INITED);
    } catch (YarnRuntimeException e) {
        LOG.warn("Application failed to init aggregation", e);
        eventResponse = new ApplicationEvent(appId, ApplicationEventType.APPLICATION_LOG_HANDLING_FAILED);
    }
    this.dispatcher.getEventHandler().handle(eventResponse);
}

19 Source : ContainerManagerImpl.java
with Apache License 2.0
from NJUJYB

private Credentials parseCredentials(ContainerLaunchContext launchContext) throws IOException {
    Credentials credentials = new Credentials();
    // //////////// Parse credentials
    ByteBuffer tokens = launchContext.getTokens();
    if (tokens != null) {
        DataInputByteBuffer buf = new DataInputByteBuffer();
        tokens.rewind();
        buf.reset(tokens);
        credentials.readTokenStorageStream(buf);
        if (LOG.isDebugEnabled()) {
            for (Token<? extends TokenIdentifier> tk : credentials.getAllTokens()) {
                LOG.debug(tk.getService() + " = " + tk.toString());
            }
        }
    }
    // //////////// End of parsing credentials
    return credentials;
}

19 Source : YarnClientImpl.java
with Apache License 2.0
from NJUJYB

private void addTimelineDelegationToken(ContainerLaunchContext clc) throws YarnException, IOException {
    Credentials credentials = new Credentials();
    DataInputByteBuffer dibb = new DataInputByteBuffer();
    ByteBuffer tokens = clc.getTokens();
    if (tokens != null) {
        dibb.reset(tokens);
        credentials.readTokenStorageStream(dibb);
        tokens.rewind();
    }
    // If the timeline delegation token is already in the CLC, no need to add
    // one more
    for (org.apache.hadoop.security.token.Token<? extends TokenIdentifier> token : credentials.getAllTokens()) {
        if (token.getKind().equals(TimelineDelegationTokenIdentifier.KIND_NAME)) {
            return;
        }
    }
    org.apache.hadoop.security.token.Token<TimelineDelegationTokenIdentifier> timelineDelegationToken = getTimelineDelegationToken();
    if (timelineDelegationToken == null) {
        return;
    }
    credentials.addToken(timelineService, timelineDelegationToken);
    if (LOG.isDebugEnabled()) {
        LOG.debug("Add timline delegation token into credentials: " + timelineDelegationToken);
    }
    DataOutputBuffer dob = new DataOutputBuffer();
    credentials.writeTokenStorageToStream(dob);
    tokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
    clc.setTokens(tokens);
}

19 Source : CopyListing.java
with Apache License 2.0
from NJUJYB

/**
 * The CopyListing abstraction is responsible for how the list of
 * sources and targets is constructed, for DistCp's copy function.
 * The copy-listing should be a SequenceFile<Text, CopyListingFileStatus>,
 * located at the path specified to buildListing(),
 * each entry being a pair of (Source relative path, source file status),
 * all the paths being fully qualified.
 */
public abstract clreplaced CopyListing extends Configured {

    private Credentials credentials;

    /**
     * Build listing function creates the input listing that distcp uses to
     * perform the copy.
     *
     * The build listing is a sequence file that has relative path of a file in the key
     * and the file status information of the source file in the value
     *
     * For instance if the source path is /tmp/data and the traversed path is
     * /tmp/data/dir1/dir2/file1, then the sequence file would contain
     *
     * key: /dir1/dir2/file1 and value: FileStatus(/tmp/data/dir1/dir2/file1)
     *
     * File would also contain directory entries. Meaning, if /tmp/data/dir1/dir2/file1
     * is the only file under /tmp/data, the resulting sequence file would contain the
     * following entries
     *
     * key: /dir1 and value: FileStatus(/tmp/data/dir1)
     * key: /dir1/dir2 and value: FileStatus(/tmp/data/dir1/dir2)
     * key: /dir1/dir2/file1 and value: FileStatus(/tmp/data/dir1/dir2/file1)
     *
     * Cases requiring special handling:
     * If source path is a file (/tmp/file1), contents of the file will be as follows
     *
     * TARGET DOES NOT EXIST: Key-"", Value-FileStatus(/tmp/file1)
     * TARGET IS FILE       : Key-"", Value-FileStatus(/tmp/file1)
     * TARGET IS DIR        : Key-"/file1", Value-FileStatus(/tmp/file1)
     *
     * @param pathToListFile - Output file where the listing would be stored
     * @param options - Input options to distcp
     * @throws IOException - Exception if any
     */
    public final void buildListing(Path pathToListFile, DistCpOptions options) throws IOException {
        validatePaths(options);
        doBuildListing(pathToListFile, options);
        Configuration config = getConf();
        config.set(DistCpConstants.CONF_LABEL_LISTING_FILE_PATH, pathToListFile.toString());
        config.setLong(DistCpConstants.CONF_LABEL_TOTAL_BYTES_TO_BE_COPIED, getBytesToCopy());
        config.setLong(DistCpConstants.CONF_LABEL_TOTAL_NUMBER_OF_RECORDS, getNumberOfPaths());
        validateFinalListing(pathToListFile, options);
    }

    /**
     * Validate input and output paths
     *
     * @param options - Input options
     * @throws InvalidInputException: If inputs are invalid
     * @throws IOException: any Exception with FS
     */
    protected abstract void validatePaths(DistCpOptions options) throws IOException, InvalidInputException;

    /**
     * The interface to be implemented by sub-clreplacedes, to create the source/target file listing.
     * @param pathToListFile Path on HDFS where the listing file is written.
     * @param options Input Options for DistCp (indicating source/target paths.)
     * @throws IOException: Thrown on failure to create the listing file.
     */
    protected abstract void doBuildListing(Path pathToListFile, DistCpOptions options) throws IOException;

    /**
     * Return the total bytes that distCp should copy for the source paths
     * This doesn't consider whether file is same should be skipped during copy
     *
     * @return total bytes to copy
     */
    protected abstract long getBytesToCopy();

    /**
     * Return the total number of paths to distcp, includes directories as well
     * This doesn't consider whether file/dir is already present and should be skipped during copy
     *
     * @return Total number of paths to distcp
     */
    protected abstract long getNumberOfPaths();

    /**
     * Validate the final resulting path listing.  Checks if there are duplicate
     * entries.  If preserving ACLs, checks that file system can support ACLs.
     * If preserving XAttrs, checks that file system can support XAttrs.
     *
     * @param pathToListFile - path listing build by doBuildListing
     * @param options - Input options to distcp
     * @throws IOException - Any issues while checking for duplicates and throws
     * @throws DuplicateFileException - if there are duplicates
     */
    private void validateFinalListing(Path pathToListFile, DistCpOptions options) throws DuplicateFileException, IOException {
        Configuration config = getConf();
        FileSystem fs = pathToListFile.getFileSystem(config);
        Path sortedList = DistCpUtils.sortListing(fs, config, pathToListFile);
        SequenceFile.Reader reader = new SequenceFile.Reader(config, SequenceFile.Reader.file(sortedList));
        try {
            // source relative path can never hold *
            Text lastKey = new Text("*");
            CopyListingFileStatus lastFileStatus = new CopyListingFileStatus();
            Text currentKey = new Text();
            Set<URI> aclSupportCheckFsSet = Sets.newHashSet();
            Set<URI> xAttrSupportCheckFsSet = Sets.newHashSet();
            while (reader.next(currentKey)) {
                if (currentKey.equals(lastKey)) {
                    CopyListingFileStatus currentFileStatus = new CopyListingFileStatus();
                    reader.getCurrentValue(currentFileStatus);
                    throw new DuplicateFileException("File " + lastFileStatus.getPath() + " and " + currentFileStatus.getPath() + " would cause duplicates. Aborting");
                }
                reader.getCurrentValue(lastFileStatus);
                if (options.shouldPreserve(DistCpOptions.FileAttribute.ACL)) {
                    FileSystem lastFs = lastFileStatus.getPath().getFileSystem(config);
                    URI lastFsUri = lastFs.getUri();
                    if (!aclSupportCheckFsSet.contains(lastFsUri)) {
                        DistCpUtils.checkFileSystemAclSupport(lastFs);
                        aclSupportCheckFsSet.add(lastFsUri);
                    }
                }
                if (options.shouldPreserve(DistCpOptions.FileAttribute.XATTR)) {
                    FileSystem lastFs = lastFileStatus.getPath().getFileSystem(config);
                    URI lastFsUri = lastFs.getUri();
                    if (!xAttrSupportCheckFsSet.contains(lastFsUri)) {
                        DistCpUtils.checkFileSystemXAttrSupport(lastFs);
                        xAttrSupportCheckFsSet.add(lastFsUri);
                    }
                }
                lastKey.set(currentKey);
            }
        } finally {
            IOUtils.closeStream(reader);
        }
    }

    /**
     * Protected constructor, to initialize configuration.
     * @param configuration The input configuration,
     *                        with which the source/target FileSystems may be accessed.
     * @param credentials - Credentials object on which the FS delegation tokens are cached.If null
     * delegation token caching is skipped
     */
    protected CopyListing(Configuration configuration, Credentials credentials) {
        setConf(configuration);
        setCredentials(credentials);
    }

    /**
     * set Credentials store, on which FS delegatin token will be cached
     * @param credentials - Credentials object
     */
    protected void setCredentials(Credentials credentials) {
        this.credentials = credentials;
    }

    /**
     * get credentials to update the delegation tokens for accessed FS objects
     * @return Credentials object
     */
    protected Credentials getCredentials() {
        return credentials;
    }

    /**
     * Public Factory method with which the appropriate CopyListing implementation may be retrieved.
     * @param configuration The input configuration.
     * @param credentials Credentials object on which the FS delegation tokens are cached
     * @param options The input Options, to help choose the appropriate CopyListing Implementation.
     * @return An instance of the appropriate CopyListing implementation.
     * @throws java.io.IOException - Exception if any
     */
    public static CopyListing getCopyListing(Configuration configuration, Credentials credentials, DistCpOptions options) throws IOException {
        String copyListingClreplacedName = configuration.get(DistCpConstants.CONF_LABEL_COPY_LISTING_CLreplaced, "");
        Clreplaced<? extends CopyListing> copyListingClreplaced;
        try {
            if (!copyListingClreplacedName.isEmpty()) {
                copyListingClreplaced = configuration.getClreplaced(DistCpConstants.CONF_LABEL_COPY_LISTING_CLreplaced, GlobbedCopyListing.clreplaced, CopyListing.clreplaced);
            } else {
                if (options.getSourceFileListing() == null) {
                    copyListingClreplaced = GlobbedCopyListing.clreplaced;
                } else {
                    copyListingClreplaced = FileBasedCopyListing.clreplaced;
                }
            }
            copyListingClreplacedName = copyListingClreplaced.getName();
            Constructor<? extends CopyListing> constructor = copyListingClreplaced.getDeclaredConstructor(Configuration.clreplaced, Credentials.clreplaced);
            return constructor.newInstance(configuration, credentials);
        } catch (Exception e) {
            throw new IOException("Unable to instantiate " + copyListingClreplacedName, e);
        }
    }

    static clreplaced DuplicateFileException extends RuntimeException {

        public DuplicateFileException(String message) {
            super(message);
        }
    }

    static clreplaced InvalidInputException extends RuntimeException {

        public InvalidInputException(String message) {
            super(message);
        }
    }

    public static clreplaced AclsNotSupportedException extends RuntimeException {

        public AclsNotSupportedException(String message) {
            super(message);
        }
    }

    public static clreplaced XAttrsNotSupportedException extends RuntimeException {

        public XAttrsNotSupportedException(String message) {
            super(message);
        }
    }
}

19 Source : CopyListing.java
with Apache License 2.0
from NJUJYB

/**
 * set Credentials store, on which FS delegatin token will be cached
 * @param credentials - Credentials object
 */
protected void setCredentials(Credentials credentials) {
    this.credentials = credentials;
}

19 Source : TestMerger.java
with Apache License 2.0
from NJUJYB

@Test
public void testEncryptedMerger() throws Throwable {
    jobConf.setBoolean(MRJobConfig.MR_ENCRYPTED_INTERMEDIATE_DATA, true);
    conf.setBoolean(MRJobConfig.MR_ENCRYPTED_INTERMEDIATE_DATA, true);
    Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials();
    TokenCache.setShuffleSecretKey(new byte[16], credentials);
    UserGroupInformation.getCurrentUser().addCredentials(credentials);
    testInMemoryAndOnDiskMerger();
}

19 Source : JobSubmitter.java
with Apache License 2.0
from NJUJYB

private void printTokens(JobID jobId, Credentials credentials) throws IOException {
    LOG.info("Submitting tokens for job: " + jobId);
    for (Token<?> token : credentials.getAllTokens()) {
        LOG.info(token);
    }
}

19 Source : TestDelegationToken.java
with Apache License 2.0
from NJUJYB

@Test
public void testDelegationTokenWithDoAs() throws Exception {
    final DistributedFileSystem dfs = cluster.getFileSystem();
    final Credentials creds = new Credentials();
    final Token<?>[] tokens = dfs.addDelegationTokens("JobTracker", creds);
    replacedert.replacedertEquals(1, tokens.length);
    @SuppressWarnings("unchecked")
    final Token<DelegationTokenIdentifier> token = (Token<DelegationTokenIdentifier>) tokens[0];
    final UserGroupInformation longUgi = UserGroupInformation.createRemoteUser("JobTracker/[email protected]");
    final UserGroupInformation shortUgi = UserGroupInformation.createRemoteUser("JobTracker");
    longUgi.doAs(new PrivilegedExceptionAction<Object>() {

        @Override
        public Object run() throws IOException {
            try {
                token.renew(config);
            } catch (Exception e) {
                replacedert.fail("Could not renew delegation token for user " + longUgi);
            }
            return null;
        }
    });
    shortUgi.doAs(new PrivilegedExceptionAction<Object>() {

        @Override
        public Object run() throws Exception {
            token.renew(config);
            return null;
        }
    });
    longUgi.doAs(new PrivilegedExceptionAction<Object>() {

        @Override
        public Object run() throws IOException {
            try {
                token.cancel(config);
            } catch (Exception e) {
                replacedert.fail("Could not cancel delegation token for user " + longUgi);
            }
            return null;
        }
    });
}

19 Source : TestDelegationToken.java
with Apache License 2.0
from NJUJYB

@Test
public void testAddDelegationTokensDFSApi() throws Exception {
    UserGroupInformation ugi = UserGroupInformation.createRemoteUser("JobTracker");
    DistributedFileSystem dfs = cluster.getFileSystem();
    Credentials creds = new Credentials();
    final Token<?>[] tokens = dfs.addDelegationTokens("JobTracker", creds);
    replacedert.replacedertEquals(1, tokens.length);
    replacedert.replacedertEquals(1, creds.numberOfTokens());
    checkTokenIdentifier(ugi, tokens[0]);
    final Token<?>[] tokens2 = dfs.addDelegationTokens("JobTracker", creds);
    // already have token
    replacedert.replacedertEquals(0, tokens2.length);
    replacedert.replacedertEquals(1, creds.numberOfTokens());
}

19 Source : KeyProviderDelegationTokenExtension.java
with Apache License 2.0
from NJUJYB

/**
 * Preplacedes the renewer and Credentials object to the underlying
 * {@link DelegationTokenExtension}
 * @param renewer the user allowed to renew the delegation tokens
 * @param credentials cache in which to add new delegation tokens
 * @return list of new delegation tokens
 * @throws IOException thrown if IOException if an IO error occurs.
 */
public Token<?>[] addDelegationTokens(final String renewer, Credentials credentials) throws IOException {
    return getExtension().addDelegationTokens(renewer, credentials);
}

19 Source : TestDelegationTokenRenewer.java
with Apache License 2.0
from naver

// Test FileSystem memory leak in obtainSystemTokensForUser.
@Test
public void testFSLeakInObtainSystemTokensForUser() throws Exception {
    Credentials credentials = new Credentials();
    String user = "test";
    int oldCounter = MyFS.getInstanceCounter();
    delegationTokenRenewer.obtainSystemTokensForUser(user, credentials);
    delegationTokenRenewer.obtainSystemTokensForUser(user, credentials);
    delegationTokenRenewer.obtainSystemTokensForUser(user, credentials);
    replacedert.replacedertEquals(oldCounter, MyFS.getInstanceCounter());
}

19 Source : DelegationTokenRenewer.java
with Apache License 2.0
from naver

/**
 * Asynchronously add application tokens for renewal.
 * @param applicationId added application
 * @param ts tokens
 * @param shouldCancelAtEnd true if tokens should be canceled when the app is
 * done else false.
 * @param user user
 */
public void addApplicationAsync(ApplicationId applicationId, Credentials ts, boolean shouldCancelAtEnd, String user) {
    processDelegationTokenRenewerEvent(new DelegationTokenRenewerAppSubmitEvent(applicationId, ts, shouldCancelAtEnd, user));
}

19 Source : CopyListing.java
with Apache License 2.0
from naver

/**
 * The CopyListing abstraction is responsible for how the list of
 * sources and targets is constructed, for DistCp's copy function.
 * The copy-listing should be a
 * SequenceFile<Text, CopyListingFileStatus>, located at the path
 * specified to buildListing(), each entry being a pair of (Source relative
 * path, source file status), all the paths being fully qualified.
 */
public abstract clreplaced CopyListing extends Configured {

    private Credentials credentials;

    /**
     * Build listing function creates the input listing that distcp uses to
     * perform the copy.
     *
     * The build listing is a sequence file that has relative path of a file in the key
     * and the file status information of the source file in the value
     *
     * For instance if the source path is /tmp/data and the traversed path is
     * /tmp/data/dir1/dir2/file1, then the sequence file would contain
     *
     * key: /dir1/dir2/file1 and value: FileStatus(/tmp/data/dir1/dir2/file1)
     *
     * File would also contain directory entries. Meaning, if /tmp/data/dir1/dir2/file1
     * is the only file under /tmp/data, the resulting sequence file would contain the
     * following entries
     *
     * key: /dir1 and value: FileStatus(/tmp/data/dir1)
     * key: /dir1/dir2 and value: FileStatus(/tmp/data/dir1/dir2)
     * key: /dir1/dir2/file1 and value: FileStatus(/tmp/data/dir1/dir2/file1)
     *
     * Cases requiring special handling:
     * If source path is a file (/tmp/file1), contents of the file will be as follows
     *
     * TARGET DOES NOT EXIST: Key-"", Value-FileStatus(/tmp/file1)
     * TARGET IS FILE       : Key-"", Value-FileStatus(/tmp/file1)
     * TARGET IS DIR        : Key-"/file1", Value-FileStatus(/tmp/file1)
     *
     * @param pathToListFile - Output file where the listing would be stored
     * @param options - Input options to distcp
     * @throws IOException - Exception if any
     */
    public final void buildListing(Path pathToListFile, DistCpOptions options) throws IOException {
        validatePaths(options);
        doBuildListing(pathToListFile, options);
        Configuration config = getConf();
        config.set(DistCpConstants.CONF_LABEL_LISTING_FILE_PATH, pathToListFile.toString());
        config.setLong(DistCpConstants.CONF_LABEL_TOTAL_BYTES_TO_BE_COPIED, getBytesToCopy());
        config.setLong(DistCpConstants.CONF_LABEL_TOTAL_NUMBER_OF_RECORDS, getNumberOfPaths());
        validateFinalListing(pathToListFile, options);
    }

    /**
     * Validate input and output paths
     *
     * @param options - Input options
     * @throws InvalidInputException If inputs are invalid
     * @throws IOException any Exception with FS
     */
    protected abstract void validatePaths(DistCpOptions options) throws IOException, InvalidInputException;

    /**
     * The interface to be implemented by sub-clreplacedes, to create the source/target file listing.
     * @param pathToListFile Path on HDFS where the listing file is written.
     * @param options Input Options for DistCp (indicating source/target paths.)
     * @throws IOException Thrown on failure to create the listing file.
     */
    protected abstract void doBuildListing(Path pathToListFile, DistCpOptions options) throws IOException;

    /**
     * Return the total bytes that distCp should copy for the source paths
     * This doesn't consider whether file is same should be skipped during copy
     *
     * @return total bytes to copy
     */
    protected abstract long getBytesToCopy();

    /**
     * Return the total number of paths to distcp, includes directories as well
     * This doesn't consider whether file/dir is already present and should be skipped during copy
     *
     * @return Total number of paths to distcp
     */
    protected abstract long getNumberOfPaths();

    /**
     * Validate the final resulting path listing.  Checks if there are duplicate
     * entries.  If preserving ACLs, checks that file system can support ACLs.
     * If preserving XAttrs, checks that file system can support XAttrs.
     *
     * @param pathToListFile - path listing build by doBuildListing
     * @param options - Input options to distcp
     * @throws IOException - Any issues while checking for duplicates and throws
     * @throws DuplicateFileException - if there are duplicates
     */
    private void validateFinalListing(Path pathToListFile, DistCpOptions options) throws DuplicateFileException, IOException {
        Configuration config = getConf();
        FileSystem fs = pathToListFile.getFileSystem(config);
        Path sortedList = DistCpUtils.sortListing(fs, config, pathToListFile);
        SequenceFile.Reader reader = new SequenceFile.Reader(config, SequenceFile.Reader.file(sortedList));
        try {
            // source relative path can never hold *
            Text lastKey = new Text("*");
            CopyListingFileStatus lastFileStatus = new CopyListingFileStatus();
            Text currentKey = new Text();
            Set<URI> aclSupportCheckFsSet = Sets.newHashSet();
            Set<URI> xAttrSupportCheckFsSet = Sets.newHashSet();
            while (reader.next(currentKey)) {
                if (currentKey.equals(lastKey)) {
                    CopyListingFileStatus currentFileStatus = new CopyListingFileStatus();
                    reader.getCurrentValue(currentFileStatus);
                    throw new DuplicateFileException("File " + lastFileStatus.getPath() + " and " + currentFileStatus.getPath() + " would cause duplicates. Aborting");
                }
                reader.getCurrentValue(lastFileStatus);
                if (options.shouldPreserve(DistCpOptions.FileAttribute.ACL)) {
                    FileSystem lastFs = lastFileStatus.getPath().getFileSystem(config);
                    URI lastFsUri = lastFs.getUri();
                    if (!aclSupportCheckFsSet.contains(lastFsUri)) {
                        DistCpUtils.checkFileSystemAclSupport(lastFs);
                        aclSupportCheckFsSet.add(lastFsUri);
                    }
                }
                if (options.shouldPreserve(DistCpOptions.FileAttribute.XATTR)) {
                    FileSystem lastFs = lastFileStatus.getPath().getFileSystem(config);
                    URI lastFsUri = lastFs.getUri();
                    if (!xAttrSupportCheckFsSet.contains(lastFsUri)) {
                        DistCpUtils.checkFileSystemXAttrSupport(lastFs);
                        xAttrSupportCheckFsSet.add(lastFsUri);
                    }
                }
                lastKey.set(currentKey);
            }
        } finally {
            IOUtils.closeStream(reader);
        }
    }

    /**
     * Protected constructor, to initialize configuration.
     * @param configuration The input configuration,
     *                        with which the source/target FileSystems may be accessed.
     * @param credentials - Credentials object on which the FS delegation tokens are cached.If null
     * delegation token caching is skipped
     */
    protected CopyListing(Configuration configuration, Credentials credentials) {
        setConf(configuration);
        setCredentials(credentials);
    }

    /**
     * set Credentials store, on which FS delegatin token will be cached
     * @param credentials - Credentials object
     */
    protected void setCredentials(Credentials credentials) {
        this.credentials = credentials;
    }

    /**
     * get credentials to update the delegation tokens for accessed FS objects
     * @return Credentials object
     */
    protected Credentials getCredentials() {
        return credentials;
    }

    /**
     * Public Factory method with which the appropriate CopyListing implementation may be retrieved.
     * @param configuration The input configuration.
     * @param credentials Credentials object on which the FS delegation tokens are cached
     * @param options The input Options, to help choose the appropriate CopyListing Implementation.
     * @return An instance of the appropriate CopyListing implementation.
     * @throws java.io.IOException - Exception if any
     */
    public static CopyListing getCopyListing(Configuration configuration, Credentials credentials, DistCpOptions options) throws IOException {
        if (options.shouldUseDiff()) {
            return new GlobbedCopyListing(configuration, credentials);
        }
        String copyListingClreplacedName = configuration.get(DistCpConstants.CONF_LABEL_COPY_LISTING_CLreplaced, "");
        Clreplaced<? extends CopyListing> copyListingClreplaced;
        try {
            if (!copyListingClreplacedName.isEmpty()) {
                copyListingClreplaced = configuration.getClreplaced(DistCpConstants.CONF_LABEL_COPY_LISTING_CLreplaced, GlobbedCopyListing.clreplaced, CopyListing.clreplaced);
            } else {
                if (options.getSourceFileListing() == null) {
                    copyListingClreplaced = GlobbedCopyListing.clreplaced;
                } else {
                    copyListingClreplaced = FileBasedCopyListing.clreplaced;
                }
            }
            copyListingClreplacedName = copyListingClreplaced.getName();
            Constructor<? extends CopyListing> constructor = copyListingClreplaced.getDeclaredConstructor(Configuration.clreplaced, Credentials.clreplaced);
            return constructor.newInstance(configuration, credentials);
        } catch (Exception e) {
            throw new IOException("Unable to instantiate " + copyListingClreplacedName, e);
        }
    }

    static clreplaced DuplicateFileException extends RuntimeException {

        public DuplicateFileException(String message) {
            super(message);
        }
    }

    static clreplaced InvalidInputException extends RuntimeException {

        public InvalidInputException(String message) {
            super(message);
        }
    }

    public static clreplaced AclsNotSupportedException extends RuntimeException {

        public AclsNotSupportedException(String message) {
            super(message);
        }
    }

    public static clreplaced XAttrsNotSupportedException extends RuntimeException {

        public XAttrsNotSupportedException(String message) {
            super(message);
        }
    }
}

19 Source : TestMerger.java
with Apache License 2.0
from naver

@Test
public void testEncryptedMerger() throws Throwable {
    jobConf.setBoolean(MRJobConfig.MR_ENCRYPTED_INTERMEDIATE_DATA, true);
    conf.setBoolean(MRJobConfig.MR_ENCRYPTED_INTERMEDIATE_DATA, true);
    Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials();
    TokenCache.setEncryptedSpillKey(new byte[16], credentials);
    UserGroupInformation.getCurrentUser().addCredentials(credentials);
    testInMemoryAndOnDiskMerger();
}

19 Source : YarnTestBase.java
with Apache License 2.0
from ljygz

public static boolean verifyTokenKindInContainerCredentials(final Collection<String> tokens, final String containerId) throws IOException {
    File cwd = new File("target/" + YARN_CONFIGURATION.get(TEST_CLUSTER_NAME_KEY));
    if (!cwd.exists() || !cwd.isDirectory()) {
        return false;
    }
    File containerTokens = findFile(cwd.getAbsolutePath(), new FilenameFilter() {

        @Override
        public boolean accept(File dir, String name) {
            return name.equals(containerId + ".tokens");
        }
    });
    if (containerTokens != null) {
        LOG.info("Verifying tokens in {}", containerTokens.getAbsolutePath());
        Credentials tmCredentials = Credentials.readTokenStorageFile(containerTokens, new Configuration());
        Collection<Token<? extends TokenIdentifier>> userTokens = tmCredentials.getAllTokens();
        Set<String> tokenKinds = new HashSet<>(4);
        for (Token<? extends TokenIdentifier> token : userTokens) {
            tokenKinds.add(token.getKind().toString());
        }
        return tokenKinds.containsAll(tokens);
    } else {
        LOG.warn("Unable to find credential file for container {}", containerId);
        return false;
    }
}

19 Source : HadoopOutputFormatCommonBase.java
with Apache License 2.0
from ljygz

/**
 * A common base for both "mapred" and "mapreduce" Hadoop output formats.
 *
 * <p>The base is taking care of handling (serializing) security credentials.
 */
@Internal
public abstract clreplaced HadoopOutputFormatCommonBase<T> extends RichOutputFormat<T> {

    protected transient Credentials credentials;

    protected HadoopOutputFormatCommonBase(Credentials creds) {
        this.credentials = creds;
    }

    protected void write(ObjectOutputStream out) throws IOException {
        this.credentials.write(out);
    }

    public void read(ObjectInputStream in) throws IOException {
        this.credentials = new Credentials();
        credentials.readFields(in);
    }
}

19 Source : HadoopInputFormatCommonBase.java
with Apache License 2.0
from ljygz

/**
 * A common base for both "mapred" and "mapreduce" Hadoop input formats.
 */
@Internal
public abstract clreplaced HadoopInputFormatCommonBase<T, SPITTYPE extends InputSplit> extends RichInputFormat<T, SPITTYPE> {

    protected transient Credentials credentials;

    protected HadoopInputFormatCommonBase(Credentials creds) {
        this.credentials = creds;
    }

    protected void write(ObjectOutputStream out) throws IOException {
        this.credentials.write(out);
    }

    public void read(ObjectInputStream in) throws IOException {
        this.credentials = new Credentials();
        credentials.readFields(in);
    }

    /**
     * This method only exists because there is no UserGroupInformation.getCredentials() method
     * in Hadoop 1.x
     *
     * <p>Note that this method returns "null" in Hadoop 1.x environments.
     *
     * @param ugi The user information
     * @return new credentials object from the user information. MAY RETURN NULL!
     */
    public static Credentials getCredentialsFromUGI(UserGroupInformation ugi) {
        Method getCredentialsMethod = null;
        for (Method m : ugi.getClreplaced().getMethods()) {
            if (m.getName().equals("getCredentials")) {
                getCredentialsMethod = m;
                break;
            }
        }
        if (getCredentialsMethod == null) {
            return null;
        } else {
            try {
                return (Credentials) getCredentialsMethod.invoke(ugi);
            } catch (InvocationTargetException | IllegalAccessException e) {
                throw new RuntimeException("Unable to get credentials from UserGroupInformation. This is only supported by Hadoop 2.2.0+");
            }
        }
    }
}

19 Source : CopyListing.java
with Apache License 2.0
from HotelsDotCom

/**
 * The CopyListing abstraction is responsible for how the list of sources and targets is constructed for DistCp's copy
 * function. The copy-listing should be a SequenceFile<Text, CopyListingFileStatus> located at the path specified to
 * buildListing(), each entry being a pair of (Source relative path, source file status) and all the paths being fully
 * qualified.
 */
public abstract clreplaced CopyListing extends Configured {

    private Credentials credentials;

    /**
     * Build listing function creates the input listing that S3MapReduceCp uses to perform the copy. The build listing is
     * a sequence file that has the relative path of a file in the key and the file status information of the source file
     * in the value For instance if the source path is /tmp/data and the traversed path is /tmp/data/dir1/dir2/file1, then
     * the sequence file would contain the key: /dir1/dir2/file1 and the value: FileStatus(/tmp/data/dir1/dir2/file1). The
     * file would also contain directory entries i.e if /tmp/data/dir1/dir2/file1 is the only file under /tmp/data, the
     * resulting sequence file would contain the following entries key: /dir1 and value: FileStatus(/tmp/data/dir1) key:
     * /dir1/dir2 and value: FileStatus(/tmp/data/dir1/dir2) key: /dir1/dir2/file1 and value:
     * FileStatus(/tmp/data/dir1/dir2/file1) Cases requiring special handling: If source path is a file (/tmp/file1),
     * contents of the file will be as follows TARGET DOES NOT EXIST: Key-"", Value-FileStatus(/tmp/file1) TARGET IS FILE
     * : Key-"", Value-FileStatus(/tmp/file1) TARGET IS DIR : Key-"/file1", Value-FileStatus(/tmp/file1)
     *
     * @param pathToListFile Output file where the listing would be stored
     * @param options Input options to S3MapReduceCp
     * @throws IOException Exception if any
     */
    public final void buildListing(Path pathToListFile, S3MapReduceCpOptions options) throws IOException {
        validatePath(options);
        doBuildListing(pathToListFile, options);
        Configuration config = getConf();
        config.set(S3MapReduceCpConstants.CONF_LABEL_LISTING_FILE_PATH, pathToListFile.toString());
        config.setLong(S3MapReduceCpConstants.CONF_LABEL_TOTAL_BYTES_TO_BE_COPIED, getBytesToCopy());
        config.setLong(S3MapReduceCpConstants.CONF_LABEL_TOTAL_NUMBER_OF_RECORDS, getNumberOfPaths());
        validateFinalListing(pathToListFile, options);
    }

    /**
     * Validate input and output paths
     *
     * @param options Input options
     * @throws InvalidInputException: If inputs are invalid
     * @throws IOException any Exception with FS
     */
    protected abstract void validatePath(S3MapReduceCpOptions options) throws IOException, InvalidInputException;

    /**
     * The interface to be implemented by sub-clreplacedes, to create the source/target file listing.
     *
     * @param pathToListFile Path on HDFS where the listing file is written.
     * @param options Input Options for S3MapReduceCp (indicating source/target paths.)
     * @throws IOException Thrown on failure to create the listing file.
     */
    protected abstract void doBuildListing(Path pathToListFile, S3MapReduceCpOptions options) throws IOException;

    /**
     * Return the total bytes that S3MapReduceCp should copy for the source paths This doesn't consider whether file is
     * same should be skipped during copy
     *
     * @return total bytes to copy
     */
    protected abstract long getBytesToCopy();

    /**
     * Return the total number of paths to S3MapReduceCp, includes directories as well This doesn't consider whether
     * file/dir is already present and should be skipped during copy
     *
     * @return Total number of paths to S3MapReduceCp
     */
    protected abstract long getNumberOfPaths();

    /**
     * Validate the final resulting path listing. Checks if there are duplicate entries. If preserving ACLs, checks that
     * file system can support ACLs. If preserving XAttrs, checks that file system can support XAttrs.
     *
     * @param pathToListFile path listing build by doBuildListing
     * @param options Input options to S3MapReduceCp
     * @throws IOException Any issues while checking for duplicates and throws
     * @throws DuplicateFileException if there are duplicates
     */
    private void validateFinalListing(Path pathToListFile, S3MapReduceCpOptions options) throws DuplicateFileException, IOException {
        Configuration config = getConf();
        FileSystem fs = pathToListFile.getFileSystem(config);
        Path sortedList = sortListing(fs, config, pathToListFile);
        SequenceFile.Reader reader = new SequenceFile.Reader(config, SequenceFile.Reader.file(sortedList));
        try {
            // source relative path can never hold *
            Text lastKey = new Text("*");
            CopyListingFileStatus lastFileStatus = new CopyListingFileStatus();
            Text currentKey = new Text();
            while (reader.next(currentKey)) {
                if (currentKey.equals(lastKey)) {
                    CopyListingFileStatus currentFileStatus = new CopyListingFileStatus();
                    reader.getCurrentValue(currentFileStatus);
                    throw new DuplicateFileException("File " + lastFileStatus.getPath() + " and " + currentFileStatus.getPath() + " would cause duplicates. Aborting");
                }
                reader.getCurrentValue(lastFileStatus);
                lastKey.set(currentKey);
            }
        } finally {
            IOUtils.closeStream(reader);
        }
    }

    /**
     * Sort sequence file containing FileStatus and Text as key and value respecitvely
     *
     * @param fs File System
     * @param conf Configuration
     * @param sourceListing Source listing file
     * @return Path of the sorted file. Is source file with _sorted appended to the name
     * @throws IOException Any exception during sort.
     */
    private static Path sortListing(FileSystem fs, Configuration conf, Path sourceListing) throws IOException {
        SequenceFile.Sorter sorter = new SequenceFile.Sorter(fs, Text.clreplaced, CopyListingFileStatus.clreplaced, conf);
        Path output = new Path(sourceListing.toString() + "_sorted");
        if (fs.exists(output)) {
            fs.delete(output, false);
        }
        sorter.sort(sourceListing, output);
        return output;
    }

    /**
     * Protected constructor, to initialize configuration.
     *
     * @param configuration The input configuration, with which the source/target FileSystems may be accessed.
     * @param credentials - Credentials object on which the FS delegation tokens are cached.If null delegation token
     *          caching is skipped
     */
    protected CopyListing(Configuration configuration, Credentials credentials) {
        setConf(configuration);
        setCredentials(credentials);
    }

    /**
     * set Credentials store, on which FS delegatin token will be cached
     *
     * @param credentials Credentials object
     */
    protected void setCredentials(Credentials credentials) {
        this.credentials = credentials;
    }

    /**
     * get credentials to update the delegation tokens for accessed FS objects
     *
     * @return Credentials object
     */
    protected Credentials getCredentials() {
        return credentials;
    }

    /**
     * Public Factory method with which the appropriate CopyListing implementation may be retrieved.
     *
     * @param configuration The input configuration.
     * @param credentials Credentials object on which the FS delegation tokens are cached
     * @param options The input Options, to help choose the appropriate CopyListing Implementation.
     * @return An instance of the appropriate CopyListing implementation.
     * @throws java.io.IOException Exception if any
     */
    public static CopyListing getCopyListing(Configuration configuration, Credentials credentials, S3MapReduceCpOptions options) throws IOException {
        String copyListingClreplacedName = configuration.get(S3MapReduceCpConstants.CONF_LABEL_COPY_LISTING_CLreplaced, "");
        Clreplaced<? extends CopyListing> copyListingClreplaced;
        try {
            if (!copyListingClreplacedName.isEmpty()) {
                copyListingClreplaced = configuration.getClreplaced(S3MapReduceCpConstants.CONF_LABEL_COPY_LISTING_CLreplaced, SimpleCopyListing.clreplaced, CopyListing.clreplaced);
            } else {
                copyListingClreplaced = SimpleCopyListing.clreplaced;
            }
            copyListingClreplacedName = copyListingClreplaced.getName();
            Constructor<? extends CopyListing> constructor = copyListingClreplaced.getDeclaredConstructor(Configuration.clreplaced, Credentials.clreplaced);
            return constructor.newInstance(configuration, credentials);
        } catch (NoSuchMethodException | SecurityException | InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException e) {
            throw new IOException("Unable to instantiate " + copyListingClreplacedName, e);
        }
    }

    static clreplaced DuplicateFileException extends RuntimeException {

        private static final long serialVersionUID = 1L;

        public DuplicateFileException(String message) {
            super(message);
        }
    }

    static clreplaced InvalidInputException extends RuntimeException {

        private static final long serialVersionUID = 1L;

        public InvalidInputException(String message) {
            super(message);
        }
    }
}

19 Source : CopyListing.java
with Apache License 2.0
from HotelsDotCom

/**
 * set Credentials store, on which FS delegatin token will be cached
 *
 * @param credentials Credentials object
 */
protected void setCredentials(Credentials credentials) {
    this.credentials = credentials;
}

19 Source : HadoopFileSystemWrapper.java
with Apache License 2.0
from dremio

@Override
@LimitedPrivate({ "HDFS", "MapReduce" })
public Token<?>[] addDelegationTokens(String renewer, Credentials credentials) throws IOException {
    try (WaitRecorder recorder = OperatorStats.getWaitRecorder(operatorStats)) {
        return underlyingFs.addDelegationTokens(renewer, credentials);
    } catch (FSError e) {
        throw propagateFSError(e);
    }
}

19 Source : HoodieWrapperFileSystem.java
with Apache License 2.0
from apache

@Override
public Token<?>[] addDelegationTokens(String renewer, Credentials credentials) throws IOException {
    return fileSystem.addDelegationTokens(renewer, credentials);
}

19 Source : YarnTwillPreparer.java
with Apache License 2.0
from apache

/**
 * Creates an {@link Credentials} by copying the {@link Credentials} of the current user.
 */
private Credentials createCredentials() {
    Credentials credentials = new Credentials();
    try {
        credentials.addAll(UserGroupInformation.getCurrentUser().getCredentials());
    } catch (IOException e) {
        LOG.warn("Failed to get current user UGI. Current user credentials not added.", e);
    }
    return credentials;
}

19 Source : YarnSecureStore.java
with Apache License 2.0
from apache

/**
 * A {@link SecureStore} for hadoop credentials.
 */
public final clreplaced YarnSecureStore implements SecureStore {

    private final Credentials credentials;

    public static SecureStore create(Credentials credentials) {
        return new YarnSecureStore(credentials);
    }

    private YarnSecureStore(Credentials credentials) {
        this.credentials = credentials;
    }

    @Override
    public Credentials getStore() {
        return credentials;
    }
}

19 Source : YarnSecureStore.java
with Apache License 2.0
from apache

public static SecureStore create(Credentials credentials) {
    return new YarnSecureStore(credentials);
}

18 Source : StoreDiag.java
with Apache License 2.0
from steveloughran

/**
 * Dump all the user's tokens.
 * @throws IOException failure.
 */
public void dumpUserTokens() throws IOException {
    UserGroupInformation currentUser = UserGroupInformation.getCurrentUser();
    Credentials credentials = currentUser.getCredentials();
    Collection<Token<? extends TokenIdentifier>> allTokens = credentials.getAllTokens();
    for (Token<? extends TokenIdentifier> token : allTokens) {
        println("%s", token);
    }
}

18 Source : TestDelegationTokenRenewer.java
with Apache License 2.0
from NJUJYB

/**
 * Basic idea of the test:
 * 1. register a token for 2 seconds with no cancel at the end
 * 2. cancel it immediately
 * 3. Sleep and check that the 2 seconds renew didn't happen
 * (totally 5 renewals)
 * 4. check cancellation
 * @throws IOException
 * @throws URISyntaxException
 */
@Test(timeout = 60000)
public void testDTRenewalWithNoCancel() throws Exception {
    MyFS dfs = (MyFS) FileSystem.get(conf);
    LOG.info("dfs=" + (Object) dfs.hashCode() + ";conf=" + conf.hashCode());
    Credentials ts = new Credentials();
    MyToken token1 = dfs.getDelegationToken("user1");
    // to cause this one to be set for renew in 2 secs
    Renewer.tokenToRenewIn2Sec = token1;
    LOG.info("token=" + token1 + " should be renewed for 2 secs");
    String nn1 = DelegationTokenRenewer.SCHEME + "://host1:0";
    ts.addToken(new Text(nn1), token1);
    ApplicationId applicationId_1 = BuilderUtils.newApplicationId(0, 1);
    delegationTokenRenewer.addApplicationAsync(applicationId_1, ts, false, "user");
    waitForEventsToGetProcessed(delegationTokenRenewer);
    delegationTokenRenewer.applicationFinished(applicationId_1);
    waitForEventsToGetProcessed(delegationTokenRenewer);
    // number of renewals so far
    int numberOfExpectedRenewals = Renewer.counter;
    try {
        // sleep 6 seconds, so it has time to renew
        Thread.sleep(6 * 1000);
    } catch (InterruptedException e) {
    }
    LOG.info("Counter = " + Renewer.counter + ";t=" + Renewer.lastRenewed);
    // counter and the token should still be the old ones
    replacedertEquals("renew wasn't called as many times as expected", numberOfExpectedRenewals, Renewer.counter);
    // also renewing of the canceled token should not fail, because it has not
    // been canceled
    token1.renew(conf);
}

18 Source : TestDelegationTokenRenewer.java
with Apache License 2.0
from NJUJYB

/**
 * Basic idea of the test:
 * 1. create tokens.
 * 2. Mark one of them to be renewed in 2 seconds (instead of
 * 24 hours)
 * 3. register them for renewal
 * 4. sleep for 3 seconds
 * 5. count number of renewals (should 3 initial ones + one extra)
 * 6. register another token for 2 seconds
 * 7. cancel it immediately
 * 8. Sleep and check that the 2 seconds renew didn't happen
 * (totally 5 renewals)
 * 9. check cancellation
 * @throws IOException
 * @throws URISyntaxException
 */
@Test(timeout = 60000)
public void testDTRenewal() throws Exception {
    MyFS dfs = (MyFS) FileSystem.get(conf);
    LOG.info("dfs=" + (Object) dfs.hashCode() + ";conf=" + conf.hashCode());
    // Test 1. - add three tokens - make sure exactly one get's renewed
    // get the delegation tokens
    MyToken token1, token2, token3;
    token1 = dfs.getDelegationToken("user1");
    token2 = dfs.getDelegationToken("user2");
    token3 = dfs.getDelegationToken("user3");
    // to cause this one to be set for renew in 2 secs
    Renewer.tokenToRenewIn2Sec = token1;
    LOG.info("token=" + token1 + " should be renewed for 2 secs");
    // three distinct Namenodes
    String nn1 = DelegationTokenRenewer.SCHEME + "://host1:0";
    String nn2 = DelegationTokenRenewer.SCHEME + "://host2:0";
    String nn3 = DelegationTokenRenewer.SCHEME + "://host3:0";
    Credentials ts = new Credentials();
    // register the token for renewal
    ts.addToken(new Text(nn1), token1);
    ts.addToken(new Text(nn2), token2);
    ts.addToken(new Text(nn3), token3);
    // register the tokens for renewal
    ApplicationId applicationId_0 = BuilderUtils.newApplicationId(0, 0);
    delegationTokenRenewer.addApplicationAsync(applicationId_0, ts, true, "user");
    waitForEventsToGetProcessed(delegationTokenRenewer);
    // first 3 initial renewals + 1 real
    int numberOfExpectedRenewals = 3 + 1;
    int attempts = 10;
    while (attempts-- > 0) {
        try {
            // sleep 3 seconds, so it has time to renew
            Thread.sleep(3 * 1000);
        } catch (InterruptedException e) {
        }
        // since we cannot guarantee timely execution - let's give few chances
        if (Renewer.counter == numberOfExpectedRenewals)
            break;
    }
    LOG.info("dfs=" + dfs.hashCode() + ";Counter = " + Renewer.counter + ";t=" + Renewer.lastRenewed);
    replacedertEquals("renew wasn't called as many times as expected(4):", numberOfExpectedRenewals, Renewer.counter);
    replacedertEquals("most recently renewed token mismatch", Renewer.lastRenewed, token1);
    // Test 2.
    // add another token ( that expires in 2 secs). Then remove it, before
    // time is up.
    // Wait for 3 secs , and make sure no renews were called
    ts = new Credentials();
    MyToken token4 = dfs.getDelegationToken("user4");
    // to cause this one to be set for renew in 2 secs
    Renewer.tokenToRenewIn2Sec = token4;
    LOG.info("token=" + token4 + " should be renewed for 2 secs");
    String nn4 = DelegationTokenRenewer.SCHEME + "://host4:0";
    ts.addToken(new Text(nn4), token4);
    ApplicationId applicationId_1 = BuilderUtils.newApplicationId(0, 1);
    delegationTokenRenewer.addApplicationAsync(applicationId_1, ts, true, "user");
    waitForEventsToGetProcessed(delegationTokenRenewer);
    delegationTokenRenewer.applicationFinished(applicationId_1);
    waitForEventsToGetProcessed(delegationTokenRenewer);
    // number of renewals so far
    numberOfExpectedRenewals = Renewer.counter;
    try {
        // sleep 6 seconds, so it has time to renew
        Thread.sleep(6 * 1000);
    } catch (InterruptedException e) {
    }
    LOG.info("Counter = " + Renewer.counter + ";t=" + Renewer.lastRenewed);
    // counter and the token should stil be the old ones
    replacedertEquals("renew wasn't called as many times as expected", numberOfExpectedRenewals, Renewer.counter);
    // also renewing of the cancelled token should fail
    try {
        token4.renew(conf);
        fail("Renewal of cancelled token should have failed");
    } catch (InvalidToken ite) {
    // expected
    }
}

18 Source : TestRMAppTransitions.java
with Apache License 2.0
from NJUJYB

@Test(timeout = 30000)
public void testAppRecoverPath() throws IOException {
    LOG.info("--- START: testAppRecoverPath ---");
    ApplicationSubmissionContext sub = Records.newRecord(ApplicationSubmissionContext.clreplaced);
    ContainerLaunchContext clc = Records.newRecord(ContainerLaunchContext.clreplaced);
    Credentials credentials = new Credentials();
    DataOutputBuffer dob = new DataOutputBuffer();
    credentials.writeTokenStorageToStream(dob);
    ByteBuffer securityTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
    clc.setTokens(securityTokens);
    sub.setAMContainerSpec(clc);
    testCreateAppSubmittedRecovery(sub);
}

18 Source : DelegationTokenRenewer.java
with Apache License 2.0
from NJUJYB

protected Token<?>[] obtainSystemTokensForUser(String user, final Credentials credentials) throws IOException, InterruptedException {
    // Get new hdfs tokens on behalf of this user
    UserGroupInformation proxyUser = UserGroupInformation.createProxyUser(user, UserGroupInformation.getLoginUser());
    Token<?>[] newTokens = proxyUser.doAs(new PrivilegedExceptionAction<Token<?>[]>() {

        @Override
        public Token<?>[] run() throws Exception {
            return FileSystem.get(getConfig()).addDelegationTokens(UserGroupInformation.getLoginUser().getUserName(), credentials);
        }
    });
    return newTokens;
}

18 Source : LogAggregationService.java
with Apache License 2.0
from NJUJYB

protected void initAppAggregator(final ApplicationId appId, String user, Credentials credentials, ContainerLogsRetentionPolicy logRetentionPolicy, Map<ApplicationAccessType, String> appAcls, LogAggregationContext logAggregationContext) {
    // Get user's FileSystem credentials
    final UserGroupInformation userUgi = UserGroupInformation.createRemoteUser(user);
    if (credentials != null) {
        userUgi.addCredentials(credentials);
    }
    // New application
    final AppLogAggregator appLogAggregator = new AppLogAggregatorImpl(this.dispatcher, this.deletionService, getConfig(), appId, userUgi, this.nodeId, dirsHandler, getRemoteNodeLogFileForApp(appId, user), logRetentionPolicy, appAcls, logAggregationContext, this.context, getLocalFileContext(getConfig()));
    if (this.appLogAggregators.putIfAbsent(appId, appLogAggregator) != null) {
        throw new YarnRuntimeException("Duplicate initApp for " + appId);
    }
    // wait until check for existing aggregator to create dirs
    try {
        // Create the app dir
        createAppDir(user, appId, userUgi);
    } catch (Exception e) {
        appLogAggregators.remove(appId);
        closeFileSystems(userUgi);
        if (!(e instanceof YarnRuntimeException)) {
            e = new YarnRuntimeException(e);
        }
        throw (YarnRuntimeException) e;
    }
    // TODO Get the user configuration for the list of containers that need log
    // aggregation.
    // Schedule the aggregator.
    Runnable aggregatorWrapper = new Runnable() {

        public void run() {
            try {
                appLogAggregator.run();
            } finally {
                appLogAggregators.remove(appId);
                closeFileSystems(userUgi);
            }
        }
    };
    this.threadPool.execute(aggregatorWrapper);
}

18 Source : ContainerManagerImpl.java
with Apache License 2.0
from NJUJYB

private ContainerManagerApplicationProto buildAppProto(ApplicationId appId, String user, Credentials credentials, Map<ApplicationAccessType, String> appAcls, LogAggregationContext logAggregationContext) {
    ContainerManagerApplicationProto.Builder builder = ContainerManagerApplicationProto.newBuilder();
    builder.setId(((ApplicationIdPBImpl) appId).getProto());
    builder.setUser(user);
    if (logAggregationContext != null) {
        builder.setLogAggregationContext(((LogAggregationContextPBImpl) logAggregationContext).getProto());
    }
    builder.clearCredentials();
    if (credentials != null) {
        DataOutputBuffer dob = new DataOutputBuffer();
        try {
            credentials.writeTokenStorageToStream(dob);
            builder.setCredentials(ByteString.copyFrom(dob.getData()));
        } catch (IOException e) {
            // should not occur
            LOG.error("Cannot serialize credentials", e);
        }
    }
    builder.clearAcls();
    if (appAcls != null) {
        for (Map.Entry<ApplicationAccessType, String> acl : appAcls.entrySet()) {
            ApplicationACLMapProto p = ApplicationACLMapProto.newBuilder().setAccessType(ProtoUtils.convertToProtoFormat(acl.getKey())).setAcl(acl.getValue()).build();
            builder.addAcls(p);
        }
    }
    return builder.build();
}

18 Source : TestAMRMClient.java
with Apache License 2.0
from NJUJYB

@SuppressWarnings("unchecked")
private org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> getAMRMToken() throws IOException {
    Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials();
    Iterator<org.apache.hadoop.security.token.Token<?>> iter = credentials.getAllTokens().iterator();
    org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> result = null;
    while (iter.hasNext()) {
        org.apache.hadoop.security.token.Token<?> token = iter.next();
        if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) {
            if (result != null) {
                replacedert.fail("credentials has more than one AMRM token." + " token1: " + result + " token2: " + token);
            }
            result = (org.apache.hadoop.security.token.Token<AMRMTokenIdentifier>) token;
        }
    }
    return result;
}

18 Source : CredentialsTestJob.java
with Apache License 2.0
from NJUJYB

private static void checkSecrets(Credentials ts) {
    if (ts == null) {
        throw new RuntimeException("The credentials are not available");
    // fail the test
    }
    for (int i = 0; i < NUM_OF_KEYS; i++) {
        String secretName = "alias" + i;
        // get token storage and a key
        byte[] secretValue = ts.getSecretKey(new Text(secretName));
        System.out.println(secretValue);
        if (secretValue == null) {
            throw new RuntimeException("The key " + secretName + " is not available. ");
        // fail the test
        }
        String secretValueStr = new String(secretValue);
        if (!("preplacedword" + i).equals(secretValueStr)) {
            throw new RuntimeException("The key " + secretName + " is not correct. Expected value is " + ("preplacedword" + i) + ". Actual value is " + // fail the test
            secretValueStr);
        }
    }
}

18 Source : TestTokenCache.java
with Apache License 2.0
from NJUJYB

private void checkToken(Credentials creds, Token<?>... tokens) {
    replacedertEquals(tokens.length, creds.getAllTokens().size());
    for (Token<?> token : tokens) {
        Token<?> credsToken = creds.getToken(token.getService());
        replacedertTrue(credsToken != null);
        replacedertEquals(token, credsToken);
    }
}

18 Source : TestTokenCache.java
with Apache License 2.0
from NJUJYB

@Test
public void testObtainTokens() throws Exception {
    Credentials credentials = new Credentials();
    FileSystem fs = mock(FileSystem.clreplaced);
    TokenCache.obtainTokensForNamenodesInternal(fs, credentials, conf);
    verify(fs).addDelegationTokens(eq(renewer), eq(credentials));
}

18 Source : TokenCache.java
with Apache License 2.0
from NJUJYB

/**
 * store job token
 * @param t
 */
@InterfaceAudience.Private
public static void setJobToken(Token<? extends TokenIdentifier> t, Credentials credentials) {
    credentials.addToken(JOB_TOKEN, t);
}

18 Source : TokenCache.java
with Apache License 2.0
from NJUJYB

/**
 * auxiliary method to get user's secret keys..
 * @param alias
 * @return secret key from the storage
 */
public static byte[] getSecretKey(Credentials credentials, Text alias) {
    if (credentials == null)
        return null;
    return credentials.getSecretKey(alias);
}

18 Source : TokenCache.java
with Apache License 2.0
from NJUJYB

@InterfaceAudience.Private
public static byte[] getShuffleSecretKey(Credentials credentials) {
    return getSecretKey(credentials, SHUFFLE_TOKEN);
}

18 Source : TestCredentials.java
with Apache License 2.0
from NJUJYB

@Test
public void testAddTokensToUGI() {
    UserGroupInformation ugi = UserGroupInformation.createRemoteUser("someone");
    Credentials creds = new Credentials();
    for (int i = 0; i < service.length; i++) {
        creds.addToken(service[i], token[i]);
    }
    ugi.addCredentials(creds);
    creds = ugi.getCredentials();
    for (int i = 0; i < service.length; i++) {
        replacedertSame(token[i], creds.getToken(service[i]));
    }
    replacedertEquals(service.length, creds.numberOfTokens());
}

18 Source : DelegationTokenRenewer.java
with Apache License 2.0
from naver

@VisibleForTesting
protected Token<?>[] obtainSystemTokensForUser(String user, final Credentials credentials) throws IOException, InterruptedException {
    // Get new hdfs tokens on behalf of this user
    UserGroupInformation proxyUser = UserGroupInformation.createProxyUser(user, UserGroupInformation.getLoginUser());
    Token<?>[] newTokens = proxyUser.doAs(new PrivilegedExceptionAction<Token<?>[]>() {

        @Override
        public Token<?>[] run() throws Exception {
            FileSystem fs = FileSystem.get(getConfig());
            try {
                return fs.addDelegationTokens(UserGroupInformation.getLoginUser().getUserName(), credentials);
            } finally {
                // Close the FileSystem created by the new proxy user,
                // So that we don't leave an entry in the FileSystem cache
                fs.close();
            }
        }
    });
    return newTokens;
}

18 Source : ApplicationAttemptStateDataPBImpl.java
with Apache License 2.0
from naver

@Override
public void setAppAttemptTokens(Credentials attemptTokens) {
    maybeInitBuilder();
    if (attemptTokens == null) {
        builder.clearAppAttemptTokens();
        return;
    }
    this.appAttemptTokens = convertCredentialsToByteBuffer(attemptTokens);
}

18 Source : ApplicationAttemptStateDataPBImpl.java
with Apache License 2.0
from naver

private static ByteBuffer convertCredentialsToByteBuffer(Credentials credentials) {
    ByteBuffer appAttemptTokens = null;
    DataOutputBuffer dob = new DataOutputBuffer();
    try {
        if (credentials != null) {
            credentials.writeTokenStorageToStream(dob);
            appAttemptTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
        }
        return appAttemptTokens;
    } catch (IOException e) {
        LOG.error("Failed to convert Credentials to ByteBuffer.");
        replacedert false;
        return null;
    } finally {
        IOUtils.closeStream(dob);
    }
}

See More Examples