Here are the examples of the java api class java.util.zip.Checksum taken from open source projects.
1. FileUtilsTestCase#testChecksum()
Project: commons-io
File: FileUtilsTestCase.java
File: FileUtilsTestCase.java
@Test public void testChecksum() throws Exception { // create a test file final String text = "Imagination is more important than knowledge - Einstein"; final File file = new File(getTestDirectory(), "checksum-test.txt"); FileUtils.writeStringToFile(file, text, "US-ASCII"); // compute the expected checksum final Checksum expectedChecksum = new CRC32(); expectedChecksum.update(text.getBytes("US-ASCII"), 0, text.length()); final long expectedValue = expectedChecksum.getValue(); // compute the checksum of the file final Checksum testChecksum = new CRC32(); final Checksum resultChecksum = FileUtils.checksum(file, testChecksum); final long resultValue = resultChecksum.getValue(); assertSame(testChecksum, resultChecksum); assertEquals(expectedValue, resultValue); }
2. CrcUtils#computeCrc()
Project: pinot
File: CrcUtils.java
File: CrcUtils.java
public long computeCrc() { CheckedInputStream cis = null; InputStream is = null; final Checksum checksum = new Adler32(); final byte[] tempBuf = new byte[BUFFER_SIZE]; for (final File file : filesToProcess) { try { is = new BufferedInputStream(new FileInputStream(file)); cis = new CheckedInputStream(is, checksum); while (cis.read(tempBuf) >= 0) { } cis.close(); is.close(); } catch (final Exception e) { LOGGER.error("Caught exception while computing CRC", e); Utils.rethrowException(e); throw new AssertionError("Should not reach this"); } } return checksum.getValue(); }
3. ChecksumHashFunctionTest#assertChecksum()
Project: guava
File: ChecksumHashFunctionTest.java
File: ChecksumHashFunctionTest.java
private static void assertChecksum(Supplier<Checksum> supplier, String input) { byte[] bytes = HashTestUtils.ascii(input); Checksum checksum = supplier.get(); checksum.update(bytes, 0, bytes.length); long value = checksum.getValue(); String toString = "name"; HashFunction func = new ChecksumHashFunction(supplier, 32, toString); assertEquals(toString, func.toString()); assertEquals(value, func.hashBytes(bytes).padToLong()); }
4. EVCacheClient#checkCRCChecksum()
Project: EVCache
File: EVCacheClient.java
File: EVCacheClient.java
private boolean checkCRCChecksum(byte[] data, final ChunkInfo ci, boolean hasZF) { if (data == null || data.length == 0) return false; final Checksum checksum = new CRC32(); checksum.update(data, 0, data.length); final long currentChecksum = checksum.getValue(); final long expectedChecksum = ci.getChecksum(); if (log.isDebugEnabled()) log.debug("CurrentChecksum : " + currentChecksum + "; ExpectedChecksum : " + expectedChecksum + " for key : " + ci.getKey()); if (currentChecksum != expectedChecksum) { if (!hasZF) { if (log.isWarnEnabled()) log.warn("CHECKSUM_ERROR : Chunks : " + ci.getChunks() + " ; " + "currentChecksum : " + currentChecksum + "; expectedChecksum : " + expectedChecksum + " for key : " + ci.getKey()); EVCacheMetricsFactory.increment(appName + "-CHECK_SUM_ERROR"); } return false; } return true; }
5. FileUtilsTestCase#testChecksumCRC32()
Project: commons-io
File: FileUtilsTestCase.java
File: FileUtilsTestCase.java
//----------------------------------------------------------------------- @Test public void testChecksumCRC32() throws Exception { // create a test file final String text = "Imagination is more important than knowledge - Einstein"; final File file = new File(getTestDirectory(), "checksum-test.txt"); FileUtils.writeStringToFile(file, text, "US-ASCII"); // compute the expected checksum final Checksum expectedChecksum = new CRC32(); expectedChecksum.update(text.getBytes("US-ASCII"), 0, text.length()); final long expectedValue = expectedChecksum.getValue(); // compute the checksum of the file final long resultValue = FileUtils.checksumCRC32(file); assertEquals(expectedValue, resultValue); }
6. HadoopFileCacheRepository#computeChecksum()
Project: asakusafw
File: HadoopFileCacheRepository.java
File: HadoopFileCacheRepository.java
private long computeChecksum(FileSystem fs, Path file) throws IOException { if (LOG.isDebugEnabled()) { LOG.debug(MessageFormat.format(//$NON-NLS-1$ "Computing checksum: {0}", file)); } Checksum checksum = new CRC32(); byte[] buf = byteBuffers.get(); try (FSDataInputStream input = fs.open(file)) { while (true) { int read = input.read(buf); if (read < 0) { break; } checksum.update(buf, 0, read); } } return checksum.getValue(); }
7. TestBufferedChecksum#testSimple()
Project: lucene-solr
File: TestBufferedChecksum.java
File: TestBufferedChecksum.java
public void testSimple() { Checksum c = new BufferedChecksum(new CRC32()); c.update(1); c.update(2); c.update(3); assertEquals(1438416925L, c.getValue()); }
8. FileUtilsTestCase#testChecksumDouble()
Project: commons-io
File: FileUtilsTestCase.java
File: FileUtilsTestCase.java
@Test public void testChecksumDouble() throws Exception { // create a test file final String text1 = "Imagination is more important than knowledge - Einstein"; final File file1 = new File(getTestDirectory(), "checksum-test.txt"); FileUtils.writeStringToFile(file1, text1, "US-ASCII"); // create a second test file final String text2 = "To be or not to be - Shakespeare"; final File file2 = new File(getTestDirectory(), "checksum-test2.txt"); FileUtils.writeStringToFile(file2, text2, "US-ASCII"); // compute the expected checksum final Checksum expectedChecksum = new CRC32(); expectedChecksum.update(text1.getBytes("US-ASCII"), 0, text1.length()); expectedChecksum.update(text2.getBytes("US-ASCII"), 0, text2.length()); final long expectedValue = expectedChecksum.getValue(); // compute the checksum of the file final Checksum testChecksum = new CRC32(); FileUtils.checksum(file1, testChecksum); FileUtils.checksum(file2, testChecksum); final long resultValue = testChecksum.getValue(); assertEquals(expectedValue, resultValue); }
9. CRC32ChecksumUtils#getCRC32Checksum()
Project: ngrinder
File: CRC32ChecksumUtils.java
File: CRC32ChecksumUtils.java
/** * Generate CRC32 Checksum For Byte Array. * * @param bytes byte array * @return checksum CRC32 checksum value * @since 3.3 */ public static long getCRC32Checksum(byte[] bytes) { Checksum checksum = new CRC32(); checksum.update(bytes, 0, bytes.length); return checksum.getValue(); }
10. Encrypt#computeChecksum()
Project: JGroups
File: Encrypt.java
File: Encrypt.java
protected long computeChecksum(byte[] input, int offset, int length) { Checksum checksummer = createChecksummer(); checksummer.update(input, offset, length); return checksummer.getValue(); }
11. ChecksumUtil#validateBlockChecksum()
Project: hindex
File: ChecksumUtil.java
File: ChecksumUtil.java
/** * Validates that the data in the specified HFileBlock matches the * checksum. Generates the checksum for the data and * then validate that it matches the value stored in the header. * If there is a checksum mismatch, then return false. Otherwise * return true. * The header is extracted from the specified HFileBlock while the * data-to-be-verified is extracted from 'data'. */ static boolean validateBlockChecksum(Path path, HFileBlock block, byte[] data, int hdrSize) throws IOException { // better to return false to indicate a checksum validation failure. if (block.getMinorVersion() < HFileBlock.MINOR_VERSION_WITH_CHECKSUM) { return false; } // Get a checksum object based on the type of checksum that is // set in the HFileBlock header. A ChecksumType.NULL indicates that // the caller is not interested in validating checksums, so we // always return true. ChecksumType cktype = ChecksumType.codeToType(block.getChecksumType()); if (cktype == ChecksumType.NULL) { // No checkums validations needed for this block. return true; } Checksum checksumObject = cktype.getChecksumObject(); checksumObject.reset(); // read in the stored value of the checksum size from the header. int bytesPerChecksum = block.getBytesPerChecksum(); // bytesPerChecksum is always larger than the size of the header if (bytesPerChecksum < hdrSize) { String msg = "Unsupported value of bytesPerChecksum. " + " Minimum is " + hdrSize + " but the configured value is " + bytesPerChecksum; HFile.LOG.warn(msg); // cannot happen case, unable to verify checksum return false; } // Extract the header and compute checksum for the header. ByteBuffer hdr = block.getBufferWithHeader(); checksumObject.update(hdr.array(), hdr.arrayOffset(), hdrSize); int off = hdrSize; int consumed = hdrSize; int bytesLeft = block.getOnDiskDataSizeWithHeader() - off; int cksumOffset = block.getOnDiskDataSizeWithHeader(); // validate each chunk while (bytesLeft > 0) { int thisChunkSize = bytesPerChecksum - consumed; int count = Math.min(bytesLeft, thisChunkSize); checksumObject.update(data, off, count); int storedChecksum = Bytes.toInt(data, cksumOffset); if (storedChecksum != (int) checksumObject.getValue()) { String msg = "File " + path + " Stored checksum value of " + storedChecksum + " at offset " + cksumOffset + " does not match computed checksum " + checksumObject.getValue() + ", total data size " + data.length + " Checksum data range offset " + off + " len " + count + HFileBlock.toStringHeader(block.getBufferReadOnly()); HFile.LOG.warn(msg); if (generateExceptions) { // this is only for unit tests throw new IOException(msg); } else { // checksum validation failure return false; } } cksumOffset += HFileBlock.CHECKSUM_SIZE; bytesLeft -= count; off += count; consumed = 0; checksumObject.reset(); } // checksum is valid return true; }
12. IOUtils#copyBytesAndGenerateCRC()
Project: hadoop-20
File: IOUtils.java
File: IOUtils.java
/** * Copies from one stream to another and generate CRC checksum. * @param in InputStrem to read from * @param out OutputStream to write to * @param buffSize the size of the buffer * @param close whether or not close the InputStream and * @param throttler IO Throttler * OutputStream at the end. The streams are closed in the finally clause. * @return CRC checksum */ public static long copyBytesAndGenerateCRC(InputStream in, OutputStream out, int buffSize, boolean close, IOThrottler throttler) throws IOException { PrintStream ps = out instanceof PrintStream ? (PrintStream) out : null; byte buf[] = new byte[buffSize]; Checksum sum = new NativeCrc32(); sum.reset(); try { if (throttler != null) { throttler.throttle((long) buffSize); } int bytesRead = in.read(buf); while (bytesRead >= 0) { sum.update(buf, 0, bytesRead); out.write(buf, 0, bytesRead); if ((ps != null) && ps.checkError()) { throw new IOException("Unable to write to output stream."); } if (throttler != null) { throttler.throttle((long) buffSize); } bytesRead = in.read(buf); } } finally { if (close) { out.close(); in.close(); } } return sum.getValue(); }
13. BaseUnsafeTest#crc32()
Project: directmemory
File: BaseUnsafeTest.java
File: BaseUnsafeTest.java
private static long crc32(byte[] payload) { final Checksum checksum = new CRC32(); checksum.update(payload, 0, payload.length); return checksum.getValue(); }
14. BaseUnsafeTest#crc()
Project: directmemory
File: BaseUnsafeTest.java
File: BaseUnsafeTest.java
public long crc(String str) { final Checksum checksum = new CRC32(); final byte bytes[] = str.getBytes(); checksum.update(bytes, 0, bytes.length); return checksum.getValue(); }
15. BaseTest#crc32()
Project: directmemory
File: BaseTest.java
File: BaseTest.java
private static long crc32(byte[] payload) { final Checksum checksum = new CRC32(); checksum.update(payload, 0, payload.length); return checksum.getValue(); }
16. BaseTest#crc()
Project: directmemory
File: BaseTest.java
File: BaseTest.java
public long crc(String str) { final Checksum checksum = new CRC32(); final byte bytes[] = str.getBytes(); checksum.update(bytes, 0, bytes.length); return checksum.getValue(); }
17. TestBufferedChecksum#testRandom()
Project: lucene-solr
File: TestBufferedChecksum.java
File: TestBufferedChecksum.java
public void testRandom() { Checksum c1 = new CRC32(); Checksum c2 = new BufferedChecksum(new CRC32()); int iterations = atLeast(10000); for (int i = 0; i < iterations; i++) { switch(random().nextInt(4)) { case 0: // update(byte[], int, int) int length = random().nextInt(1024); byte bytes[] = new byte[length]; random().nextBytes(bytes); c1.update(bytes, 0, bytes.length); c2.update(bytes, 0, bytes.length); break; case 1: // update(int) int b = random().nextInt(256); c1.update(b); c2.update(b); break; case 2: // reset() c1.reset(); c2.reset(); break; case 3: // getValue() assertEquals(c1.getValue(), c2.getValue()); break; } } assertEquals(c1.getValue(), c2.getValue()); }
18. CommitLogTest#testRecoveryWithBadSizeArgument()
Project: stratio-cassandra
File: CommitLogTest.java
File: CommitLogTest.java
protected void testRecoveryWithBadSizeArgument(int size, int dataSize) throws Exception { Checksum checksum = new CRC32(); checksum.update(size); testRecoveryWithBadSizeArgument(size, dataSize, checksum.getValue()); }
19. CommitLogTest#testRecoveryWithBadSizeChecksum()
Project: stratio-cassandra
File: CommitLogTest.java
File: CommitLogTest.java
@Test public void testRecoveryWithBadSizeChecksum() throws Exception { Checksum checksum = new CRC32(); checksum.update(100); testRecoveryWithBadSizeArgument(100, 100, ~checksum.getValue()); }
20. EventBuilder#calculateChecksum()
Project: raven-java
File: EventBuilder.java
File: EventBuilder.java
/** * Calculates a checksum for a given string. * * @param string string from which a checksum should be obtained * @return a checksum allowing two events with the same properties to be grouped later. */ private static String calculateChecksum(String string) { byte[] bytes = string.getBytes(UTF_8); Checksum checksum = new CRC32(); checksum.update(bytes, 0, bytes.length); return Long.toHexString(checksum.getValue()).toUpperCase(); }
21. PopcornUpdater#crc32()
Project: popcorn-android-legacy
File: PopcornUpdater.java
File: PopcornUpdater.java
private static int crc32(String str) { byte bytes[] = str.getBytes(); Checksum checksum = new CRC32(); checksum.update(bytes, 0, bytes.length); return (int) checksum.getValue(); }
22. HashKernel#getCRC32()
Project: Metronome
File: HashKernel.java
File: HashKernel.java
/* feature_vector = [0 for _ in xrange(1 << bits)] for word in input_text.split(): hash = crc32(word) # CRC-32 hash function, available in the zlib module index = hash & ((1 << bits) - 1) # Use the nth bit, zero-indexed, to determine if we add or subtract # one from the index. sign = (((hash & (1 << bits)) >> bits) << 1) - 1 feature_vector[index] += sign * */ private static long getCRC32(String word) { byte bytes[] = word.getBytes(); Checksum checksum = new CRC32(); /* * To compute the CRC32 checksum for byte array, use * * void update(bytes[] b, int start, int length) * method of CRC32 class. */ checksum.update(bytes, 0, bytes.length); /* * Get the generated checksum using * getValue method of CRC32 class. */ long lngChecksum = checksum.getValue(); return lngChecksum; }
23. FileUtil#computeChecksum()
Project: indextank-engine
File: FileUtil.java
File: FileUtil.java
/** * Returns the checksum of a given directory tree. * @throws java.io.IOException */ public static long computeChecksum(String path) throws IOException { String dir = getDir(path); Checksum checksum = new CRC32(); checksumDir(checksum, dir); return checksum.getValue(); }
24. BlockWithChecksumFileWriter#computePartialChunkCrc()
Project: hadoop-20
File: BlockWithChecksumFileWriter.java
File: BlockWithChecksumFileWriter.java
/** * reads in the partial crc chunk and computes checksum of pre-existing data * in partial chunk. */ private void computePartialChunkCrc(long blkoff, long ckoff, int bytesPerChecksum, DataChecksum checksum) throws IOException { // find offset of the beginning of partial chunk. // int sizePartialChunk = (int) (blkoff % bytesPerChecksum); int checksumSize = checksum.getChecksumSize(); blkoff = blkoff - sizePartialChunk; LOG.info("computePartialChunkCrc sizePartialChunk " + sizePartialChunk + " block " + block + " offset in block " + blkoff + " offset in metafile " + ckoff); // create an input stream from the block file // and read in partial crc chunk into temporary buffer // byte[] buf = new byte[sizePartialChunk]; byte[] crcbuf = new byte[checksumSize]; FileInputStream dataIn = null, metaIn = null; try { DatanodeBlockInfo info = datanode.data.getDatanodeBlockInfo(namespaceId, block); if (info == null) { throw new IOException("Block " + block + " does not exist in volumeMap."); } File blockFile = info.getDataFileToRead(); if (blockFile == null) { blockFile = info.getBlockDataFile().getTmpFile(namespaceId, block); } RandomAccessFile blockInFile = new RandomAccessFile(blockFile, "r"); if (blkoff > 0) { blockInFile.seek(blkoff); } File metaFile = getMetaFile(blockFile, block); RandomAccessFile metaInFile = new RandomAccessFile(metaFile, "r"); if (ckoff > 0) { metaInFile.seek(ckoff); } dataIn = new FileInputStream(blockInFile.getFD()); metaIn = new FileInputStream(metaInFile.getFD()); IOUtils.readFully(dataIn, buf, 0, sizePartialChunk); // open meta file and read in crc value computer earlier IOUtils.readFully(metaIn, crcbuf, 0, crcbuf.length); } finally { if (dataIn != null) { dataIn.close(); } if (metaIn != null) { metaIn.close(); } } // compute crc of partial chunk from data read in the block file. Checksum partialCrc = new NativeCrc32(); partialCrc.update(buf, 0, sizePartialChunk); LOG.info("Read in partial CRC chunk from disk for block " + block); // recalculated just now if (partialCrc.getValue() != FSInputChecker.checksum2long(crcbuf)) { String msg = "Partial CRC " + partialCrc.getValue() + " does not match value computed the " + " last time file was closed " + FSInputChecker.checksum2long(crcbuf); throw new IOException(msg); } // LOG.debug("Partial CRC matches 0x" + // Long.toHexString(partialCrc.getValue())); partialCrcInt = (int) partialCrc.getValue(); }
25. BlockInlineChecksumWriter#computePartialChunkCrc()
Project: hadoop-20
File: BlockInlineChecksumWriter.java
File: BlockInlineChecksumWriter.java
/** * reads in the partial crc chunk and computes checksum of pre-existing data * in partial chunk. */ private void computePartialChunkCrc(long blkoff, int bytesPerChecksum, DataChecksum checksum) throws IOException { // find offset of the beginning of partial chunk. // int sizePartialChunk = (int) (blkoff % bytesPerChecksum); int checksumSize = checksum.getChecksumSize(); long fileOff = BlockInlineChecksumReader.getPosFromBlockOffset(blkoff - sizePartialChunk, bytesPerChecksum, checksumSize); LOG.info("computePartialChunkCrc sizePartialChunk " + sizePartialChunk + " block " + block + " offset in block " + blkoff); // create an input stream from the block file // and read in partial crc chunk into temporary buffer // byte[] buf = new byte[sizePartialChunk]; byte[] crcbuf = new byte[checksumSize]; // FileInputStream dataIn = null; /* RandomAccessFile blockInFile = new RandomAccessFile(blockFile, "r"); dataIn = new FileInputStream(blockInFile.getFD()); if (fileOff > 0) { blockInFile.seek(fileOff); } IOUtils.readFully(dataIn, buf, 0, sizePartialChunk); // open meta file and read in crc value computer earlier IOUtils.readFully(dataIn, crcbuf, 0, crcbuf.length); */ BlockDataFile.Reader blockReader = blockDataFile.getReader(datanode); blockReader.readFully(buf, 0, sizePartialChunk, fileOff, true); blockReader.readFully(crcbuf, 0, crcbuf.length, fileOff + sizePartialChunk, true); // compute crc of partial chunk from data read in the block file. Checksum partialCrc = new CRC32(); partialCrc.update(buf, 0, sizePartialChunk); LOG.info("Read in partial CRC chunk from disk for block " + block); // recalculated just now if (partialCrc.getValue() != FSInputChecker.checksum2long(crcbuf)) { String msg = "Partial CRC " + partialCrc.getValue() + " does not match value computed the " + " last time file was closed " + FSInputChecker.checksum2long(crcbuf); throw new IOException(msg); } partialCrcInt = (int) partialCrc.getValue(); }
26. CRCChecksumChecker#generateChecksum()
Project: fred
File: CRCChecksumChecker.java
File: CRCChecksumChecker.java
@Override public byte[] generateChecksum(byte[] data, int offset, int length) { Checksum crc = new CRC32(); crc.update(data, offset, length); return Fields.intToBytes((int) crc.getValue()); }
27. CRCChecksumChecker#appendChecksum()
Project: fred
File: CRCChecksumChecker.java
File: CRCChecksumChecker.java
@Override public byte[] appendChecksum(byte[] data) { byte[] output = new byte[data.length + 4]; System.arraycopy(data, 0, output, 0, data.length); Checksum crc = new CRC32(); crc.update(data, 0, data.length); byte[] checksum = Fields.intToBytes((int) crc.getValue()); System.arraycopy(checksum, 0, output, data.length, 4); return output; }
28. EVCacheClient#createChunks()
Project: EVCache
File: EVCacheClient.java
File: EVCacheClient.java
private CachedData[] createChunks(CachedData cd, String key) { final int cSize = chunkSize.get(); if ((key.length() + 3) > cSize) throw new IllegalArgumentException("The chunksize " + cSize + " is smaller than the key size. Will not be able to proceed. key size = " + key.length()); final int len = cd.getData().length; /* the format of headers in memcached */ // Key size + 1 + Header( Flags (Characters Number) + Key (Characters Numbers) + 2 bytes ( \r\n ) + 4 bytes (2 spaces and 1 \r)) + Chunk Size + CAS Size // final int overheadSize = key.length() // Key Size // + 1 // Space // + 4 // Flags (Characters Number) // + 4 // Key (Characters Numbers) // + 2 // /r/n // + 4 // 2 spaces and 1 \r // + 48 // Header Size // + 8; // CAS final int overheadSize = key.length() + 71 + 3; // 3 because we will suffix _00, _01 ... _99; 68 is the size of the memcached header final int actualChunkSize = cSize - overheadSize; int lastChunkSize = len % actualChunkSize; final int numOfChunks = len / actualChunkSize + ((lastChunkSize > 0) ? 1 : 0) + 1; final CachedData[] chunkData = new CachedData[numOfChunks]; if (lastChunkSize == 0) lastChunkSize = actualChunkSize; final long sTime = System.nanoTime(); final Checksum checksum = new CRC32(); checksum.update(cd.getData(), 0, len); final long checkSumValue = checksum.getValue(); int srcPos = 0; if (log.isDebugEnabled()) log.debug("Ths size of data is " + len + " ; we will create " + (numOfChunks - 1) + " of " + actualChunkSize + " bytes. Checksum : " + checkSumValue + "; Checksum Duration : " + (System.nanoTime() - sTime)); chunkData[0] = decodingTranscoder.encode(numOfChunks + ":" + actualChunkSize + ":" + lastChunkSize + ":" + cd.getFlags() + ":" + checkSumValue); for (int i = 1; i < numOfChunks; i++) { int lengthOfArray = actualChunkSize; if (srcPos + actualChunkSize > len) { lengthOfArray = len - srcPos; } byte[] dest = new byte[actualChunkSize]; System.arraycopy(cd.getData(), srcPos, dest, 0, lengthOfArray); if (actualChunkSize > lengthOfArray) { for (int j = lengthOfArray; j < actualChunkSize; j++) { // Adding filler data dest[j] = Character.UNASSIGNED; } } srcPos += lengthOfArray; chunkData[i] = decodingTranscoder.encode(dest); } EVCacheConfig.getInstance().getDistributionSummary(appName + "-ChunkData-NumberOfChunks").record(numOfChunks); EVCacheConfig.getInstance().getDistributionSummary(appName + "-ChunkData-TotalSize").record(len); return chunkData; }
29. CommitLogTest#testRecoveryWithBadSizeArgument()
Project: cassandra
File: CommitLogTest.java
File: CommitLogTest.java
protected void testRecoveryWithBadSizeArgument(int size, int dataSize) throws Exception { Checksum checksum = new CRC32(); checksum.update(size); testRecoveryWithBadSizeArgument(size, dataSize, checksum.getValue()); }
30. CommitLogTest#testRecoveryWithBadSizeChecksum()
Project: cassandra
File: CommitLogTest.java
File: CommitLogTest.java
@Test public void testRecoveryWithBadSizeChecksum() throws Exception { Checksum checksum = new CRC32(); checksum.update(100); testRecoveryWithBadSizeArgument(100, 100, ~checksum.getValue()); }
31. FastLzFrameEncoder#encode()
Project: netty
File: FastLzFrameEncoder.java
File: FastLzFrameEncoder.java
@Override protected void encode(ChannelHandlerContext ctx, ByteBuf in, ByteBuf out) throws Exception { final Checksum checksum = this.checksum; for (; ; ) { if (!in.isReadable()) { return; } final int idx = in.readerIndex(); final int length = Math.min(in.readableBytes(), MAX_CHUNK_LENGTH); final int outputIdx = out.writerIndex(); out.setMedium(outputIdx, MAGIC_NUMBER); int outputOffset = outputIdx + CHECKSUM_OFFSET + (checksum != null ? 4 : 0); final byte blockType; final int chunkLength; if (length < MIN_LENGTH_TO_COMPRESSION) { blockType = BLOCK_TYPE_NON_COMPRESSED; out.ensureWritable(outputOffset + 2 + length); final byte[] output = out.array(); final int outputPtr = out.arrayOffset() + outputOffset + 2; if (checksum != null) { final byte[] input; final int inputPtr; if (in.hasArray()) { input = in.array(); inputPtr = in.arrayOffset() + idx; } else { input = new byte[length]; in.getBytes(idx, input); inputPtr = 0; } checksum.reset(); checksum.update(input, inputPtr, length); out.setInt(outputIdx + CHECKSUM_OFFSET, (int) checksum.getValue()); System.arraycopy(input, inputPtr, output, outputPtr, length); } else { in.getBytes(idx, output, outputPtr, length); } chunkLength = length; } else { // try to compress final byte[] input; final int inputPtr; if (in.hasArray()) { input = in.array(); inputPtr = in.arrayOffset() + idx; } else { input = new byte[length]; in.getBytes(idx, input); inputPtr = 0; } if (checksum != null) { checksum.reset(); checksum.update(input, inputPtr, length); out.setInt(outputIdx + CHECKSUM_OFFSET, (int) checksum.getValue()); } final int maxOutputLength = calculateOutputBufferLength(length); out.ensureWritable(outputOffset + 4 + maxOutputLength); final byte[] output = out.array(); final int outputPtr = out.arrayOffset() + outputOffset + 4; final int compressedLength = compress(input, inputPtr, length, output, outputPtr, level); if (compressedLength < length) { blockType = BLOCK_TYPE_COMPRESSED; chunkLength = compressedLength; out.setShort(outputOffset, chunkLength); outputOffset += 2; } else { blockType = BLOCK_TYPE_NON_COMPRESSED; System.arraycopy(input, inputPtr, output, outputPtr - 2, length); chunkLength = length; } } out.setShort(outputOffset, length); out.setByte(outputIdx + OPTIONS_OFFSET, blockType | (checksum != null ? BLOCK_WITH_CHECKSUM : BLOCK_WITHOUT_CHECKSUM)); out.writerIndex(outputOffset + 2 + chunkLength); in.skipBytes(length); } }
32. GenSort#outputRecords()
Project: incubator-tez
File: GenSort.java
File: GenSort.java
public static void outputRecords(OutputStream out, boolean useAscii, Unsigned16 firstRecordNumber, Unsigned16 recordsToGenerate, Unsigned16 checksum) throws IOException { byte[] row = new byte[100]; Unsigned16 recordNumber = new Unsigned16(firstRecordNumber); Unsigned16 lastRecordNumber = new Unsigned16(firstRecordNumber); Checksum crc = new PureJavaCrc32(); Unsigned16 tmp = new Unsigned16(); lastRecordNumber.add(recordsToGenerate); Unsigned16 ONE = new Unsigned16(1); Unsigned16 rand = Random16.skipAhead(firstRecordNumber); while (!recordNumber.equals(lastRecordNumber)) { Random16.nextRand(rand); if (useAscii) { generateAsciiRecord(row, rand, recordNumber); } else { generateRecord(row, rand, recordNumber); } if (checksum != null) { crc.reset(); crc.update(row, 0, row.length); tmp.set(crc.getValue()); checksum.add(tmp); } recordNumber.add(ONE); out.write(row); } }
33. ChecksumUtil#generateChecksums()
Project: hindex
File: ChecksumUtil.java
File: ChecksumUtil.java
/** * Generates a checksum for all the data in indata. The checksum is * written to outdata. * @param indata input data stream * @param startOffset starting offset in the indata stream from where to * compute checkums from * @param endOffset ending offset in the indata stream upto * which checksums needs to be computed * @param outData the output buffer where checksum values are written * @param outOffset the starting offset in the outdata where the * checksum values are written * @param checksumType type of checksum * @param bytesPerChecksum number of bytes per checksum value */ static void generateChecksums(byte[] indata, int startOffset, int endOffset, byte[] outdata, int outOffset, ChecksumType checksumType, int bytesPerChecksum) throws IOException { if (checksumType == ChecksumType.NULL) { // No checkums for this block. return; } Checksum checksum = checksumType.getChecksumObject(); int bytesLeft = endOffset - startOffset; int chunkNum = 0; while (bytesLeft > 0) { // generate the checksum for one chunk checksum.reset(); int count = Math.min(bytesLeft, bytesPerChecksum); checksum.update(indata, startOffset, count); // write the checksum value to the output buffer. int cksumValue = (int) checksum.getValue(); outOffset = Bytes.putInt(outdata, outOffset, cksumValue); chunkNum++; startOffset += count; bytesLeft -= count; } }
34. GenSort#outputRecords()
Project: hadoop-mapreduce
File: GenSort.java
File: GenSort.java
public static void outputRecords(OutputStream out, boolean useAscii, Unsigned16 firstRecordNumber, Unsigned16 recordsToGenerate, Unsigned16 checksum) throws IOException { byte[] row = new byte[100]; Unsigned16 recordNumber = new Unsigned16(firstRecordNumber); Unsigned16 lastRecordNumber = new Unsigned16(firstRecordNumber); Checksum crc = new PureJavaCrc32(); Unsigned16 tmp = new Unsigned16(); lastRecordNumber.add(recordsToGenerate); Unsigned16 ONE = new Unsigned16(1); Unsigned16 rand = Random16.skipAhead(firstRecordNumber); while (!recordNumber.equals(lastRecordNumber)) { Random16.nextRand(rand); if (useAscii) { generateAsciiRecord(row, rand, recordNumber); } else { generateRecord(row, rand, recordNumber); } if (checksum != null) { crc.reset(); crc.update(row, 0, row.length); tmp.set(crc.getValue()); checksum.add(tmp); } recordNumber.add(ONE); out.write(row); } }