Here are the examples of the java api class java.io.DataOutput taken from open source projects.
1. ParameterVector#Serialize()
Project: Metronome
File: ParameterVector.java
File: ParameterVector.java
public byte[] Serialize() throws IOException { ByteArrayOutputStream out = new ByteArrayOutputStream(); DataOutput d = new DataOutputStream(out); d.writeDouble(this.y_partial_sum); d.writeDouble(this.y_avg); d.writeDouble(this.SSyy_partial_sum); d.writeDouble(this.SSE_partial_sum); d.writeInt(this.IterationComplete); d.writeInt(this.CurrentIteration); d.writeLong(this.batchTimeMS); d.writeInt(this.TrainedRecords); d.writeFloat(this.AvgError); MatrixWritable.writeMatrix(d, this.parameter_vector); return out.toByteArray(); }
2. ParameterVector#Serialize()
Project: Metronome
File: ParameterVector.java
File: ParameterVector.java
public byte[] Serialize() throws IOException { // DataOutput d ByteArrayOutputStream out = new ByteArrayOutputStream(); DataOutput d = new DataOutputStream(out); // d.writeUTF(src_host); d.writeInt(this.SrcWorkerPassCount); d.writeInt(this.GlobalPassCount); d.writeInt(this.IterationComplete); d.writeInt(this.CurrentIteration); d.writeInt(this.TrainedRecords); d.writeFloat(this.AvgLogLikelihood); d.writeFloat(this.PercentCorrect); // buf.write // MatrixWritable.writeMatrix(d, this.worker_gradient.getMatrix()); MatrixWritable.writeMatrix(d, this.parameter_vector); return out.toByteArray(); }
3. SAXContentHandler#writeDocument()
Project: vxquery
File: SAXContentHandler.java
File: SAXContentHandler.java
public void writeDocument(ArrayBackedValueStorage abvs) throws IOException { DataOutput out = abvs.getDataOutput(); out.write(ValueTag.NODE_TREE_TAG); byte header = NodeTreePointable.HEADER_DICTIONARY_EXISTS_MASK; if (attachTypes) { header |= NodeTreePointable.HEADER_TYPE_EXISTS_MASK; } if (createNodeIds) { header |= NodeTreePointable.HEADER_NODEID_EXISTS_MASK; } out.write(header); if (createNodeIds) { out.writeInt(nodeIdProvider.getId()); } db.writeFromCache(abvs); out.write(resultABVS.getByteArray(), resultABVS.getStartOffset(), resultABVS.getLength()); }
4. SAXContentHandler#writeElement()
Project: vxquery
File: SAXContentHandler.java
File: SAXContentHandler.java
public void writeElement() throws IOException { tempABVS.reset(); DataOutput out = tempABVS.getDataOutput(); out.write(ValueTag.NODE_TREE_TAG); byte header = NodeTreePointable.HEADER_DICTIONARY_EXISTS_MASK; if (attachTypes) { header |= NodeTreePointable.HEADER_TYPE_EXISTS_MASK; } if (createNodeIds) { header |= NodeTreePointable.HEADER_NODEID_EXISTS_MASK; } out.write(header); if (createNodeIds) { out.writeInt(nodeIdProvider.getId()); } db.writeFromCache(tempABVS); out.write(resultABVS.getByteArray(), resultABVS.getStartOffset(), resultABVS.getLength()); tvp.set(tempABVS.getByteArray(), tempABVS.getStartOffset(), tempABVS.getLength()); addNodeToTuple(tvp, tupleIndex); skipping = true; }
5. AtomizeHelper#buildStringConcatenation()
Project: vxquery
File: AtomizeHelper.java
File: AtomizeHelper.java
public static void buildStringConcatenation(SequencePointable sp, PointablePool pp, ArrayBackedValueStorage tempABVS, NodeTreePointable ntp) throws IOException { tempABVS.reset(); DataOutput out = tempABVS.getDataOutput(); out.write(ValueTag.XS_UNTYPED_ATOMIC_TAG); // Leave room for the utf-8 length out.write(0); out.write(0); buildConcatenationRec(sp, pp, out, ntp); int utflen = tempABVS.getLength() - 3; byte[] bytes = tempABVS.getByteArray(); // Patch utf-8 length at bytes 1 and 2 bytes[1] = (byte) ((utflen >>> 8) & 0xFF); bytes[2] = (byte) ((utflen >>> 0) & 0xFF); }
6. DictionaryBuilder#write()
Project: vxquery
File: DictionaryBuilder.java
File: DictionaryBuilder.java
public void write(ArrayBackedValueStorage abvs) throws IOException { DataOutput out = abvs.getDataOutput(); int sizeOffset = abvs.getLength(); out.writeInt(0); int entryCount = stringEndOffsets.getSize(); out.writeInt(entryCount); int[] entryOffsets = stringEndOffsets.getArray(); for (int i = 0; i < entryCount; ++i) { out.writeInt(entryOffsets[i]); } if (hashSlotIndexes.isEmpty()) { int[] sortedOffsets = sortedSlotIndexes.getArray(); for (int i = 0; i < entryCount; ++i) { out.writeInt(sortedOffsets[i]); } } else { for (Entry<String, Integer> me : hashSlotIndexes.entrySet()) { out.writeInt((Integer) me.getValue()); } } out.write(dataBuffer.getByteArray(), 0, dataBuffer.size()); IntegerPointable.setInteger(abvs.getByteArray(), sizeOffset, abvs.getLength() - sizeOffset); }
7. LogFileManager#storeState()
Project: activemq-activeio
File: LogFileManager.java
File: LogFileManager.java
private void storeState() throws IOException { Packet controlData = controlFile.getControlData(); if (controlData.remaining() == 0) return; DataOutput data = new DataOutputStream(new PacketOutputStream(controlData)); data.writeInt(lastLogFileId); data.writeBoolean(lastMark != null); if (lastMark != null) lastMark.writeToDataOutput(data); data.writeBoolean(loadedFromCleanShutDown); // Load each node's state LogFileNode log = firstNode; do { log.writeExternal(data); log = log.getNext(); } while (log != firstNode); controlFile.store(); }
8. MetricTimeSeries#toBytes()
Project: pinot
File: MetricTimeSeries.java
File: MetricTimeSeries.java
public byte[] toBytes() throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutput out = new DataOutputStream(baos); // write the number of timeWindows out.writeInt(timeseries.size()); // write the size of the metric buffer for each timeWindow out.writeInt(schema.getRowSizeInBytes()); for (long time : timeseries.keySet()) { out.writeLong(time); out.write(timeseries.get(time).array()); } return baos.toByteArray(); }
9. DimensionKey#toBytes()
Project: pinot
File: DimensionKey.java
File: DimensionKey.java
/** * @return * @throws IOException */ public byte[] toBytes() throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutput out = new DataOutputStream(baos); // write the number of dimensions out.writeInt(dimensionValues.length); // values for (String dimensionValue : dimensionValues) { byte[] bytes = dimensionValue.getBytes(Charset.forName("utf-8")); out.writeInt(bytes.length); out.write(bytes); } baos.close(); byte[] byteArray = baos.toByteArray(); try { DimensionKey key = fromBytes(byteArray); } catch (Exception e) { LOGGER.info("input key:{}", Arrays.toString(dimensionValues)); LOGGER.info("generated:{}", Arrays.toString(byteArray)); throw new RuntimeException(e); } return byteArray; }
10. HttpResponseImpl#writeMessage()
Project: openejb
File: HttpResponseImpl.java
File: HttpResponseImpl.java
/** Takes care of sending the response line, headers and body * * HTTP/1.1 200 OK * Server: Netscape-Enterprise/3.6 SP3 * Date: Thu, 07 Jun 2001 17:30:42 GMT * Content-Type: text/html * Connection: close * @param output the output to send the response to * @throws IOException if an exception is thrown */ protected void writeMessage(OutputStream output) throws IOException { DataOutput out = new DataOutputStream(output); DataOutput log = new DataOutputStream(System.out); //System.out.println("\nRESPONSE"); closeMessage(); //writeResponseLine(log); // writeHeaders(log); // writeBody(log); writeResponseLine(out); writeHeaders(out); writeBody(out); }
11. FnCurrentTimeScalarEvaluatorFactory#createEvaluator()
Project: vxquery
File: FnCurrentTimeScalarEvaluatorFactory.java
File: FnCurrentTimeScalarEvaluatorFactory.java
@Override protected IScalarEvaluator createEvaluator(IHyracksTaskContext ctx, IScalarEvaluator[] args) throws AlgebricksException { final DynamicContext dCtx = (DynamicContext) ctx.getJobletContext().getGlobalJobData(); final XSDateTimePointable datetimep = (XSDateTimePointable) XSDateTimePointable.FACTORY.createPointable(); final CastToTimeOperation castToTime = new CastToTimeOperation(); final ArrayBackedValueStorage abvsInner = new ArrayBackedValueStorage(); final DataOutput dOutInner = abvsInner.getDataOutput(); return new AbstractTaggedValueArgumentScalarEvaluator(args) { @Override protected void evaluate(TaggedValuePointable[] args, IPointable result) throws SystemException { try { dCtx.getCurrentDateTime(datetimep); abvsInner.reset(); castToTime.convertDatetime(datetimep, dOutInner); result.set(abvsInner); } catch (IOException e) { throw new SystemException(ErrorCode.SYSE0001, e); } } }; }
12. FnCurrentDateTimeScalarEvaluatorFactory#createEvaluator()
Project: vxquery
File: FnCurrentDateTimeScalarEvaluatorFactory.java
File: FnCurrentDateTimeScalarEvaluatorFactory.java
@Override protected IScalarEvaluator createEvaluator(IHyracksTaskContext ctx, IScalarEvaluator[] args) throws AlgebricksException { final DynamicContext dCtx = (DynamicContext) ctx.getJobletContext().getGlobalJobData(); final XSDateTimePointable datetimep = (XSDateTimePointable) XSDateTimePointable.FACTORY.createPointable(); final CastToDateTimeOperation castToDateTime = new CastToDateTimeOperation(); final ArrayBackedValueStorage abvsInner = new ArrayBackedValueStorage(); final DataOutput dOutInner = abvsInner.getDataOutput(); return new AbstractTaggedValueArgumentScalarEvaluator(args) { @Override protected void evaluate(TaggedValuePointable[] args, IPointable result) throws SystemException { try { dCtx.getCurrentDateTime(datetimep); abvsInner.reset(); castToDateTime.convertDatetime(datetimep, dOutInner); result.set(abvsInner); } catch (IOException e) { throw new SystemException(ErrorCode.SYSE0001, e); } } }; }
13. FnCurrentDateScalarEvaluatorFactory#createEvaluator()
Project: vxquery
File: FnCurrentDateScalarEvaluatorFactory.java
File: FnCurrentDateScalarEvaluatorFactory.java
@Override protected IScalarEvaluator createEvaluator(IHyracksTaskContext ctx, IScalarEvaluator[] args) throws AlgebricksException { final DynamicContext dCtx = (DynamicContext) ctx.getJobletContext().getGlobalJobData(); final XSDateTimePointable datetimep = (XSDateTimePointable) XSDateTimePointable.FACTORY.createPointable(); final CastToDateOperation castToDate = new CastToDateOperation(); final ArrayBackedValueStorage abvsInner = new ArrayBackedValueStorage(); final DataOutput dOutInner = abvsInner.getDataOutput(); return new AbstractTaggedValueArgumentScalarEvaluator(args) { @Override protected void evaluate(TaggedValuePointable[] args, IPointable result) throws SystemException { try { dCtx.getCurrentDateTime(datetimep); abvsInner.reset(); castToDate.convertDatetime(datetimep, dOutInner); result.set(abvsInner); } catch (IOException e) { throw new SystemException(ErrorCode.SYSE0001, e); } } }; }
14. HttpResponseImpl#writeMessage()
Project: openejb
File: HttpResponseImpl.java
File: HttpResponseImpl.java
/** Takes care of sending the response line, headers and body * * HTTP/1.1 200 OK * Server: Netscape-Enterprise/3.6 SP3 * Date: Thu, 07 Jun 2001 17:30:42 GMT * Content-Type: text/html * Connection: close * @param output the output to send the response to * @throws java.io.IOException if an exception is thrown */ protected void writeMessage(OutputStream output) throws IOException { DataOutput out = new DataOutputStream(output); //DataOutput log = new DataOutputStream(System.out); //System.out.println("\nRESPONSE"); closeMessage(); // writeResponseLine(log); // writeHeaders(log); // writeBody(log); writeResponseLine(out); writeHeaders(out); writeBody(out); }
15. HadoopSerializationWrapperSelfTest#testIntJavaSerialization()
Project: ignite
File: HadoopSerializationWrapperSelfTest.java
File: HadoopSerializationWrapperSelfTest.java
/** * Tests read/write of Integer via native JavaleSerialization. * @throws Exception If fails. */ public void testIntJavaSerialization() throws Exception { HadoopSerialization ser = new HadoopSerializationWrapper(new JavaSerialization(), Integer.class); ByteArrayOutputStream buf = new ByteArrayOutputStream(); DataOutput out = new DataOutputStream(buf); ser.write(out, 3); ser.write(out, -5); ser.close(); DataInput in = new DataInputStream(new ByteArrayInputStream(buf.toByteArray())); assertEquals(3, ((Integer) ser.read(in, null)).intValue()); assertEquals(-5, ((Integer) ser.read(in, null)).intValue()); }
16. HadoopSerializationWrapperSelfTest#testIntWritableSerialization()
Project: ignite
File: HadoopSerializationWrapperSelfTest.java
File: HadoopSerializationWrapperSelfTest.java
/** * Tests read/write of IntWritable via native WritableSerialization. * @throws Exception If fails. */ public void testIntWritableSerialization() throws Exception { HadoopSerialization ser = new HadoopSerializationWrapper(new WritableSerialization(), IntWritable.class); ByteArrayOutputStream buf = new ByteArrayOutputStream(); DataOutput out = new DataOutputStream(buf); ser.write(out, new IntWritable(3)); ser.write(out, new IntWritable(-5)); assertEquals("[0, 0, 0, 3, -1, -1, -1, -5]", Arrays.toString(buf.toByteArray())); DataInput in = new DataInputStream(new ByteArrayInputStream(buf.toByteArray())); assertEquals(3, ((IntWritable) ser.read(in, null)).get()); assertEquals(-5, ((IntWritable) ser.read(in, null)).get()); }
17. TestScan#testAttributesSerialization()
Project: hindex
File: TestScan.java
File: TestScan.java
@Test public void testAttributesSerialization() throws IOException { Scan scan = new Scan(); scan.setAttribute("attribute1", Bytes.toBytes("value1")); scan.setAttribute("attribute2", Bytes.toBytes("value2")); scan.setAttribute("attribute3", Bytes.toBytes("value3")); ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); DataOutput out = new DataOutputStream(byteArrayOutputStream); scan.write(out); Scan scan2 = new Scan(); Assert.assertTrue(scan2.getAttributesMap().isEmpty()); scan2.readFields(new DataInputStream(new ByteArrayInputStream(byteArrayOutputStream.toByteArray()))); Assert.assertNull(scan2.getAttribute("absent")); Assert.assertTrue(Arrays.equals(Bytes.toBytes("value1"), scan2.getAttribute("attribute1"))); Assert.assertTrue(Arrays.equals(Bytes.toBytes("value2"), scan2.getAttribute("attribute2"))); Assert.assertTrue(Arrays.equals(Bytes.toBytes("value3"), scan2.getAttribute("attribute3"))); Assert.assertEquals(3, scan2.getAttributesMap().size()); }
18. TestGet#testAttributesSerialization()
Project: hindex
File: TestGet.java
File: TestGet.java
@Test public void testAttributesSerialization() throws IOException { Get get = new Get(); get.setAttribute("attribute1", Bytes.toBytes("value1")); get.setAttribute("attribute2", Bytes.toBytes("value2")); get.setAttribute("attribute3", Bytes.toBytes("value3")); ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); DataOutput out = new DataOutputStream(byteArrayOutputStream); get.write(out); Get get2 = new Get(); Assert.assertTrue(get2.getAttributesMap().isEmpty()); get2.readFields(new DataInputStream(new ByteArrayInputStream(byteArrayOutputStream.toByteArray()))); Assert.assertNull(get2.getAttribute("absent")); Assert.assertTrue(Arrays.equals(Bytes.toBytes("value1"), get2.getAttribute("attribute1"))); Assert.assertTrue(Arrays.equals(Bytes.toBytes("value2"), get2.getAttribute("attribute2"))); Assert.assertTrue(Arrays.equals(Bytes.toBytes("value3"), get2.getAttribute("attribute3"))); Assert.assertEquals(3, get2.getAttributesMap().size()); }
19. TestAttributes#testAttributesSerialization()
Project: hindex
File: TestAttributes.java
File: TestAttributes.java
@Test public void testAttributesSerialization() throws IOException { Put put = new Put(); put.setAttribute("attribute1", Bytes.toBytes("value1")); put.setAttribute("attribute2", Bytes.toBytes("value2")); put.setAttribute("attribute3", Bytes.toBytes("value3")); ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); DataOutput out = new DataOutputStream(byteArrayOutputStream); put.write(out); Put put2 = new Put(); Assert.assertTrue(put2.getAttributesMap().isEmpty()); put2.readFields(new DataInputStream(new ByteArrayInputStream(byteArrayOutputStream.toByteArray()))); Assert.assertNull(put2.getAttribute("absent")); Assert.assertTrue(Arrays.equals(Bytes.toBytes("value1"), put2.getAttribute("attribute1"))); Assert.assertTrue(Arrays.equals(Bytes.toBytes("value2"), put2.getAttribute("attribute2"))); Assert.assertTrue(Arrays.equals(Bytes.toBytes("value3"), put2.getAttribute("attribute3"))); Assert.assertEquals(3, put2.getAttributesMap().size()); }
20. TestTypedBytesWritable#testIO()
Project: hadoop-mapreduce
File: TestTypedBytesWritable.java
File: TestTypedBytesWritable.java
public void testIO() throws IOException { TypedBytesWritable tbw = new TypedBytesWritable(); tbw.setValue(12345); ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutput dout = new DataOutputStream(baos); tbw.write(dout); ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray()); DataInput din = new DataInputStream(bais); TypedBytesWritable readTbw = new TypedBytesWritable(); readTbw.readFields(din); assertEquals(tbw, readTbw); }
21. TestTypedBytesWritable#testIO()
Project: hadoop-common
File: TestTypedBytesWritable.java
File: TestTypedBytesWritable.java
public void testIO() throws IOException { TypedBytesWritable tbw = new TypedBytesWritable(); tbw.setValue(12345); ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutput dout = new DataOutputStream(baos); tbw.write(dout); ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray()); DataInput din = new DataInputStream(bais); TypedBytesWritable readTbw = new TypedBytesWritable(); readTbw.readFields(din); assertEquals(tbw, readTbw); }
22. TestWritableUtils#testWritesReads()
Project: gora
File: TestWritableUtils.java
File: TestWritableUtils.java
@Test public void testWritesReads() throws Exception { Properties props = new Properties(); props.put("keyBlah", "valueBlah"); props.put("keyBlah2", "valueBlah2"); ByteArrayOutputStream bos = new ByteArrayOutputStream(); DataOutput out = new DataOutputStream(bos); WritableUtils.writeProperties(out, props); ((DataOutputStream) out).flush(); DataInput in = new DataInputStream(new ByteArrayInputStream(bos.toByteArray())); Properties propsRead = WritableUtils.readProperties(in); assertEquals(propsRead.get("keyBlah"), props.get("keyBlah")); assertEquals(propsRead.get("keyBlah2"), props.get("keyBlah2")); }
23. WritableUtils#writeListToByteArray()
Project: giraph
File: WritableUtils.java
File: WritableUtils.java
/** * Write list of object to a byte array. * * @param writableList List of object to write from. * @return Byte array with serialized objects. */ public static byte[] writeListToByteArray(List<? extends Writable> writableList) { ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); DataOutput output = new DataOutputStream(outputStream); try { output.writeInt(writableList.size()); for (Writable writable : writableList) { writable.write(output); } } catch (IOException e) { throw new IllegalStateException("writeListToByteArray: IOException", e); } return outputStream.toByteArray(); }
24. WritableUtils#writeToByteArray()
Project: giraph
File: WritableUtils.java
File: WritableUtils.java
/** * Write object to a byte array. * * @param writableObject Object to write from. * @return Byte array with serialized object. */ public static byte[] writeToByteArray(Writable writableObject) { ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); DataOutput output = new DataOutputStream(outputStream); try { writableObject.write(output); } catch (IOException e) { throw new IllegalStateException("writeToByteArray: IOStateException", e); } return outputStream.toByteArray(); }
25. BatchingDecorator#batchevent()
Project: flume
File: BatchingDecorator.java
File: BatchingDecorator.java
Event batchevent(List<Event> evts) throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(2 >> 15); DataOutput out = new DataOutputStream(baos); for (Event evt : events) { WriteableEvent we = new WriteableEvent(evt); we.write(out); } Event be = new EventImpl(new byte[0]); ByteBuffer b = ByteBuffer.allocate(4); b.putInt(events.size()); be.set(BATCH_SIZE, b.array()); be.set(BATCH_DATA, baos.toByteArray()); return be; }
26. FileMetadataSerializerTest#writesProperties()
Project: che
File: FileMetadataSerializerTest.java
File: FileMetadataSerializerTest.java
@Test public void writesProperties() throws Exception { DataOutput output = mock(DataOutput.class); Map<String, String> properties = ImmutableMap.of("a", "x", "b", "z"); metadataSerializer.write(output, properties); InOrder inOrder = inOrder(output); inOrder.verify(output).writeInt(properties.size()); inOrder.verify(output).writeUTF("a"); inOrder.verify(output).writeInt(1); inOrder.verify(output).writeUTF("b"); inOrder.verify(output).writeInt(1); inOrder.verify(output).writeUTF("z"); }
27. FileLockSerializerTest#writesLockObject()
Project: che
File: FileLockSerializerTest.java
File: FileLockSerializerTest.java
@Test public void writesLockObject() throws Exception { String token = Long.toString(System.currentTimeMillis()); long expired = System.currentTimeMillis() + 10000; FileLock lock = new FileLock(token, expired); DataOutput data = mock(DataOutput.class); lockSerializer.write(data, lock); InOrder inOrder = inOrder(data); inOrder.verify(data).writeUTF(token); inOrder.verify(data).writeLong(expired); }
28. StreamConsumerStateStore#save()
Project: cdap
File: StreamConsumerStateStore.java
File: StreamConsumerStateStore.java
@Override public final void save(Iterable<? extends StreamConsumerState> states) throws IOException { ImmutableSortedMap.Builder<byte[], byte[]> values = ImmutableSortedMap.orderedBy(Bytes.BYTES_COMPARATOR); ByteArrayOutputStream os = new ByteArrayOutputStream(); DataOutput output = new DataOutputStream(os); for (StreamConsumerState state : states) { os.reset(); encodeOffsets(state.getState(), output); values.put(getColumn(state.getGroupId(), state.getInstanceId()), os.toByteArray()); } store(streamId.toBytes(), values.build()); }
29. SparkBatchSinkFactory#serialize()
Project: cdap
File: SparkBatchSinkFactory.java
File: SparkBatchSinkFactory.java
void serialize(OutputStream outputStream) throws IOException { DataOutput output = new DataOutputStream(outputStream); Serializations.serializeMap(outputFormatProviders, new Serializations.ObjectWriter<OutputFormatProvider>() { @Override public void write(OutputFormatProvider outputFormatProvider, DataOutput output) throws IOException { output.writeUTF(outputFormatProvider.getOutputFormatClassName()); Serializations.serializeMap(outputFormatProvider.getOutputFormatConfiguration(), Serializations.createStringObjectWriter(), output); } }, output); Serializations.serializeMap(datasetInfos, new Serializations.ObjectWriter<DatasetInfo>() { @Override public void write(DatasetInfo datasetInfo, DataOutput output) throws IOException { datasetInfo.serialize(output); } }, output); Serializations.serializeMap(sinkOutputs, Serializations.createStringSetObjectWriter(), output); }
30. BufferedDataOutputStreamTest#testWriteUTFBigChar()
Project: cassandra
File: BufferedDataOutputStreamTest.java
File: BufferedDataOutputStreamTest.java
@Test public void testWriteUTFBigChar() throws Exception { ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutput dataOut = new DataOutputStream(baos); StringBuilder sb = new StringBuilder(65535); for (int ii = 0; ii < 1 << 15; ii++) { String s = sb.toString(); UnbufferedDataOutputStreamPlus.writeUTF(s, dataOut); DataInput dataIn = new DataInputStream(new ByteArrayInputStream(baos.toByteArray())); assertEquals(s, dataIn.readUTF()); baos.reset(); if (ii == (1 << 15) - 1) sb.append("a"); else sb.append(twoByte); } }
31. BufferedDataOutputStreamTest#testWriteUTF()
Project: cassandra
File: BufferedDataOutputStreamTest.java
File: BufferedDataOutputStreamTest.java
@Test public void testWriteUTF() throws Exception { ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutput dataOut = new DataOutputStream(baos); StringBuilder sb = new StringBuilder(65535); for (int ii = 0; ii < 1 << 16; ii++) { String s = sb.toString(); UnbufferedDataOutputStreamPlus.writeUTF(s, dataOut); DataInput dataIn = new DataInputStream(new ByteArrayInputStream(baos.toByteArray())); assertEquals(s, dataIn.readUTF()); baos.reset(); sb.append("a"); } }
32. FixedFileTrailer#serialize()
Project: hindex
File: FixedFileTrailer.java
File: FixedFileTrailer.java
/** * Write the trailer to a data stream. We support writing version 1 for * testing and for determining version 1 trailer size. It is also easy to see * what fields changed in version 2. * * @param outputStream * @throws IOException */ void serialize(DataOutputStream outputStream) throws IOException { HFile.checkFormatVersion(majorVersion); ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutput baosDos = new DataOutputStream(baos); BlockType.TRAILER.write(baosDos); baosDos.writeLong(fileInfoOffset); baosDos.writeLong(loadOnOpenDataOffset); baosDos.writeInt(dataIndexCount); if (majorVersion == 1) { // This used to be metaIndexOffset, but it was not used in version 1. baosDos.writeLong(0); } else { baosDos.writeLong(uncompressedDataIndexSize); } baosDos.writeInt(metaIndexCount); baosDos.writeLong(totalUncompressedBytes); if (majorVersion == 1) { baosDos.writeInt((int) Math.min(Integer.MAX_VALUE, entryCount)); } else { // This field is long from version 2 onwards. baosDos.writeLong(entryCount); } baosDos.writeInt(compressionCodec.ordinal()); if (majorVersion > 1) { baosDos.writeInt(numDataIndexLevels); baosDos.writeLong(firstDataBlockOffset); baosDos.writeLong(lastDataBlockOffset); Bytes.writeStringFixedSize(baosDos, comparatorClassName, MAX_COMPARATOR_NAME_LENGTH); } // serialize the major and minor versions baosDos.writeInt(materializeVersion(majorVersion, minorVersion)); outputStream.write(baos.toByteArray()); }
33. DBNParameterVector#Serialize()
Project: Metronome
File: DBNParameterVector.java
File: DBNParameterVector.java
public byte[] Serialize() throws IOException { // DataOutput d ByteArrayOutputStream out = new ByteArrayOutputStream(); DataOutput d = new DataOutputStream(out); // write the DBN data right into // this.dbn.write(out); //return this.dbn_payload; d.writeInt(this.iteration); d.writeBoolean(this.datasetPassComplete); d.writeBoolean(this.preTrainPhaseComplete); d.writeBoolean(this.masterSignalToStartFineTunePhase); d.writeBoolean(this.masterSignalToStartNextDatasetPass); d.writeInt(this.dbn_payload.length); out.write(this.dbn_payload); return out.toByteArray(); }
34. NeuralNetworkWeightsDelta#Serialize()
Project: Metronome
File: NeuralNetworkWeightsDelta.java
File: NeuralNetworkWeightsDelta.java
public byte[] Serialize() throws IOException { // DataOutput d ByteArrayOutputStream out = new ByteArrayOutputStream(); DataOutput d = new DataOutputStream(out); // d.writeUTF(src_host); //d.writeInt(this.SrcWorkerPassCount); d.writeInt(this.GlobalPassCount); d.writeInt(this.IterationComplete); d.writeInt(this.CurrentIteration); d.writeInt(this.TrainedRecords); //d.writeFloat(this.AvgLogLikelihood); d.writeFloat(this.PercentCorrect); d.writeDouble(this.RMSE); //d.write // buf.write // MatrixWritable.writeMatrix(d, this.worker_gradient.getMatrix()); //MatrixWritable.writeMatrix(d, this.parameter_vector); // MatrixWritable. ObjectOutputStream oos = new ObjectOutputStream(out); //System.out.println("Worker:Serialize() > " + this.network.getClass()); oos.writeObject(this.network); oos.flush(); oos.close(); return out.toByteArray(); }
35. RubixFile#extract()
Project: Cubert
File: RubixFile.java
File: RubixFile.java
private static void extract(List<RubixFile<Tuple, Object>> rfiles, long blockId, int numBlocks, String output) throws IOException, ClassNotFoundException, InstantiationException, IllegalAccessException { Configuration conf = new JobConf(); File outFile = new File(output); if (outFile.exists()) { outFile.delete(); } outFile.createNewFile(); BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(outFile)); ByteArrayOutputStream keySectionStream = new ByteArrayOutputStream(); DataOutput keySectionOut = new DataOutputStream(keySectionStream); SerializationFactory serializationFactory = new SerializationFactory(conf); RubixFile<Tuple, Object> lastrFile = null; JsonNode json; long totalLength = 0; final int BUF_SIZE = 32 * 1024; long blockIds[] = new long[numBlocks]; int foundBlocks = 0; for (int i = 0; i < numBlocks; i++) blockIds[i] = blockId + i; for (int i = 0; i < numBlocks; i++) { boolean found = false; for (RubixFile<Tuple, Object> rfile : rfiles) { print.f("Checking %s", rfile.path.toString()); List<KeyData<Tuple>> keyDataList = rfile.getKeyData(); for (KeyData<Tuple> keyData : keyDataList) { if (keyData.getBlockId() == blockIds[i]) { long offset = keyData.getOffset(); long length = keyData.getLength(); Tuple key = keyData.getKey(); print.f("Extracting block %d (off=%d len=%d) from %s", keyData.getBlockId(), offset, length, rfile.path.toString()); // copy the data if (length > 0) { FileSystem fs = FileSystem.get(conf); FSDataInputStream in = fs.open(rfile.path); in.seek(offset); byte[] data = new byte[BUF_SIZE]; long toRead = length; while (toRead > 0) { int thisRead = toRead > BUF_SIZE ? BUF_SIZE : (int) toRead; in.readFully(data, 0, thisRead); bos.write(data, 0, thisRead); toRead -= thisRead; System.out.print("."); } System.out.println(); } // copy the key section Serializer<Tuple> keySerializer = serializationFactory.getSerializer(rfile.getKeyClass()); keySerializer.open(keySectionStream); keySerializer.serialize(key); // position keySectionOut.writeLong(totalLength); keySectionOut.writeLong(keyData.getBlockId()); keySectionOut.writeLong(keyData.getNumRecords()); foundBlocks++; totalLength += length; lastrFile = rfile; found = true; break; } } if (found) { break; } } if (!found) System.err.println("Cannot locate block with id " + blockIds[i]); } byte[] trailerBytes = keySectionStream.toByteArray(); json = JsonUtils.cloneNode(lastrFile.metadataJson); ((ObjectNode) json).put("numberOfBlocks", foundBlocks); DataOutput out = new DataOutputStream(bos); out.writeUTF(json.toString()); out.writeInt(trailerBytes.length); out.write(trailerBytes); // trailer start offset out.writeLong(totalLength); bos.close(); }
36. AbstractValueComparisonScalarEvaluatorFactory#createEvaluator()
Project: vxquery
File: AbstractValueComparisonScalarEvaluatorFactory.java
File: AbstractValueComparisonScalarEvaluatorFactory.java
@Override protected IScalarEvaluator createEvaluator(IHyracksTaskContext ctx, IScalarEvaluator[] args) throws AlgebricksException { final ArrayBackedValueStorage abvs = new ArrayBackedValueStorage(); final DataOutput dOut = abvs.getDataOutput(); final AbstractValueComparisonOperation aOp = createValueComparisonOperation(); final DynamicContext dCtx = (DynamicContext) ctx.getJobletContext().getGlobalJobData(); final SequencePointable seqp = (SequencePointable) SequencePointable.FACTORY.createPointable(); final ArrayBackedValueStorage abvsInteger1 = new ArrayBackedValueStorage(); final DataOutput dOutInteger1 = abvsInteger1.getDataOutput(); final ArrayBackedValueStorage abvsInteger2 = new ArrayBackedValueStorage(); final DataOutput dOutInteger2 = abvsInteger2.getDataOutput(); final TaggedValuePointable tvp1new = (TaggedValuePointable) TaggedValuePointable.FACTORY.createPointable(); final TaggedValuePointable tvp2new = (TaggedValuePointable) TaggedValuePointable.FACTORY.createPointable(); final TypedPointables tp1 = new TypedPointables(); final TypedPointables tp2 = new TypedPointables(); return new AbstractTaggedValueArgumentScalarEvaluator(args) { @Override protected void evaluate(TaggedValuePointable[] args, IPointable result) throws SystemException { TaggedValuePointable tvp1 = args[0]; TaggedValuePointable tvp2 = args[1]; if (tvp1.getTag() == ValueTag.SEQUENCE_TAG) { tvp1.getValue(seqp); if (seqp.getEntryCount() == 0) { XDMConstants.setEmptySequence(result); return; } throw new SystemException(ErrorCode.XPTY0004); } if (tvp2.getTag() == ValueTag.SEQUENCE_TAG) { tvp2.getValue(seqp); if (seqp.getEntryCount() == 0) { XDMConstants.setEmptySequence(result); return; } throw new SystemException(ErrorCode.XPTY0004); } boolean booleanResult = transformThenCompareTaggedValues(aOp, tvp1, tvp2, dCtx); try { abvs.reset(); dOut.write(ValueTag.XS_BOOLEAN_TAG); dOut.write(booleanResult ? 1 : 0); result.set(abvs); } catch (Exception e) { throw new SystemException(ErrorCode.SYSE0001, e); } } protected boolean transformThenCompareTaggedValues(AbstractValueComparisonOperation aOp, TaggedValuePointable tvp1, TaggedValuePointable tvp2, DynamicContext dCtx) throws SystemException { try { if (tvp1.getTag() == ValueTag.XS_UNTYPED_ATOMIC_TAG) { // Only need to change tag since the storage is the same for untyped atomic and string. tvp1.getByteArray()[0] = ValueTag.XS_STRING_TAG; tvp1new.set(tvp1); } else if (FunctionHelper.isDerivedFromInteger(tvp1.getTag())) { abvsInteger1.reset(); FunctionHelper.getIntegerPointable(tvp1, dOutInteger1, tp1); tvp1new.set(abvsInteger1.getByteArray(), abvsInteger1.getStartOffset(), LongPointable.TYPE_TRAITS.getFixedLength() + 1); } else { tvp1new.set(tvp1); } if (tvp2.getTag() == ValueTag.XS_UNTYPED_ATOMIC_TAG) { // Only need to change tag since the storage is the same for untyped atomic and string. tvp2.getByteArray()[0] = ValueTag.XS_STRING_TAG; tvp2new.set(tvp2); } else if (FunctionHelper.isDerivedFromInteger(tvp2.getTag())) { abvsInteger2.reset(); FunctionHelper.getIntegerPointable(tvp2, dOutInteger2, tp2); tvp2new.set(abvsInteger2.getByteArray(), abvsInteger2.getStartOffset(), LongPointable.TYPE_TRAITS.getFixedLength() + 1); } else { tvp2new.set(tvp2); } return FunctionHelper.compareTaggedValues(aOp, tvp1new, tvp2new, dCtx, tp1, tp2); } catch (SystemException se) { throw se; } catch (Exception e) { throw new SystemException(ErrorCode.SYSE0001, e); } } }; }
37. FunctionHelper#transformThenCompareMinMaxTaggedValues()
Project: vxquery
File: FunctionHelper.java
File: FunctionHelper.java
public static boolean transformThenCompareMinMaxTaggedValues(AbstractValueComparisonOperation aOp, TaggedValuePointable tvp1, TaggedValuePointable tvp2, DynamicContext dCtx, TypedPointables tp1, TypedPointables tp2) throws SystemException { TaggedValuePointable tvp1new = (TaggedValuePointable) TaggedValuePointable.FACTORY.createPointable(); TaggedValuePointable tvp2new = (TaggedValuePointable) TaggedValuePointable.FACTORY.createPointable(); ArrayBackedValueStorage abvsArgument1 = new ArrayBackedValueStorage(); DataOutput dOutArgument1 = abvsArgument1.getDataOutput(); ArrayBackedValueStorage abvsArgument2 = new ArrayBackedValueStorage(); DataOutput dOutArgument2 = abvsArgument2.getDataOutput(); CastToDoubleOperation castToDouble = new CastToDoubleOperation(); try { abvsArgument1.reset(); if (tvp1.getTag() == ValueTag.XS_UNTYPED_ATOMIC_TAG) { tvp1.getValue(tp1.utf8sp); castToDouble.convertUntypedAtomic(tp1.utf8sp, dOutArgument1); tvp1new.set(abvsArgument1.getByteArray(), abvsArgument1.getStartOffset(), DoublePointable.TYPE_TRAITS.getFixedLength() + 1); } else if (isDerivedFromInteger(tvp1.getTag())) { getIntegerPointable(tvp1, dOutArgument1, tp1); tvp1new.set(abvsArgument1.getByteArray(), abvsArgument1.getStartOffset(), LongPointable.TYPE_TRAITS.getFixedLength() + 1); } else { tvp1new = tvp1; } abvsArgument2.reset(); if (tvp2.getTag() == ValueTag.XS_UNTYPED_ATOMIC_TAG) { tvp2.getValue(tp2.utf8sp); castToDouble.convertUntypedAtomic(tp2.utf8sp, dOutArgument2); tvp2new.set(abvsArgument2.getByteArray(), abvsArgument2.getStartOffset(), DoublePointable.TYPE_TRAITS.getFixedLength() + 1); } else if (isDerivedFromInteger(tvp2.getTag())) { getIntegerPointable(tvp2, dOutArgument2, tp1); tvp2new.set(abvsArgument2.getByteArray(), abvsArgument2.getStartOffset(), LongPointable.TYPE_TRAITS.getFixedLength() + 1); } else { tvp2new = tvp2; } return compareTaggedValues(aOp, tvp1new, tvp2new, dCtx, tp1, tp2); } catch (SystemException se) { throw se; } catch (Exception e) { throw new SystemException(ErrorCode.SYSE0001, e); } }
38. FnQNameScalarEvaluatorFactory#createEvaluator()
Project: vxquery
File: FnQNameScalarEvaluatorFactory.java
File: FnQNameScalarEvaluatorFactory.java
@Override protected IScalarEvaluator createEvaluator(IHyracksTaskContext ctx, IScalarEvaluator[] args) throws AlgebricksException { final UTF8StringPointable paramURI = (UTF8StringPointable) UTF8StringPointable.FACTORY.createPointable(); final UTF8StringPointable paramQName = (UTF8StringPointable) UTF8StringPointable.FACTORY.createPointable(); final ArrayBackedValueStorage abvs = new ArrayBackedValueStorage(); final DataOutput dOut = abvs.getDataOutput(); final ArrayBackedValueStorage abvsParamQName = new ArrayBackedValueStorage(); final DataOutput dOutParamQName = abvsParamQName.getDataOutput(); final SequencePointable seqp = (SequencePointable) SequencePointable.FACTORY.createPointable(); final TaggedValuePointable tvp = (TaggedValuePointable) TaggedValuePointable.FACTORY.createPointable(); return new AbstractTaggedValueArgumentScalarEvaluator(args) { @Override protected void evaluate(TaggedValuePointable[] args, IPointable result) throws SystemException { TaggedValuePointable tvp1 = args[0]; TaggedValuePointable tvp2 = args[1]; // Only accept a strings. if (args.length == 2) { if (tvp1.getTag() == ValueTag.SEQUENCE_TAG) { tvp1.getValue(seqp); if (seqp.getEntryCount() == 0) { XDMConstants.setEmptyString(tvp); tvp.getValue(paramURI); } else { throw new SystemException(ErrorCode.FORG0006); } } else { if (!FunctionHelper.isDerivedFromString(tvp1.getTag())) { throw new SystemException(ErrorCode.FORG0006); } tvp1.getValue(paramURI); } if (tvp2.getTag() != ValueTag.XS_STRING_TAG) { throw new SystemException(ErrorCode.FORG0006); } tvp2.getValue(paramQName); } else if (args.length == 1) { if (tvp1.getTag() != ValueTag.XS_STRING_TAG) { throw new SystemException(ErrorCode.FORG0006); } XDMConstants.setEmptyString(tvp); tvp.getValue(paramURI); tvp2.getValue(paramQName); } else { throw new SystemException(ErrorCode.FORG0006); } try { abvs.reset(); dOut.write(ValueTag.XS_QNAME_TAG); dOut.write(paramURI.getByteArray(), paramURI.getStartOffset(), paramURI.getLength()); // Separate the local name and prefix. abvsParamQName.reset(); ICharacterIterator charIterator = new UTF8StringCharacterIterator(paramQName); charIterator.reset(); int c = 0; int prefixLength = 0; while ((c = charIterator.next()) != ICharacterIterator.EOS_CHAR) { if (c == Character.valueOf(':')) { prefixLength = abvsParamQName.getLength(); } else { FunctionHelper.writeChar((char) c, dOutParamQName); } } dOut.write((byte) ((prefixLength >>> 8) & 0xFF)); dOut.write((byte) ((prefixLength >>> 0) & 0xFF)); dOut.write(abvsParamQName.getByteArray(), abvsParamQName.getStartOffset(), prefixLength); int localNameLength = abvsParamQName.getLength() - prefixLength; dOut.write((byte) ((localNameLength >>> 8) & 0xFF)); dOut.write((byte) ((localNameLength >>> 0) & 0xFF)); dOut.write(abvsParamQName.getByteArray(), abvsParamQName.getStartOffset() + prefixLength, localNameLength); result.set(abvs); } catch (Exception e) { throw new SystemException(ErrorCode.SYSE0001, e); } } }; }
39. FnAdjustTimeToTimezoneScalarEvaluatorFactory#createEvaluator()
Project: vxquery
File: FnAdjustTimeToTimezoneScalarEvaluatorFactory.java
File: FnAdjustTimeToTimezoneScalarEvaluatorFactory.java
@Override protected IScalarEvaluator createEvaluator(IHyracksTaskContext ctx, IScalarEvaluator[] args) throws AlgebricksException { final DynamicContext dCtx = (DynamicContext) ctx.getJobletContext().getGlobalJobData(); final XSTimePointable timep = (XSTimePointable) XSTimePointable.FACTORY.createPointable(); final XSDateTimePointable datetimep = (XSDateTimePointable) XSDateTimePointable.FACTORY.createPointable(); final XSDateTimePointable ctxDatetimep = (XSDateTimePointable) XSDateTimePointable.FACTORY.createPointable(); final LongPointable longp = (LongPointable) LongPointable.FACTORY.createPointable(); final ArrayBackedValueStorage abvsInner = new ArrayBackedValueStorage(); final DataOutput dOutInner = abvsInner.getDataOutput(); final ArrayBackedValueStorage abvs = new ArrayBackedValueStorage(); final DataOutput dOut = abvs.getDataOutput(); return new AbstractTaggedValueArgumentScalarEvaluator(args) { @Override protected void evaluate(TaggedValuePointable[] args, IPointable result) throws SystemException { dCtx.getCurrentDateTime(ctxDatetimep); TaggedValuePointable tvp1 = args[0]; if (tvp1.getTag() != ValueTag.XS_TIME_TAG) { throw new SystemException(ErrorCode.FORG0006); } tvp1.getValue(timep); // Second argument is optional and will used the dynamic context if not supplied. long tz; if (args.length == 2) { TaggedValuePointable tvp2 = args[1]; if (tvp2.getTag() == ValueTag.XS_DAY_TIME_DURATION_TAG) { tvp2.getValue(longp); if (Math.abs(longp.getLong()) > DateTime.CHRONON_OF_HOUR * 14) { throw new SystemException(ErrorCode.FODT0003); } tz = longp.getLong() / DateTime.CHRONON_OF_MINUTE; } else { throw new SystemException(ErrorCode.FORG0006); } } else { tz = ctxDatetimep.getTimezoneHour() * 60 + ctxDatetimep.getTimezoneMinute(); } try { abvs.reset(); abvsInner.reset(); // Convert to DateTime to help have a good date. datetimep.set(abvsInner.getByteArray(), abvsInner.getStartOffset(), XSDateTimePointable.TYPE_TRAITS.getFixedLength()); datetimep.setDateTime(1970, 1, 1, timep.getHour(), timep.getMinute(), timep.getMilliSecond(), timep.getTimezoneHour(), timep.getTimezoneMinute()); DateTime.adjustDateTimeToTimezone(datetimep, tz, dOutInner); byte[] bytes = abvsInner.getByteArray(); int startOffset = abvsInner.getStartOffset() + 1; // Convert to time. dOut.write(ValueTag.XS_TIME_TAG); dOut.write(bytes, startOffset + XSDateTimePointable.HOUR_OFFSET, XSTimePointable.TYPE_TRAITS.getFixedLength()); result.set(abvs); } catch (IOException e) { throw new SystemException(ErrorCode.SYSE0001, e); } } }; }
40. FnAdjustDateToTimezoneScalarEvaluatorFactory#createEvaluator()
Project: vxquery
File: FnAdjustDateToTimezoneScalarEvaluatorFactory.java
File: FnAdjustDateToTimezoneScalarEvaluatorFactory.java
@Override protected IScalarEvaluator createEvaluator(IHyracksTaskContext ctx, IScalarEvaluator[] args) throws AlgebricksException { final DynamicContext dCtx = (DynamicContext) ctx.getJobletContext().getGlobalJobData(); final XSDatePointable datep = (XSDatePointable) XSDatePointable.FACTORY.createPointable(); final XSDateTimePointable ctxDatetimep = (XSDateTimePointable) XSDateTimePointable.FACTORY.createPointable(); final LongPointable longp = (LongPointable) LongPointable.FACTORY.createPointable(); final ArrayBackedValueStorage abvsInner = new ArrayBackedValueStorage(); final DataOutput dOutInner = abvsInner.getDataOutput(); final ArrayBackedValueStorage abvs = new ArrayBackedValueStorage(); final DataOutput dOut = abvs.getDataOutput(); return new AbstractTaggedValueArgumentScalarEvaluator(args) { @Override protected void evaluate(TaggedValuePointable[] args, IPointable result) throws SystemException { dCtx.getCurrentDateTime(ctxDatetimep); TaggedValuePointable tvp1 = args[0]; if (tvp1.getTag() != ValueTag.XS_DATE_TAG) { throw new SystemException(ErrorCode.FORG0006); } tvp1.getValue(datep); // Second argument is optional and will used the dynamic context if not supplied. long tz; if (args.length == 2) { TaggedValuePointable tvp2 = args[1]; if (tvp2.getTag() == ValueTag.XS_DAY_TIME_DURATION_TAG) { tvp2.getValue(longp); if (Math.abs(longp.getLong()) > DateTime.CHRONON_OF_HOUR * 14) { throw new SystemException(ErrorCode.FODT0003); } tz = longp.getLong() / DateTime.CHRONON_OF_MINUTE; } else { throw new SystemException(ErrorCode.FORG0006); } } else { tz = ctxDatetimep.getTimezoneHour() * 60 + ctxDatetimep.getTimezoneMinute(); } try { abvs.reset(); abvsInner.reset(); DateTime.adjustDateTimeToTimezone(datep, tz, dOutInner); byte[] bytes = abvsInner.getByteArray(); int startOffset = abvsInner.getStartOffset() + 1; // Convert to date. bytes[startOffset + XSDatePointable.TIMEZONE_HOUR_OFFSET] = bytes[startOffset + XSDateTimePointable.TIMEZONE_HOUR_OFFSET]; bytes[startOffset + XSDatePointable.TIMEZONE_MINUTE_OFFSET] = bytes[startOffset + XSDateTimePointable.TIMEZONE_MINUTE_OFFSET]; dOut.write(ValueTag.XS_DATE_TAG); dOut.write(bytes, startOffset, XSDatePointable.TYPE_TRAITS.getFixedLength()); result.set(abvs); } catch (IOException e) { throw new SystemException(ErrorCode.SYSE0001, e); } } }; }
41. FnAvgAggregateEvaluatorFactory#createEvaluator()
Project: vxquery
File: FnAvgAggregateEvaluatorFactory.java
File: FnAvgAggregateEvaluatorFactory.java
@Override protected IAggregateEvaluator createEvaluator(IScalarEvaluator[] args) throws AlgebricksException { final TaggedValuePointable tvpCount = (TaggedValuePointable) TaggedValuePointable.FACTORY.createPointable(); final ArrayBackedValueStorage abvsSum = new ArrayBackedValueStorage(); final DataOutput dOutSum = abvsSum.getDataOutput(); final ArrayBackedValueStorage abvsCount = new ArrayBackedValueStorage(); final DataOutput dOutCount = abvsCount.getDataOutput(); final AddOperation aOpAdd = new AddOperation(); final ArithmeticHelper add = new ArithmeticHelper(aOpAdd, dCtx); final DivideOperation aOpDivide = new DivideOperation(); final ArithmeticHelper divide = new ArithmeticHelper(aOpDivide, dCtx); return new AbstractTaggedValueArgumentAggregateEvaluator(args) { long count; TaggedValuePointable tvpSum = (TaggedValuePointable) TaggedValuePointable.FACTORY.createPointable(); @Override public void init() throws AlgebricksException { count = 0; } @Override public void finishPartial(IPointable result) throws AlgebricksException { finish(result); } @Override public void finish(IPointable result) throws AlgebricksException { if (count == 0) { XDMConstants.setEmptySequence(result); } else { // Set count as a TaggedValuePointable. try { abvsCount.reset(); dOutCount.write(ValueTag.XS_INTEGER_TAG); dOutCount.writeLong(count); tvpCount.set(abvsCount); divide.compute(tvpSum, tvpCount, tvpSum); result.set(tvpSum); } catch (Exception e) { throw new AlgebricksException(e); } } } @Override protected void step(TaggedValuePointable[] args) throws SystemException { TaggedValuePointable tvp = args[0]; if (count == 0) { // Init. try { abvsSum.reset(); dOutSum.write(tvp.getByteArray(), tvp.getStartOffset(), tvp.getLength()); tvpSum.set(abvsSum); } catch (IOException e) { throw new SystemException(ErrorCode.SYSE0001, e.toString()); } } else { add.compute(tvp, tvpSum, tvpSum); } count++; } }; }
42. AvgLocalAggregateEvaluatorFactory#createEvaluator()
Project: vxquery
File: AvgLocalAggregateEvaluatorFactory.java
File: AvgLocalAggregateEvaluatorFactory.java
@Override protected IAggregateEvaluator createEvaluator(IScalarEvaluator[] args) throws AlgebricksException { final TaggedValuePointable tvpCount = (TaggedValuePointable) TaggedValuePointable.FACTORY.createPointable(); final ArrayBackedValueStorage abvsCount = new ArrayBackedValueStorage(); final DataOutput dOutCount = abvsCount.getDataOutput(); final ArrayBackedValueStorage abvsSum = new ArrayBackedValueStorage(); final DataOutput dOutSum = abvsSum.getDataOutput(); final ArrayBackedValueStorage abvsSeq = new ArrayBackedValueStorage(); final SequenceBuilder sb = new SequenceBuilder(); final AddOperation aOpAdd = new AddOperation(); final ArithmeticHelper add = new ArithmeticHelper(aOpAdd, dCtx); return new AbstractTaggedValueArgumentAggregateEvaluator(args) { long count; TaggedValuePointable tvpSum = (TaggedValuePointable) TaggedValuePointable.FACTORY.createPointable(); @Override public void init() throws AlgebricksException { count = 0; try { abvsSum.reset(); dOutSum.write(ValueTag.XS_INTEGER_TAG); dOutSum.writeLong(0); tvpSum.set(abvsSum); } catch (Exception e) { throw new AlgebricksException(e); } } @Override public void finishPartial(IPointable result) throws AlgebricksException { finish(result); } @Override public void finish(IPointable result) throws AlgebricksException { if (count == 0) { XDMConstants.setEmptySequence(result); } else { // Set count as a TaggedValuePointable. try { abvsCount.reset(); dOutCount.write(ValueTag.XS_INTEGER_TAG); dOutCount.writeLong(count); tvpCount.set(abvsCount); // Save intermediate result. abvsSeq.reset(); sb.reset(abvsSeq); sb.addItem(tvpCount); sb.addItem(tvpSum); sb.finish(); result.set(abvsSeq); } catch (Exception e) { throw new AlgebricksException(e); } } } @Override protected void step(TaggedValuePointable[] args) throws SystemException { TaggedValuePointable tvp = args[0]; add.compute(tvp, tvpSum, tvpSum); count++; } }; }
43. AvgGlobalAggregateEvaluatorFactory#createEvaluator()
Project: vxquery
File: AvgGlobalAggregateEvaluatorFactory.java
File: AvgGlobalAggregateEvaluatorFactory.java
@Override protected IAggregateEvaluator createEvaluator(IScalarEvaluator[] args) throws AlgebricksException { final ArrayBackedValueStorage abvsCount = new ArrayBackedValueStorage(); final DataOutput dOutCount = abvsCount.getDataOutput(); final ArrayBackedValueStorage abvsSum = new ArrayBackedValueStorage(); final DataOutput dOutSum = abvsSum.getDataOutput(); final AddOperation aOpAdd = new AddOperation(); final ArithmeticHelper add1 = new ArithmeticHelper(aOpAdd, dCtx); final ArithmeticHelper add2 = new ArithmeticHelper(aOpAdd, dCtx); final DivideOperation aOpDivide = new DivideOperation(); final ArithmeticHelper divide = new ArithmeticHelper(aOpDivide, dCtx); final LongPointable longp = (LongPointable) LongPointable.FACTORY.createPointable(); final SequencePointable seq = (SequencePointable) SequencePointable.FACTORY.createPointable(); final TaggedValuePointable tvpArg = (TaggedValuePointable) TaggedValuePointable.FACTORY.createPointable(); return new AbstractTaggedValueArgumentAggregateEvaluator(args) { TaggedValuePointable tvpSum = (TaggedValuePointable) TaggedValuePointable.FACTORY.createPointable(); TaggedValuePointable tvpCount = (TaggedValuePointable) TaggedValuePointable.FACTORY.createPointable(); @Override public void init() throws AlgebricksException { try { abvsCount.reset(); dOutCount.write(ValueTag.XS_INTEGER_TAG); dOutCount.writeLong(0); tvpCount.set(abvsCount); abvsSum.reset(); dOutSum.write(ValueTag.XS_INTEGER_TAG); dOutSum.writeLong(0); tvpSum.set(abvsSum); } catch (Exception e) { throw new AlgebricksException(e); } } @Override public void finishPartial(IPointable result) throws AlgebricksException { finish(result); } @Override public void finish(IPointable result) throws AlgebricksException { tvpCount.getValue(longp); if (longp.getLong() == 0) { XDMConstants.setEmptySequence(result); } else { // Set count as a TaggedValuePointable. try { divide.compute(tvpSum, tvpCount, tvpSum); result.set(tvpSum); } catch (Exception e) { throw new AlgebricksException(e); } } } @Override protected void step(TaggedValuePointable[] args) throws SystemException { TaggedValuePointable tvp = args[0]; if (tvp.getTag() == ValueTag.SEQUENCE_TAG) { tvp.getValue(seq); int seqLen = seq.getEntryCount(); if (seqLen == 0) { // No results from nodes. return; } else if (seqLen == 2) { seq.getEntry(0, tvpArg); add1.compute(tvpArg, tvpCount, tvpCount); seq.getEntry(1, tvpArg); add2.compute(tvpArg, tvpSum, tvpSum); } else { throw new SystemException(ErrorCode.SYSE0001); } } } }; }
44. DictionaryBuilder#writeFromCache()
Project: vxquery
File: DictionaryBuilder.java
File: DictionaryBuilder.java
public void writeFromCache(ArrayBackedValueStorage abvs) throws IOException { if (!cacheReady) { cache.reset(); write(cache); cacheReady = true; } DataOutput out = abvs.getDataOutput(); out.write(cache.getByteArray(), cache.getStartOffset(), cache.getLength()); }
45. TestMockDAGAppMaster#testBasicStatistics()
Project: tez
File: TestMockDAGAppMaster.java
File: TestMockDAGAppMaster.java
@Test(timeout = 10000) public void testBasicStatistics() throws Exception { TezConfiguration tezconf = new TezConfiguration(defaultConf); MockTezClient tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null, null, false, false); tezClient.start(); final String vAName = "A"; final String vBName = "B"; final String sourceName = "In"; final String sinkName = "Out"; DAG dag = DAG.create("testBasisStatistics"); Vertex vA = Vertex.create(vAName, ProcessorDescriptor.create("Proc.class"), 3); Vertex vB = Vertex.create(vBName, ProcessorDescriptor.create("Proc.class"), 2); vA.addDataSource(sourceName, DataSourceDescriptor.create(InputDescriptor.create("In"), null, null)); vB.addDataSink(sinkName, DataSinkDescriptor.create(OutputDescriptor.create("Out"), null, null)); dag.addVertex(vA).addVertex(vB).addEdge(Edge.create(vA, vB, EdgeProperty.create(DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("Out"), InputDescriptor.create("In")))); IOStatistics ioStats = new IOStatistics(); ioStats.setDataSize(1); ioStats.setItemsProcessed(1); TaskStatistics vAStats = new TaskStatistics(); vAStats.addIO(vBName, ioStats); vAStats.addIO(sourceName, ioStats); TaskStatistics vBStats = new TaskStatistics(); vBStats.addIO(vAName, ioStats); vBStats.addIO(sinkName, ioStats); ByteArrayOutputStream bosA = new ByteArrayOutputStream(); DataOutput outA = new DataOutputStream(bosA); vAStats.write(outA); final byte[] payloadA = bosA.toByteArray(); ByteArrayOutputStream bosB = new ByteArrayOutputStream(); DataOutput outB = new DataOutputStream(bosB); vBStats.write(outB); final byte[] payloadB = bosB.toByteArray(); MockDAGAppMaster mockApp = tezClient.getLocalClient().getMockApp(); MockContainerLauncher mockLauncher = mockApp.getContainerLauncher(); mockLauncher.startScheduling(false); mockApp.statsDelegate = new StatisticsDelegate() { @Override public TaskStatistics getStatistics(TaskSpec taskSpec) { byte[] payload = payloadA; TaskStatistics stats = new TaskStatistics(); if (taskSpec.getVertexName().equals(vBName)) { payload = payloadB; } final DataInputByteBuffer in = new DataInputByteBuffer(); in.reset(ByteBuffer.wrap(payload)); try { // this ensures that the serde code path is covered. stats.readFields(in); } catch (IOException e) { Assert.fail(e.getMessage()); } return stats; } }; mockApp.doSleep = false; DAGClient dagClient = tezClient.submitDAG(dag); mockLauncher.waitTillContainersLaunched(); DAGImpl dagImpl = (DAGImpl) mockApp.getContext().getCurrentDAG(); mockLauncher.startScheduling(true); DAGStatus status = dagClient.waitForCompletion(); Assert.assertEquals(DAGStatus.State.SUCCEEDED, status.getState()); // verify that the values have been correct aggregated for (org.apache.tez.dag.app.dag.Vertex v : dagImpl.getVertices().values()) { VertexStatistics vStats = v.getStatistics(); if (v.getName().equals(vAName)) { Assert.assertEquals(3, vStats.getOutputStatistics(vBName).getDataSize()); Assert.assertEquals(3, vStats.getInputStatistics(sourceName).getDataSize()); Assert.assertEquals(3, vStats.getOutputStatistics(vBName).getItemsProcessed()); Assert.assertEquals(3, vStats.getInputStatistics(sourceName).getItemsProcessed()); } else { Assert.assertEquals(2, vStats.getInputStatistics(vAName).getDataSize()); Assert.assertEquals(2, vStats.getOutputStatistics(sinkName).getDataSize()); Assert.assertEquals(2, vStats.getInputStatistics(vAName).getItemsProcessed()); Assert.assertEquals(2, vStats.getOutputStatistics(sinkName).getItemsProcessed()); } } tezClient.stop(); }
46. FnStringToCodepointsEvaluatorFactory#createEvaluator()
Project: vxquery
File: FnStringToCodepointsEvaluatorFactory.java
File: FnStringToCodepointsEvaluatorFactory.java
@Override protected IScalarEvaluator createEvaluator(IHyracksTaskContext ctx, IScalarEvaluator[] args) throws AlgebricksException { final UTF8StringPointable stringp = (UTF8StringPointable) UTF8StringPointable.FACTORY.createPointable(); final UTF8StringCharacterIterator charIterator = new UTF8StringCharacterIterator(stringp); final SequencePointable seqp = (SequencePointable) SequencePointable.FACTORY.createPointable(); final ArrayBackedValueStorage abvs = new ArrayBackedValueStorage(); final SequenceBuilder sb = new SequenceBuilder(); final ArrayBackedValueStorage abvsInner = new ArrayBackedValueStorage(); final DataOutput dOutInner = abvsInner.getDataOutput(); return new AbstractTaggedValueArgumentScalarEvaluator(args) { @Override protected void evaluate(TaggedValuePointable[] args, IPointable result) throws SystemException { TaggedValuePointable tvp1 = args[0]; // Only accept strings as input. if (tvp1.getTag() == ValueTag.SEQUENCE_TAG) { tvp1.getValue(seqp); if (seqp.getEntryCount() == 0) { XDMConstants.setEmptySequence(result); return; } else { throw new SystemException(ErrorCode.FORG0006); } } else { if (!FunctionHelper.isDerivedFromString(tvp1.getTag())) { throw new SystemException(ErrorCode.FORG0006); } tvp1.getValue(stringp); if (stringp.getLength() == 2) { XDMConstants.setEmptySequence(result); return; } } // Return sequence of character values. try { charIterator.reset(); abvs.reset(); sb.reset(abvs); int c; while ((c = charIterator.next()) != ICharacterIterator.EOS_CHAR) { abvsInner.reset(); dOutInner.write(ValueTag.XS_INTEGER_TAG); dOutInner.writeLong(c); sb.addItem(abvsInner); } sb.finish(); result.set(abvs); } catch (IOException e) { throw new SystemException(ErrorCode.SYSE0001); } } }; }
47. FnStringLengthEvaluatorFactory#createEvaluator()
Project: vxquery
File: FnStringLengthEvaluatorFactory.java
File: FnStringLengthEvaluatorFactory.java
@Override protected IScalarEvaluator createEvaluator(IHyracksTaskContext ctx, IScalarEvaluator[] args) throws AlgebricksException { final UTF8StringPointable stringp = (UTF8StringPointable) UTF8StringPointable.FACTORY.createPointable(); final ArrayBackedValueStorage abvs = new ArrayBackedValueStorage(); final DataOutput dOut = abvs.getDataOutput(); final SequencePointable seqp = (SequencePointable) SequencePointable.FACTORY.createPointable(); final TaggedValuePointable tvp = (TaggedValuePointable) TaggedValuePointable.FACTORY.createPointable(); return new AbstractTaggedValueArgumentScalarEvaluator(args) { @Override protected void evaluate(TaggedValuePointable[] args, IPointable result) throws SystemException { TaggedValuePointable tvp1 = args[0]; // Only accept strings as input. if (tvp1.getTag() == ValueTag.SEQUENCE_TAG) { tvp1.getValue(seqp); if (seqp.getEntryCount() == 0) { XDMConstants.setEmptyString(tvp); tvp.getValue(stringp); } else { throw new SystemException(ErrorCode.FORG0006); } } else { if (!FunctionHelper.isDerivedFromString(tvp1.getTag())) { throw new SystemException(ErrorCode.FORG0006); } tvp1.getValue(stringp); } // Return the string length of the UTF8 String. try { abvs.reset(); dOut.write(ValueTag.XS_INTEGER_TAG); dOut.writeLong(stringp.getStringLength()); result.set(abvs); } catch (Exception e) { throw new SystemException(ErrorCode.SYSE0001, e); } } }; }
48. OpToScalarEvaluatorFactory#createEvaluator()
Project: vxquery
File: OpToScalarEvaluatorFactory.java
File: OpToScalarEvaluatorFactory.java
@Override protected IScalarEvaluator createEvaluator(IHyracksTaskContext ctx, IScalarEvaluator[] args) throws AlgebricksException { final ArrayBackedValueStorage abvs = new ArrayBackedValueStorage(); final ArrayBackedValueStorage abvsInner = new ArrayBackedValueStorage(); final DataOutput dOutInner = abvsInner.getDataOutput(); final SequenceBuilder sb = new SequenceBuilder(); final LongPointable longp = (LongPointable) LongPointable.FACTORY.createPointable(); return new AbstractTaggedValueArgumentScalarEvaluator(args) { @Override protected void evaluate(TaggedValuePointable[] args, IPointable result) throws SystemException { try { TaggedValuePointable tvp1 = args[0]; if (tvp1.getTag() != ValueTag.XS_INTEGER_TAG) { throw new SystemException(ErrorCode.FORG0006); } tvp1.getValue(longp); long start = longp.getLong(); TaggedValuePointable tvp2 = args[1]; if (tvp2.getTag() != ValueTag.XS_INTEGER_TAG) { throw new SystemException(ErrorCode.FORG0006); } tvp2.getValue(longp); long end = longp.getLong(); abvs.reset(); sb.reset(abvs); if (start > end) { XDMConstants.setEmptySequence(result); return; } else { for (long j = start; j <= end; ++j) { abvsInner.reset(); dOutInner.write(ValueTag.XS_INTEGER_TAG); dOutInner.writeLong(j); sb.addItem(abvsInner); } } sb.finish(); result.set(abvs); } catch (IOException e) { throw new SystemException(ErrorCode.SYSE0001); } } }; }
49. FnIndexOfScalarEvaluatorFactory#createEvaluator()
Project: vxquery
File: FnIndexOfScalarEvaluatorFactory.java
File: FnIndexOfScalarEvaluatorFactory.java
@Override protected IScalarEvaluator createEvaluator(IHyracksTaskContext ctx, IScalarEvaluator[] args) throws AlgebricksException { final ArrayBackedValueStorage abvs = new ArrayBackedValueStorage(); final ArrayBackedValueStorage abvsInner = new ArrayBackedValueStorage(); final DataOutput dOutInner = abvsInner.getDataOutput(); final SequenceBuilder sb = new SequenceBuilder(); final SequencePointable seq = (SequencePointable) SequencePointable.FACTORY.createPointable(); final DynamicContext dCtx = (DynamicContext) ctx.getJobletContext().getGlobalJobData(); final AbstractValueComparisonOperation aOp = new ValueEqComparisonOperation(); final TaggedValuePointable tvp = (TaggedValuePointable) TaggedValuePointable.FACTORY.createPointable(); final VoidPointable p = (VoidPointable) VoidPointable.FACTORY.createPointable(); final TypedPointables tp1 = new TypedPointables(); final TypedPointables tp2 = new TypedPointables(); return new AbstractTaggedValueArgumentScalarEvaluator(args) { @Override protected void evaluate(TaggedValuePointable[] args, IPointable result) throws SystemException { try { abvs.reset(); sb.reset(abvs); TaggedValuePointable tvp1 = args[0]; TaggedValuePointable tvp2 = args[1]; if (tvp1.getTag() == ValueTag.SEQUENCE_TAG) { tvp1.getValue(seq); int seqLen = seq.getEntryCount(); for (int j = 0; j < seqLen; ++j) { seq.getEntry(j, p); tvp.set(p.getByteArray(), p.getStartOffset(), p.getLength()); if (FunctionHelper.compareTaggedValues(aOp, tvp, tvp2, dCtx, tp1, tp2)) { abvsInner.reset(); dOutInner.write(ValueTag.XS_INTEGER_TAG); dOutInner.writeLong(j + 1); sb.addItem(abvsInner); } } } else { if (FunctionHelper.compareTaggedValues(aOp, tvp1, tvp2, dCtx, tp1, tp2)) { abvsInner.reset(); dOutInner.write(ValueTag.XS_INTEGER_TAG); dOutInner.writeLong(1); sb.addItem(abvsInner); } } sb.finish(); result.set(abvs); } catch (IOException e) { throw new SystemException(ErrorCode.SYSE0001); } } }; }
50. FnPrefixFromQNameScalarEvaluatorFactory#createEvaluator()
Project: vxquery
File: FnPrefixFromQNameScalarEvaluatorFactory.java
File: FnPrefixFromQNameScalarEvaluatorFactory.java
@Override protected IScalarEvaluator createEvaluator(IHyracksTaskContext ctx, IScalarEvaluator[] args) throws AlgebricksException { final XSQNamePointable qnamep = (XSQNamePointable) XSQNamePointable.FACTORY.createPointable(); final SequencePointable seqp = (SequencePointable) SequencePointable.FACTORY.createPointable(); final UTF8StringPointable stringp = (UTF8StringPointable) UTF8StringPointable.FACTORY.createPointable(); final ArrayBackedValueStorage abvs = new ArrayBackedValueStorage(); final DataOutput dOut = abvs.getDataOutput(); return new AbstractTaggedValueArgumentScalarEvaluator(args) { @Override protected void evaluate(TaggedValuePointable[] args, IPointable result) throws SystemException { TaggedValuePointable tvp1 = args[0]; // Only accept a QNames or empty sequence. if (tvp1.getTag() == ValueTag.SEQUENCE_TAG) { tvp1.getValue(seqp); if (seqp.getEntryCount() == 0) { XDMConstants.setEmptySequence(result); return; } // Pass through. } if (tvp1.getTag() != ValueTag.XS_QNAME_TAG) { throw new SystemException(ErrorCode.FORG0006); } tvp1.getValue(qnamep); qnamep.getPrefix(stringp); try { // Return empty sequence if no prefix. if (qnamep.getPrefixLength() == 0) { XDMConstants.setEmptySequence(result); } else { abvs.reset(); dOut.write(ValueTag.XS_NCNAME_TAG); dOut.write(stringp.getByteArray(), stringp.getStartOffset(), stringp.getLength()); result.set(abvs); } } catch (Exception e) { throw new SystemException(ErrorCode.SYSE0001, e); } } }; }
51. FnNamespaceUriFromQNameScalarEvaluatorFactory#createEvaluator()
Project: vxquery
File: FnNamespaceUriFromQNameScalarEvaluatorFactory.java
File: FnNamespaceUriFromQNameScalarEvaluatorFactory.java
@Override protected IScalarEvaluator createEvaluator(IHyracksTaskContext ctx, IScalarEvaluator[] args) throws AlgebricksException { final XSQNamePointable qnamep = (XSQNamePointable) XSQNamePointable.FACTORY.createPointable(); final SequencePointable seqp = (SequencePointable) SequencePointable.FACTORY.createPointable(); final UTF8StringPointable stringp = (UTF8StringPointable) UTF8StringPointable.FACTORY.createPointable(); final ArrayBackedValueStorage abvs = new ArrayBackedValueStorage(); final DataOutput dOut = abvs.getDataOutput(); return new AbstractTaggedValueArgumentScalarEvaluator(args) { @Override protected void evaluate(TaggedValuePointable[] args, IPointable result) throws SystemException { TaggedValuePointable tvp1 = args[0]; // Only accept a QNames or empty sequence. if (tvp1.getTag() == ValueTag.SEQUENCE_TAG) { tvp1.getValue(seqp); if (seqp.getEntryCount() == 0) { XDMConstants.setEmptySequence(result); return; } // Pass through. } if (tvp1.getTag() != ValueTag.XS_QNAME_TAG) { throw new SystemException(ErrorCode.FORG0006); } tvp1.getValue(qnamep); qnamep.getUri(stringp); try { if (qnamep.getUriLength() == 0) { XDMConstants.setEmptyString(result); } else { abvs.reset(); dOut.write(ValueTag.XS_NCNAME_TAG); dOut.write(stringp.getByteArray(), stringp.getStartOffset(), stringp.getLength()); result.set(abvs); } } catch (Exception e) { throw new SystemException(ErrorCode.SYSE0001, e); } } }; }
52. FnLocalNameFromQNameScalarEvaluatorFactory#createEvaluator()
Project: vxquery
File: FnLocalNameFromQNameScalarEvaluatorFactory.java
File: FnLocalNameFromQNameScalarEvaluatorFactory.java
@Override protected IScalarEvaluator createEvaluator(IHyracksTaskContext ctx, IScalarEvaluator[] args) throws AlgebricksException { final XSQNamePointable qnamep = (XSQNamePointable) XSQNamePointable.FACTORY.createPointable(); final SequencePointable seqp = (SequencePointable) SequencePointable.FACTORY.createPointable(); final UTF8StringPointable stringp = (UTF8StringPointable) UTF8StringPointable.FACTORY.createPointable(); final ArrayBackedValueStorage abvs = new ArrayBackedValueStorage(); final DataOutput dOut = abvs.getDataOutput(); return new AbstractTaggedValueArgumentScalarEvaluator(args) { @Override protected void evaluate(TaggedValuePointable[] args, IPointable result) throws SystemException { TaggedValuePointable tvp1 = args[0]; // Only accept a QNames or empty sequence. if (tvp1.getTag() == ValueTag.SEQUENCE_TAG) { tvp1.getValue(seqp); if (seqp.getEntryCount() == 0) { XDMConstants.setEmptySequence(result); return; } // Pass through. } if (tvp1.getTag() != ValueTag.XS_QNAME_TAG) { throw new SystemException(ErrorCode.FORG0006); } tvp1.getValue(qnamep); qnamep.getLocalName(stringp); try { abvs.reset(); dOut.write(ValueTag.XS_NCNAME_TAG); dOut.write(stringp.getByteArray(), stringp.getStartOffset(), stringp.getLength()); result.set(abvs); } catch (Exception e) { throw new SystemException(ErrorCode.SYSE0001, e); } } }; }
53. TreeIdFromNodeScalarEvaluatorFactory#createEvaluator()
Project: vxquery
File: TreeIdFromNodeScalarEvaluatorFactory.java
File: TreeIdFromNodeScalarEvaluatorFactory.java
@Override protected IScalarEvaluator createEvaluator(IHyracksTaskContext ctx, IScalarEvaluator[] args) throws AlgebricksException { final ArrayBackedValueStorage abvs = new ArrayBackedValueStorage(); final DataOutput dOut = abvs.getDataOutput(); final NodeTreePointable ntp = (NodeTreePointable) NodeTreePointable.FACTORY.createPointable(); return new AbstractTaggedValueArgumentScalarEvaluator(args) { @Override protected void evaluate(TaggedValuePointable[] args, IPointable result) throws SystemException { TaggedValuePointable tvp1 = args[0]; // Only accept node trees as input. if (tvp1.getTag() == ValueTag.NODE_TREE_TAG) { try { abvs.reset(); tvp1.getValue(ntp); dOut.write(ValueTag.XS_INT_TAG); tvp1.getValue(ntp); dOut.writeInt(ntp.getRootNodeId()); result.set(abvs); } catch (Exception e) { throw new SystemException(ErrorCode.SYSE0001, e); } } else { XDMConstants.setEmptySequence(result); } } }; }
54. LocalIdFromNodeScalarEvaluatorFactory#createEvaluator()
Project: vxquery
File: LocalIdFromNodeScalarEvaluatorFactory.java
File: LocalIdFromNodeScalarEvaluatorFactory.java
@Override protected IScalarEvaluator createEvaluator(IHyracksTaskContext ctx, IScalarEvaluator[] args) throws AlgebricksException { final ArrayBackedValueStorage abvs = new ArrayBackedValueStorage(); final DataOutput dOut = abvs.getDataOutput(); final TypedPointables tp = new TypedPointables(); return new AbstractTaggedValueArgumentScalarEvaluator(args) { @Override protected void evaluate(TaggedValuePointable[] args, IPointable result) throws SystemException { TaggedValuePointable tvp1 = args[0]; // Only accept node trees as input. try { int localNodeId = FunctionHelper.getLocalNodeId(tvp1, tp); if (localNodeId == -1) { XDMConstants.setEmptySequence(result); } else { abvs.reset(); dOut.write(ValueTag.XS_INT_TAG); dOut.writeInt(localNodeId); result.set(abvs); } } catch (Exception e) { throw new SystemException(ErrorCode.SYSE0001, e); } } }; }
55. IdFromNodeScalarEvaluatorFactory#createEvaluator()
Project: vxquery
File: IdFromNodeScalarEvaluatorFactory.java
File: IdFromNodeScalarEvaluatorFactory.java
@Override protected IScalarEvaluator createEvaluator(IHyracksTaskContext ctx, IScalarEvaluator[] args) throws AlgebricksException { final ArrayBackedValueStorage abvs = new ArrayBackedValueStorage(); final DataOutput dOut = abvs.getDataOutput(); final NodeTreePointable ntp = (NodeTreePointable) NodeTreePointable.FACTORY.createPointable(); final TypedPointables tp = new TypedPointables(); return new AbstractTaggedValueArgumentScalarEvaluator(args) { @Override protected void evaluate(TaggedValuePointable[] args, IPointable result) throws SystemException { TaggedValuePointable tvp1 = args[0]; // Only accept node trees as input. try { int localNodeId = FunctionHelper.getLocalNodeId(tvp1, tp); if (localNodeId == -1) { XDMConstants.setEmptySequence(result); } else { tvp1.getValue(ntp); abvs.reset(); dOut.write(ValueTag.XS_LONG_TAG); dOut.writeInt(ntp.getRootNodeId()); dOut.writeInt(localNodeId); result.set(abvs); } } catch (Exception e) { throw new SystemException(ErrorCode.SYSE0001, e); } } }; }
56. FnNumberScalarEvaluatorFactory#createEvaluator()
Project: vxquery
File: FnNumberScalarEvaluatorFactory.java
File: FnNumberScalarEvaluatorFactory.java
@Override protected IScalarEvaluator createEvaluator(IHyracksTaskContext ctx, IScalarEvaluator[] args) throws AlgebricksException { final ArrayBackedValueStorage abvs = new ArrayBackedValueStorage(); final DataOutput dOut = abvs.getDataOutput(); final CastToDoubleOperation castToDouble = new CastToDoubleOperation(); final TypedPointables tp = new TypedPointables(); return new AbstractTaggedValueArgumentScalarEvaluator(args) { @Override protected void evaluate(TaggedValuePointable[] args, IPointable result) throws SystemException { TaggedValuePointable tvp1 = args[0]; try { abvs.reset(); switch(tvp1.getTag()) { case ValueTag.XS_STRING_TAG: tvp1.getValue(tp.utf8sp); castToDouble.convertString(tp.utf8sp, dOut); break; case ValueTag.XS_UNTYPED_ATOMIC_TAG: tvp1.getValue(tp.utf8sp); castToDouble.convertUntypedAtomic(tp.utf8sp, dOut); break; case ValueTag.XS_BOOLEAN_TAG: tvp1.getValue(tp.boolp); castToDouble.convertBoolean(tp.boolp, dOut); break; // case ValueTag.XS_YEAR_MONTH_DURATION_TAG: case ValueTag.XS_INTEGER_TAG: case ValueTag.XS_NON_POSITIVE_INTEGER_TAG: case ValueTag.XS_NEGATIVE_INTEGER_TAG: case ValueTag.XS_LONG_TAG: case ValueTag.XS_NON_NEGATIVE_INTEGER_TAG: case ValueTag.XS_UNSIGNED_LONG_TAG: case ValueTag.XS_POSITIVE_INTEGER_TAG: case ValueTag.XS_UNSIGNED_INT_TAG: tvp1.getValue(tp.longp); castToDouble.convertInteger(tp.longp, dOut); break; // case ValueTag.XS_DAY_TIME_DURATION_TAG: case ValueTag.XS_INT_TAG: case ValueTag.XS_UNSIGNED_SHORT_TAG: tvp1.getValue(tp.intp); castToDouble.convertInt(tp.intp, dOut); break; case ValueTag.XS_DECIMAL_TAG: tvp1.getValue(tp.decp); castToDouble.convertDecimal(tp.decp, dOut); break; case ValueTag.XS_DOUBLE_TAG: tvp1.getValue(tp.doublep); castToDouble.convertDouble(tp.doublep, dOut); break; case ValueTag.XS_FLOAT_TAG: tvp1.getValue(tp.floatp); castToDouble.convertFloat(tp.floatp, dOut); break; case ValueTag.XS_SHORT_TAG: case ValueTag.XS_UNSIGNED_BYTE_TAG: tvp1.getValue(tp.shortp); castToDouble.convertShort(tp.shortp, dOut); break; case ValueTag.XS_BYTE_TAG: tvp1.getValue(tp.bytep); castToDouble.convertByte(tp.bytep, dOut); break; default: dOut.write(ValueTag.XS_DOUBLE_TAG); dOut.writeDouble(Double.NaN); } result.set(abvs); } catch (SystemException e) { try { abvs.reset(); dOut.write(ValueTag.XS_DOUBLE_TAG); dOut.writeDouble(Double.NaN); result.set(abvs); } catch (IOException e1) { throw new SystemException(ErrorCode.SYSE0001, e); } } catch (IOException e) { throw new SystemException(ErrorCode.SYSE0001, e); } } }; }
57. FnImplicitTimezoneScalarEvaluatorFactory#createEvaluator()
Project: vxquery
File: FnImplicitTimezoneScalarEvaluatorFactory.java
File: FnImplicitTimezoneScalarEvaluatorFactory.java
@Override protected IScalarEvaluator createEvaluator(IHyracksTaskContext ctx, IScalarEvaluator[] args) throws AlgebricksException { final DynamicContext dCtx = (DynamicContext) ctx.getJobletContext().getGlobalJobData(); final XSDateTimePointable datetimep = (XSDateTimePointable) XSDateTimePointable.FACTORY.createPointable(); final ArrayBackedValueStorage abvs = new ArrayBackedValueStorage(); final DataOutput dOut = abvs.getDataOutput(); return new AbstractTaggedValueArgumentScalarEvaluator(args) { @Override protected void evaluate(TaggedValuePointable[] args, IPointable result) throws SystemException { try { dCtx.getCurrentDateTime(datetimep); long value = datetimep.getTimezoneHour() * DateTime.CHRONON_OF_HOUR + datetimep.getTimezoneMinute() * DateTime.CHRONON_OF_MINUTE; abvs.reset(); dOut.write(ValueTag.XS_DAY_TIME_DURATION_TAG); dOut.writeLong(value); result.set(abvs); } catch (IOException e) { throw new SystemException(ErrorCode.SYSE0001, e); } } }; }
58. FnDateTimeScalarEvaluatorFactory#createEvaluator()
Project: vxquery
File: FnDateTimeScalarEvaluatorFactory.java
File: FnDateTimeScalarEvaluatorFactory.java
@Override protected IScalarEvaluator createEvaluator(IHyracksTaskContext ctx, IScalarEvaluator[] args) throws AlgebricksException { final XSDatePointable datep = (XSDatePointable) XSDatePointable.FACTORY.createPointable(); final XSTimePointable timep = (XSTimePointable) XSTimePointable.FACTORY.createPointable(); final SequencePointable seqp = (SequencePointable) SequencePointable.FACTORY.createPointable(); final ArrayBackedValueStorage abvs = new ArrayBackedValueStorage(); final DataOutput dOut = abvs.getDataOutput(); return new AbstractTaggedValueArgumentScalarEvaluator(args) { @Override protected void evaluate(TaggedValuePointable[] args, IPointable result) throws SystemException { TaggedValuePointable tvp1 = args[0]; if (tvp1.getTag() == ValueTag.SEQUENCE_TAG) { tvp1.getValue(seqp); if (seqp.getEntryCount() == 0) { XDMConstants.setEmptySequence(result); return; } } if (tvp1.getTag() != ValueTag.XS_DATE_TAG) { throw new SystemException(ErrorCode.FORG0006); } tvp1.getValue(datep); TaggedValuePointable tvp2 = args[1]; if (tvp2.getTag() == ValueTag.SEQUENCE_TAG) { tvp2.getValue(seqp); if (seqp.getEntryCount() == 0) { XDMConstants.setEmptySequence(result); return; } } if (tvp2.getTag() != ValueTag.XS_TIME_TAG) { throw new SystemException(ErrorCode.FORG0006); } tvp2.getValue(timep); // Set the timezone. byte timezoneHour, timezoneMinute; if (datep.getTimezoneHour() == DateTime.TIMEZONE_HOUR_NULL && datep.getTimezoneMinute() == DateTime.TIMEZONE_MINUTE_NULL && timep.getTimezoneHour() == DateTime.TIMEZONE_HOUR_NULL && timep.getTimezoneMinute() == DateTime.TIMEZONE_MINUTE_NULL) { // both null. timezoneHour = DateTime.TIMEZONE_HOUR_NULL; timezoneMinute = DateTime.TIMEZONE_MINUTE_NULL; } else if (datep.getTimezoneHour() == DateTime.TIMEZONE_HOUR_NULL && datep.getTimezoneMinute() == DateTime.TIMEZONE_MINUTE_NULL && timep.getTimezoneHour() != DateTime.TIMEZONE_HOUR_NULL && timep.getTimezoneMinute() != DateTime.TIMEZONE_MINUTE_NULL) { // date is null. timezoneHour = (byte) timep.getTimezoneHour(); timezoneMinute = (byte) timep.getTimezoneMinute(); } else if (datep.getTimezoneHour() != DateTime.TIMEZONE_HOUR_NULL && datep.getTimezoneMinute() != DateTime.TIMEZONE_MINUTE_NULL && timep.getTimezoneHour() == DateTime.TIMEZONE_HOUR_NULL && timep.getTimezoneMinute() == DateTime.TIMEZONE_MINUTE_NULL) { // time is null. timezoneHour = (byte) datep.getTimezoneHour(); timezoneMinute = (byte) datep.getTimezoneMinute(); } else if (datep.getTimezoneHour() == timep.getTimezoneHour() && datep.getTimezoneMinute() == timep.getTimezoneMinute()) { // timezones are the same. timezoneHour = (byte) datep.getTimezoneHour(); timezoneMinute = (byte) datep.getTimezoneMinute(); } else { // Neither match. throw new SystemException(ErrorCode.FORG0008); } try { abvs.reset(); dOut.write(ValueTag.XS_DATETIME_TAG); dOut.write(datep.getByteArray(), datep.getStartOffset(), XSDatePointable.TYPE_TRAITS.getFixedLength() - 2); dOut.write(timep.getByteArray(), timep.getStartOffset(), XSTimePointable.TYPE_TRAITS.getFixedLength() - 2); dOut.write(timezoneHour); dOut.write(timezoneMinute); result.set(abvs.getByteArray(), abvs.getStartOffset(), XSDateTimePointable.TYPE_TRAITS.getFixedLength() + 1); } catch (IOException e) { throw new SystemException(ErrorCode.SYSE0001, e); } } }; }
59. FnAdjustDateTimeToTimezoneScalarEvaluatorFactory#createEvaluator()
Project: vxquery
File: FnAdjustDateTimeToTimezoneScalarEvaluatorFactory.java
File: FnAdjustDateTimeToTimezoneScalarEvaluatorFactory.java
@Override protected IScalarEvaluator createEvaluator(IHyracksTaskContext ctx, IScalarEvaluator[] args) throws AlgebricksException { final DynamicContext dCtx = (DynamicContext) ctx.getJobletContext().getGlobalJobData(); final XSDateTimePointable datetimep = (XSDateTimePointable) XSDateTimePointable.FACTORY.createPointable(); final XSDateTimePointable ctxDatetimep = (XSDateTimePointable) XSDateTimePointable.FACTORY.createPointable(); final LongPointable longp = (LongPointable) LongPointable.FACTORY.createPointable(); final ArrayBackedValueStorage abvsInner = new ArrayBackedValueStorage(); final DataOutput dOutInner = abvsInner.getDataOutput(); return new AbstractTaggedValueArgumentScalarEvaluator(args) { @Override protected void evaluate(TaggedValuePointable[] args, IPointable result) throws SystemException { dCtx.getCurrentDateTime(ctxDatetimep); TaggedValuePointable tvp1 = args[0]; if (tvp1.getTag() != ValueTag.XS_DATETIME_TAG) { throw new SystemException(ErrorCode.FORG0006); } tvp1.getValue(datetimep); // Second argument is optional and will used the dynamic context if not supplied. long tz; if (args.length == 2) { TaggedValuePointable tvp2 = args[1]; if (tvp2.getTag() == ValueTag.XS_DAY_TIME_DURATION_TAG) { tvp2.getValue(longp); if (Math.abs(longp.getLong()) > DateTime.CHRONON_OF_HOUR * 14) { throw new SystemException(ErrorCode.FODT0003); } tz = longp.getLong() / DateTime.CHRONON_OF_MINUTE; } else { throw new SystemException(ErrorCode.FORG0006); } } else { tz = ctxDatetimep.getTimezoneHour() * 60 + ctxDatetimep.getTimezoneMinute(); } try { abvsInner.reset(); DateTime.adjustDateTimeToTimezone(datetimep, tz, dOutInner); result.set(abvsInner); } catch (IOException e) { throw new SystemException(ErrorCode.SYSE0001, e); } } }; }
60. AbstractValueFromDurationScalarEvaluatorFactory#createEvaluator()
Project: vxquery
File: AbstractValueFromDurationScalarEvaluatorFactory.java
File: AbstractValueFromDurationScalarEvaluatorFactory.java
@Override protected IScalarEvaluator createEvaluator(IHyracksTaskContext ctx, IScalarEvaluator[] args) throws AlgebricksException { final XSDurationPointable durationp = (XSDurationPointable) XSDurationPointable.FACTORY.createPointable(); final LongPointable longp = (LongPointable) LongPointable.FACTORY.createPointable(); final IntegerPointable intp = (IntegerPointable) IntegerPointable.FACTORY.createPointable(); final SequencePointable seqp = (SequencePointable) SequencePointable.FACTORY.createPointable(); final ArrayBackedValueStorage abvsInner = new ArrayBackedValueStorage(); final DataOutput dOutInner = abvsInner.getDataOutput(); return new AbstractTaggedValueArgumentScalarEvaluator(args) { @Override protected void evaluate(TaggedValuePointable[] args, IPointable result) throws SystemException { TaggedValuePointable tvp1 = args[0]; long value; long YMDuration = 0, DTDuration = 0; switch(tvp1.getTag()) { case ValueTag.SEQUENCE_TAG: tvp1.getValue(seqp); if (seqp.getEntryCount() == 0) { XDMConstants.setEmptySequence(result); return; } break; case ValueTag.XS_DURATION_TAG: tvp1.getValue(durationp); YMDuration = durationp.getYearMonth(); DTDuration = durationp.getDayTime(); break; case ValueTag.XS_DAY_TIME_DURATION_TAG: tvp1.getValue(longp); DTDuration = longp.getLong(); break; case ValueTag.XS_YEAR_MONTH_DURATION_TAG: tvp1.getValue(intp); YMDuration = intp.getInteger(); break; default: throw new SystemException(ErrorCode.FORG0006); } value = convertDuration(YMDuration, DTDuration); try { abvsInner.reset(); switch(getReturnTag()) { case ValueTag.XS_INTEGER_TAG: dOutInner.write(ValueTag.XS_INTEGER_TAG); dOutInner.writeLong(value); break; case ValueTag.XS_DECIMAL_TAG: long decimalPlace = 3; // Normalize to decimal. if (value % 1000 == 0) { value = value / 1000; decimalPlace = 0; } else if (value % 100 == 0) { value = value / 100; decimalPlace = 1; } else if (value % 10 == 0) { value = value / 10; decimalPlace = 2; } dOutInner.write(ValueTag.XS_DECIMAL_TAG); dOutInner.write((byte) decimalPlace); dOutInner.writeLong(value); break; } result.set(abvsInner); } catch (IOException e) { throw new SystemException(ErrorCode.SYSE0001, e); } } }; }
61. AbstractValueFromDateTimeScalarEvaluatorFactory#createEvaluator()
Project: vxquery
File: AbstractValueFromDateTimeScalarEvaluatorFactory.java
File: AbstractValueFromDateTimeScalarEvaluatorFactory.java
@Override protected IScalarEvaluator createEvaluator(IHyracksTaskContext ctx, IScalarEvaluator[] args) throws AlgebricksException { final ArrayBackedValueStorage abvsInner = new ArrayBackedValueStorage(); final DataOutput dOutInner = abvsInner.getDataOutput(); final SequencePointable seqp = (SequencePointable) SequencePointable.FACTORY.createPointable(); return new AbstractTaggedValueArgumentScalarEvaluator(args) { @Override protected void evaluate(TaggedValuePointable[] args, IPointable result) throws SystemException { TaggedValuePointable tvp1 = args[0]; if (tvp1.getTag() == ValueTag.SEQUENCE_TAG) { tvp1.getValue(seqp); if (seqp.getEntryCount() == 0) { XDMConstants.setEmptySequence(result); return; } } if (tvp1.getTag() != getInputTag()) { throw new SystemException(ErrorCode.FORG0006); } try { abvsInner.reset(); switch(getReturnTag()) { case ValueTag.XS_INTEGER_TAG: dOutInner.write(ValueTag.XS_INTEGER_TAG); dOutInner.writeLong(getValueAsInteger(tvp1)); break; case ValueTag.XS_DAY_TIME_DURATION_TAG: dOutInner.write(ValueTag.XS_DAY_TIME_DURATION_TAG); dOutInner.writeLong(getValueAsInteger(tvp1)); break; case ValueTag.XS_DECIMAL_TAG: long value = getValueAsInteger(tvp1); long decimalPlace = 3; // Normalize to decimal. if (value % 1000 == 0) { value = value / 1000; decimalPlace = 0; } else if (value % 100 == 0) { value = value / 100; decimalPlace = 1; } else if (value % 10 == 0) { value = value / 10; decimalPlace = 2; } dOutInner.write(ValueTag.XS_DECIMAL_TAG); dOutInner.write((byte) decimalPlace); dOutInner.writeLong(value); break; } result.set(abvsInner); } catch (IOException e) { throw new SystemException(ErrorCode.SYSE0001, e); } } }; }
62. FnSumScalarEvaluatorFactory#createEvaluator()
Project: vxquery
File: FnSumScalarEvaluatorFactory.java
File: FnSumScalarEvaluatorFactory.java
@Override protected IScalarEvaluator createEvaluator(IHyracksTaskContext ctx, IScalarEvaluator[] args) throws AlgebricksException { final DynamicContext dCtx = (DynamicContext) ctx.getJobletContext().getGlobalJobData(); final SequencePointable seqp = (SequencePointable) SequencePointable.FACTORY.createPointable(); final ArrayBackedValueStorage abvs = new ArrayBackedValueStorage(); final DataOutput dOut = abvs.getDataOutput(); final TaggedValuePointable tvpNext = (TaggedValuePointable) TaggedValuePointable.FACTORY.createPointable(); final TaggedValuePointable tvpSum = (TaggedValuePointable) TaggedValuePointable.FACTORY.createPointable(); final VoidPointable p = (VoidPointable) VoidPointable.FACTORY.createPointable(); final AddOperation aOpAdd = new AddOperation(); final ArithmeticHelper add = new ArithmeticHelper(aOpAdd, dCtx); return new AbstractTaggedValueArgumentScalarEvaluator(args) { @Override protected void evaluate(TaggedValuePointable[] args, IPointable result) throws SystemException { TaggedValuePointable tvp = args[0]; if (tvp.getTag() == ValueTag.SEQUENCE_TAG) { tvp.getValue(seqp); int seqLen = seqp.getEntryCount(); if (seqLen == 0) { // Default zero value as second argument. if (args.length == 2) { TaggedValuePointable tvp2 = args[1]; result.set(tvp2); } else { // No argument return an integer. try { abvs.reset(); dOut.write(ValueTag.XS_INTEGER_TAG); dOut.writeLong(0); result.set(abvs); } catch (Exception e) { throw new SystemException(ErrorCode.SYSE0001, e); } } } else { // Add up the sequence. for (int j = 0; j < seqLen; ++j) { seqp.getEntry(j, p); tvpNext.set(p.getByteArray(), p.getStartOffset(), p.getLength()); if (j == 0) { // Init. tvpSum.set(tvpNext); } else { add.compute(tvpNext, tvpSum, tvpSum); } } result.set(tvpSum); } } else { // Only one result. result.set(tvp); } } }; }
63. FnSumAggregateEvaluatorFactory#createEvaluator()
Project: vxquery
File: FnSumAggregateEvaluatorFactory.java
File: FnSumAggregateEvaluatorFactory.java
@Override protected IAggregateEvaluator createEvaluator(IScalarEvaluator[] args) throws AlgebricksException { final ArrayBackedValueStorage abvsSum = new ArrayBackedValueStorage(); final DataOutput dOutSum = abvsSum.getDataOutput(); final AddOperation aOpAdd = new AddOperation(); final ArithmeticHelper add = new ArithmeticHelper(aOpAdd, dCtx); return new AbstractTaggedValueArgumentAggregateEvaluator(args) { TaggedValuePointable tvpSum = (TaggedValuePointable) TaggedValuePointable.FACTORY.createPointable(); // TODO Check if the second argument is supplied as the zero value. @Override public void init() throws AlgebricksException { try { abvsSum.reset(); dOutSum.write(ValueTag.XS_INTEGER_TAG); dOutSum.writeLong(0); tvpSum.set(abvsSum); } catch (IOException e) { throw new AlgebricksException(e.toString()); } } @Override public void finishPartial(IPointable result) throws AlgebricksException { finish(result); } @Override public void finish(IPointable result) throws AlgebricksException { result.set(tvpSum); } @Override protected void step(TaggedValuePointable[] args) throws SystemException { TaggedValuePointable tvp = args[0]; add.compute(tvp, tvpSum, tvpSum); } }; }
64. FnCountScalarEvaluatorFactory#createEvaluator()
Project: vxquery
File: FnCountScalarEvaluatorFactory.java
File: FnCountScalarEvaluatorFactory.java
@Override protected IScalarEvaluator createEvaluator(IHyracksTaskContext ctx, IScalarEvaluator[] args) throws AlgebricksException { final SequencePointable seqp = (SequencePointable) SequencePointable.FACTORY.createPointable(); final ArrayBackedValueStorage abvs = new ArrayBackedValueStorage(); final DataOutput dOut = abvs.getDataOutput(); return new AbstractTaggedValueArgumentScalarEvaluator(args) { @Override protected void evaluate(TaggedValuePointable[] args, IPointable result) throws SystemException { TaggedValuePointable tvp = args[0]; long count = 0; if (tvp.getTag() == ValueTag.SEQUENCE_TAG) { tvp.getValue(seqp); count = seqp.getEntryCount(); } else { count = 1; } try { abvs.reset(); dOut.write(ValueTag.XS_INTEGER_TAG); dOut.writeLong(count); result.set(abvs); } catch (Exception e) { throw new SystemException(ErrorCode.SYSE0001, e); } } }; }
65. FnCountAggregateEvaluatorFactory#createEvaluator()
Project: vxquery
File: FnCountAggregateEvaluatorFactory.java
File: FnCountAggregateEvaluatorFactory.java
@Override protected IAggregateEvaluator createEvaluator(IScalarEvaluator[] args) throws AlgebricksException { final ArrayBackedValueStorage abvs = new ArrayBackedValueStorage(); final DataOutput dOut = abvs.getDataOutput(); return new AbstractTaggedValueArgumentAggregateEvaluator(args) { long count; @Override public void init() throws AlgebricksException { count = 0; } @Override public void finishPartial(IPointable result) throws AlgebricksException { finish(result); } @Override public void finish(IPointable result) throws AlgebricksException { try { abvs.reset(); dOut.write(ValueTag.XS_INTEGER_TAG); dOut.writeLong(count); result.set(abvs); } catch (Exception e) { throw new AlgebricksException(e); } } @Override protected void step(TaggedValuePointable[] args) throws SystemException { count++; } }; }
66. FnAvgScalarEvaluatorFactory#createEvaluator()
Project: vxquery
File: FnAvgScalarEvaluatorFactory.java
File: FnAvgScalarEvaluatorFactory.java
@Override protected IScalarEvaluator createEvaluator(IHyracksTaskContext ctx, IScalarEvaluator[] args) throws AlgebricksException { final DynamicContext dCtx = (DynamicContext) ctx.getJobletContext().getGlobalJobData(); final SequencePointable seqp = (SequencePointable) SequencePointable.FACTORY.createPointable(); final TaggedValuePointable tvpNext = (TaggedValuePointable) TaggedValuePointable.FACTORY.createPointable(); final TaggedValuePointable tvpSum = (TaggedValuePointable) TaggedValuePointable.FACTORY.createPointable(); final TaggedValuePointable tvpCount = (TaggedValuePointable) TaggedValuePointable.FACTORY.createPointable(); final ArrayBackedValueStorage abvs = new ArrayBackedValueStorage(); final DataOutput dOut = abvs.getDataOutput(); final AddOperation aOpAdd = new AddOperation(); final ArithmeticHelper add = new ArithmeticHelper(aOpAdd, dCtx); final DivideOperation aOpDivide = new DivideOperation(); final ArithmeticHelper divide = new ArithmeticHelper(aOpDivide, dCtx); return new AbstractTaggedValueArgumentScalarEvaluator(args) { @Override protected void evaluate(TaggedValuePointable[] args, IPointable result) throws SystemException { TaggedValuePointable tvp = args[0]; if (tvp.getTag() == ValueTag.SEQUENCE_TAG) { tvp.getValue(seqp); int seqLen = seqp.getEntryCount(); if (seqLen == 0) { XDMConstants.setEmptySequence(result); } else { // Add up the sequence. for (int j = 0; j < seqLen; ++j) { seqp.getEntry(j, tvpNext); if (j == 0) { // Init. tvpSum.set(tvpNext); } else { add.compute(tvpNext, tvpSum, tvpSum); } } // Set count as a TaggedValuePointable. try { abvs.reset(); dOut.write(ValueTag.XS_INTEGER_TAG); dOut.writeLong(seqLen); tvpCount.set(abvs); } catch (Exception e) { throw new SystemException(ErrorCode.SYSE0001, e); } divide.compute(tvpSum, tvpCount, tvpSum); result.set(tvpSum); } } else { // Only one result. result.set(tvp); } } }; }
67. AbstractMaxMinAggregateEvaluatorFactory#createEvaluator()
Project: vxquery
File: AbstractMaxMinAggregateEvaluatorFactory.java
File: AbstractMaxMinAggregateEvaluatorFactory.java
protected IAggregateEvaluator createEvaluator(IScalarEvaluator[] args) throws AlgebricksException { final AbstractValueComparisonOperation aOpComparison = createValueComparisonOperation(); final TaggedValuePointable tvp2 = (TaggedValuePointable) TaggedValuePointable.FACTORY.createPointable(); final SequencePointable seqp = (SequencePointable) SequencePointable.FACTORY.createPointable(); final TypedPointables tp1 = new TypedPointables(); final TypedPointables tp2 = new TypedPointables(); final ArrayBackedValueStorage abvs = new ArrayBackedValueStorage(); final DataOutput dOut = abvs.getDataOutput(); return new AbstractTaggedValueArgumentAggregateEvaluator(args) { long count; @Override public void init() throws AlgebricksException { count = 0; } @Override public void finishPartial(IPointable result) throws AlgebricksException { finish(result); } @Override public void finish(IPointable result) throws AlgebricksException { if (count == 0) { XDMConstants.setEmptySequence(result); } else { result.set(abvs); } } @Override protected void step(TaggedValuePointable[] args) throws SystemException { TaggedValuePointable tvp1 = args[0]; if (tvp1.getTag() == ValueTag.SEQUENCE_TAG) { // The local aggregate did not find a value so the global aggregate is receiving a empty sequence. tvp1.getValue(seqp); int seqLen = seqp.getEntryCount(); if (seqLen != 0) { throw new SystemException(ErrorCode.FORG0006); } } else { if (count != 0) { tvp2.set(abvs.getByteArray(), abvs.getStartOffset(), abvs.getLength()); } if (count == 0 || FunctionHelper.transformThenCompareMinMaxTaggedValues(aOpComparison, tvp1, tvp2, dCtx, tp1, tp2)) { try { abvs.reset(); dOut.write(tvp1.getByteArray(), tvp1.getStartOffset(), tvp1.getLength()); } catch (IOException e) { throw new SystemException(ErrorCode.SYSE0001, e); } } count++; } } }; }
68. TestTaskSpec#testSerDe()
Project: tez
File: TestTaskSpec.java
File: TestTaskSpec.java
@Test(timeout = 5000) public void testSerDe() throws IOException { ByteBuffer payload = null; ProcessorDescriptor procDesc = ProcessorDescriptor.create("proc").setUserPayload(UserPayload.create(payload)).setHistoryText("historyText"); List<InputSpec> inputSpecs = new ArrayList<>(); InputSpec inputSpec = new InputSpec("src1", InputDescriptor.create("inputClass"), 10); inputSpecs.add(inputSpec); List<OutputSpec> outputSpecs = new ArrayList<>(); OutputSpec outputSpec = new OutputSpec("dest1", OutputDescriptor.create("outputClass"), 999); outputSpecs.add(outputSpec); List<GroupInputSpec> groupInputSpecs = null; Configuration taskConf = new Configuration(false); taskConf.set("foo", "bar"); TezTaskAttemptID taId = TezTaskAttemptID.getInstance(TezTaskID.getInstance(TezVertexID.getInstance(TezDAGID.getInstance("1234", 1, 1), 1), 1), 1); TaskSpec taskSpec = new TaskSpec(taId, "dagName", "vName", -1, procDesc, inputSpecs, outputSpecs, groupInputSpecs, taskConf); ByteArrayOutputStream bos = new ByteArrayOutputStream(); DataOutput out = new DataOutputStream(bos); taskSpec.write(out); TaskSpec deSerTaskSpec = new TaskSpec(); ByteArrayInputStream bis = new ByteArrayInputStream(bos.toByteArray()); DataInput in = new DataInputStream(bis); deSerTaskSpec.readFields(in); Assert.assertEquals(taskSpec.getDAGName(), deSerTaskSpec.getDAGName()); Assert.assertEquals(taskSpec.getVertexName(), deSerTaskSpec.getVertexName()); Assert.assertEquals(taskSpec.getVertexParallelism(), deSerTaskSpec.getVertexParallelism()); Assert.assertEquals(taskSpec.getInputs().size(), deSerTaskSpec.getInputs().size()); Assert.assertEquals(taskSpec.getOutputs().size(), deSerTaskSpec.getOutputs().size()); Assert.assertNull(deSerTaskSpec.getGroupInputs()); Assert.assertEquals(taskSpec.getInputs().get(0).getSourceVertexName(), deSerTaskSpec.getInputs().get(0).getSourceVertexName()); Assert.assertEquals(taskSpec.getOutputs().get(0).getDestinationVertexName(), deSerTaskSpec.getOutputs().get(0).getDestinationVertexName()); Assert.assertEquals(taskConf.get("foo"), deSerTaskSpec.getTaskConf().get("foo")); }
69. TestMockDAGAppMaster#testBasicStatisticsMemory()
Project: tez
File: TestMockDAGAppMaster.java
File: TestMockDAGAppMaster.java
@Ignore @Test(timeout = 60000) public void testBasicStatisticsMemory() throws Exception { Logger.getRootLogger().setLevel(Level.WARN); TezConfiguration tezconf = new TezConfiguration(defaultConf); MockTezClient tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null, null, false, false); tezClient.start(); final String vAName = "abcdefghijklmnopqrstuvwxyz"; int numTasks = 10000; int numSources = 10; IOStatistics ioStats = new IOStatistics(); ioStats.setDataSize(1); ioStats.setItemsProcessed(1); TaskStatistics vAStats = new TaskStatistics(); DAG dag = DAG.create("testBasicStatisticsMemory"); Vertex vA = Vertex.create(vAName, ProcessorDescriptor.create("Proc.class"), numTasks); for (int i = 0; i < numSources; ++i) { final String sourceName = i + vAName; vA.addDataSource(sourceName, DataSourceDescriptor.create(InputDescriptor.create(sourceName), null, null)); vAStats.addIO(sourceName, ioStats); } dag.addVertex(vA); ByteArrayOutputStream bosA = new ByteArrayOutputStream(); DataOutput outA = new DataOutputStream(bosA); vAStats.write(outA); final byte[] payloadA = bosA.toByteArray(); MockDAGAppMaster mockApp = tezClient.getLocalClient().getMockApp(); MockContainerLauncher mockLauncher = mockApp.getContainerLauncher(); mockLauncher.startScheduling(false); mockApp.statsDelegate = new StatisticsDelegate() { @Override public TaskStatistics getStatistics(TaskSpec taskSpec) { byte[] payload = payloadA; TaskStatistics stats = new TaskStatistics(); final DataInputByteBuffer in = new DataInputByteBuffer(); in.reset(ByteBuffer.wrap(payload)); try { // this ensures that the serde code path is covered. stats.readFields(in); } catch (IOException e) { Assert.fail(e.getMessage()); } return stats; } }; mockApp.doSleep = false; DAGClient dagClient = tezClient.submitDAG(dag); mockLauncher.waitTillContainersLaunched(); DAGImpl dagImpl = (DAGImpl) mockApp.getContext().getCurrentDAG(); mockLauncher.startScheduling(true); DAGStatus status = dagClient.waitForCompletion(); Assert.assertEquals(DAGStatus.State.SUCCEEDED, status.getState()); Assert.assertEquals(numTasks, dagImpl.getVertex(vAName).getStatistics().getInputStatistics(0 + vAName).getDataSize()); Assert.assertEquals(numTasks, dagImpl.getVertex(vAName).getStatistics().getInputStatistics(0 + vAName).getItemsProcessed()); checkMemory(dag.getName(), mockApp); tezClient.stop(); }
70. TestMockDAGAppMaster#testBasicCounters()
Project: tez
File: TestMockDAGAppMaster.java
File: TestMockDAGAppMaster.java
@Test(timeout = 10000) public void testBasicCounters() throws Exception { TezConfiguration tezconf = new TezConfiguration(defaultConf); MockTezClient tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null, null, false, false); tezClient.start(); final String vAName = "A"; final String vBName = "B"; final String procCounterName = "Proc"; final String globalCounterName = "Global"; DAG dag = DAG.create("testBasicCounters"); Vertex vA = Vertex.create(vAName, ProcessorDescriptor.create("Proc.class"), 10); Vertex vB = Vertex.create(vBName, ProcessorDescriptor.create("Proc.class"), 1); dag.addVertex(vA).addVertex(vB).addEdge(Edge.create(vA, vB, EdgeProperty.create(DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("Out"), InputDescriptor.create("In")))); TezCounters temp = new TezCounters(); temp.findCounter(new String(globalCounterName), new String(globalCounterName)).increment(1); ByteArrayOutputStream bos = new ByteArrayOutputStream(); DataOutput out = new DataOutputStream(bos); temp.write(out); final byte[] payload = bos.toByteArray(); MockDAGAppMaster mockApp = tezClient.getLocalClient().getMockApp(); MockContainerLauncher mockLauncher = mockApp.getContainerLauncher(); mockLauncher.startScheduling(false); mockApp.countersDelegate = new CountersDelegate() { @Override public TezCounters getCounters(TaskSpec taskSpec) { String vName = taskSpec.getVertexName(); TezCounters counters = new TezCounters(); final DataInputByteBuffer in = new DataInputByteBuffer(); in.reset(ByteBuffer.wrap(payload)); try { // this ensures that the serde code path is covered. // the internal merges of counters covers the constructor code path. counters.readFields(in); } catch (IOException e) { Assert.fail(e.getMessage()); } counters.findCounter(vName, procCounterName).increment(1); for (OutputSpec output : taskSpec.getOutputs()) { counters.findCounter(vName, output.getDestinationVertexName()).increment(1); } for (InputSpec input : taskSpec.getInputs()) { counters.findCounter(vName, input.getSourceVertexName()).increment(1); } return counters; } }; mockApp.doSleep = false; DAGClient dagClient = tezClient.submitDAG(dag); mockLauncher.waitTillContainersLaunched(); DAGImpl dagImpl = (DAGImpl) mockApp.getContext().getCurrentDAG(); mockLauncher.startScheduling(true); DAGStatus status = dagClient.waitForCompletion(); Assert.assertEquals(DAGStatus.State.SUCCEEDED, status.getState()); TezCounters counters = dagImpl.getAllCounters(); String osName = System.getProperty("os.name").toLowerCase(Locale.ENGLISH); if (SystemUtils.IS_OS_LINUX) { Assert.assertTrue(counters.findCounter(DAGCounter.AM_CPU_MILLISECONDS).getValue() > 0); } // verify processor counters Assert.assertEquals(10, counters.findCounter(vAName, procCounterName).getValue()); Assert.assertEquals(1, counters.findCounter(vBName, procCounterName).getValue()); // verify edge counters Assert.assertEquals(10, counters.findCounter(vAName, vBName).getValue()); Assert.assertEquals(1, counters.findCounter(vBName, vAName).getValue()); // verify global counters Assert.assertEquals(11, counters.findCounter(globalCounterName, globalCounterName).getValue()); VertexImpl vAImpl = (VertexImpl) dagImpl.getVertex(vAName); VertexImpl vBImpl = (VertexImpl) dagImpl.getVertex(vBName); TezCounters vACounters = vAImpl.getAllCounters(); TezCounters vBCounters = vBImpl.getAllCounters(); String vACounterName = vACounters.findCounter(globalCounterName, globalCounterName).getName(); String vBCounterName = vBCounters.findCounter(globalCounterName, globalCounterName).getName(); if (vACounterName != vBCounterName) { Assert.fail("String counter name objects dont match despite interning."); } CounterGroup vaGroup = vACounters.getGroup(globalCounterName); String vaGrouName = vaGroup.getName(); CounterGroup vBGroup = vBCounters.getGroup(globalCounterName); String vBGrouName = vBGroup.getName(); if (vaGrouName != vBGrouName) { Assert.fail("String group name objects dont match despite interning."); } tezClient.stop(); }
71. PlayerTracker#onPlayerLogin()
Project: TerrainControl
File: PlayerTracker.java
File: PlayerTracker.java
@SubscribeEvent public void onPlayerLogin(PlayerEvent.PlayerLoggedInEvent event) { // Get the config if (!(event.player instanceof EntityPlayerMP)) { return; } EntityPlayerMP player = (EntityPlayerMP) event.player; LocalWorld worldTC = WorldHelper.toLocalWorld(player.getEntityWorld()); if (worldTC == null) { // World not loaded return; } ConfigProvider configs = worldTC.getConfigs(); // Serialize it ByteBuf nettyBuffer = Unpooled.buffer(); PacketBuffer mojangBuffer = new PacketBuffer(nettyBuffer); DataOutput stream = new ByteBufOutputStream(nettyBuffer); try { stream.writeInt(PluginStandardValues.ProtocolVersion); ConfigToNetworkSender.send(configs, stream); } catch (IOException e) { TerrainControl.printStackTrace(LogMarker.FATAL, e); } // Make the packet SPacketCustomPayload packet = new SPacketCustomPayload(PluginStandardValues.ChannelName, mojangBuffer); // Send the packet player.playerNetServerHandler.sendPacket(packet); }
72. HyperLogLog#getBytes()
Project: stream-lib
File: HyperLogLog.java
File: HyperLogLog.java
@Override public byte[] getBytes() throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutput dos = new DataOutputStream(baos); writeBytes(dos); return baos.toByteArray(); }
73. DataOutputTest#testWrite()
Project: stratio-cassandra
File: DataOutputTest.java
File: DataOutputTest.java
private DataInput testWrite(DataOutputPlus test) throws IOException { final ByteArrayOutputStream bos = new ByteArrayOutputStream(); final DataOutput canon = new DataOutputStream(bos); Random rnd = ThreadLocalRandom.current(); byte[] bytes = new byte[50]; rnd.nextBytes(bytes); ByteBufferUtil.writeWithLength(bytes, test); ByteBufferUtil.writeWithLength(bytes, canon); bytes = new byte[50]; rnd.nextBytes(bytes); ByteBufferUtil.writeWithLength(wrap(bytes, false), test); ByteBufferUtil.writeWithLength(bytes, canon); bytes = new byte[50]; rnd.nextBytes(bytes); ByteBufferUtil.writeWithLength(wrap(bytes, true), test); ByteBufferUtil.writeWithLength(bytes, canon); bytes = new byte[50]; rnd.nextBytes(bytes); ByteBufferUtil.writeWithShortLength(bytes, test); ByteBufferUtil.writeWithShortLength(bytes, canon); bytes = new byte[50]; rnd.nextBytes(bytes); ByteBufferUtil.writeWithShortLength(wrap(bytes, false), test); ByteBufferUtil.writeWithShortLength(bytes, canon); bytes = new byte[50]; rnd.nextBytes(bytes); ByteBufferUtil.writeWithShortLength(wrap(bytes, true), test); ByteBufferUtil.writeWithShortLength(bytes, canon); // 318 { long v = rnd.nextLong(); test.writeLong(v); canon.writeLong(v); } { int v = rnd.nextInt(); test.writeInt(v); canon.writeInt(v); } { short v = (short) rnd.nextInt(); test.writeShort(v); canon.writeShort(v); } { byte v = (byte) rnd.nextInt(); test.write(v); canon.write(v); } { double v = rnd.nextDouble(); test.writeDouble(v); canon.writeDouble(v); } { float v = (float) rnd.nextDouble(); test.writeFloat(v); canon.writeFloat(v); } // 27 return new DataInputStream(new ByteArrayInputStream(bos.toByteArray())); }
74. TestDataModel#testReadWriteInternal()
Project: pig
File: TestDataModel.java
File: TestDataModel.java
@Test public void testReadWriteInternal() throws Exception { // Create a tuple with every internal data type in it, and then read and // write it, both via DataReaderWriter and Tuple.readFields TupleFactory tf = TupleFactory.getInstance(); Tuple t1 = tf.newTuple(1); InternalMap map = new InternalMap(2); map.put(new Integer(1), new String("world")); map.put(new Long(3L), new String("all")); t1.set(0, map); File file = File.createTempFile("Tuple", "put"); FileOutputStream fos = new FileOutputStream(file); DataOutput out = new DataOutputStream(fos); t1.write(out); fos.close(); FileInputStream fis = new FileInputStream(file); DataInput in = new DataInputStream(fis); Tuple after = tf.newTuple(); after.readFields(in); Object o = after.get(0); assertTrue("isa InternalMap", o instanceof InternalMap); InternalMap m = (InternalMap) o; assertEquals("world", (String) m.get(new Integer(1))); assertEquals("all", (String) m.get(new Long(3L))); assertNull(m.get("fred")); file.delete(); }
75. TestDataModel#testReadWrite()
Project: pig
File: TestDataModel.java
File: TestDataModel.java
@Test public void testReadWrite() throws Exception { // Create a tuple with every data type in it, and then read and // write it, both via DataReaderWriter and Tuple.readFields TupleFactory tf = TupleFactory.getInstance(); Tuple t1 = giveMeOneOfEach(); File file = File.createTempFile("Tuple", "put"); FileOutputStream fos = new FileOutputStream(file); DataOutput out = new DataOutputStream(fos); t1.write(out); // twice in a row on purpose t1.write(out); fos.close(); FileInputStream fis = new FileInputStream(file); DataInput in = new DataInputStream(fis); for (int i = 0; i < 2; i++) { Tuple after = tf.newTuple(); after.readFields(in); Object o = after.get(0); assertTrue("isa Tuple", o instanceof Tuple); Tuple t3 = (Tuple) o; o = t3.get(0); assertTrue("isa Integer", o instanceof Integer); assertEquals(new Integer(3), (Integer) o); o = t3.get(1); assertTrue("isa Float", o instanceof Float); assertEquals(new Float(3.0), (Float) o); o = after.get(1); assertTrue("isa Bag", o instanceof DataBag); DataBag b = (DataBag) o; Iterator<Tuple> j = b.iterator(); Tuple[] ts = new Tuple[2]; assertTrue("first tuple in bag", j.hasNext()); ts[0] = j.next(); assertTrue("second tuple in bag", j.hasNext()); ts[1] = j.next(); o = ts[0].get(0); assertTrue("isa Integer", o instanceof Integer); assertEquals(new Integer(4), (Integer) o); o = ts[1].get(0); assertTrue("isa String", o instanceof String); assertEquals("mary had a little lamb", (String) o); o = after.get(2); assertTrue("isa Map", o instanceof Map); Map<String, Object> m = (Map<String, Object>) o; assertEquals("world", (String) m.get("hello")); assertEquals("all", (String) m.get("goodbye")); assertNull(m.get("fred")); o = after.get(3); assertTrue("isa Integer", o instanceof Integer); Integer ii = (Integer) o; assertEquals(new Integer(42), ii); o = after.get(4); assertTrue("isa Long", o instanceof Long); Long l = (Long) o; assertEquals(new Long(5000000000L), l); o = after.get(5); assertTrue("isa Float", o instanceof Float); Float f = (Float) o; assertEquals(new Float(3.141592654), f); o = after.get(6); assertTrue("isa Double", o instanceof Double); Double d = (Double) o; assertEquals(new Double(2.99792458e8), d); o = after.get(7); assertTrue("isa Boolean", o instanceof Boolean); Boolean bool = (Boolean) o; assertTrue(bool); o = after.get(8); assertTrue("isa DataByteArray", o instanceof DataByteArray); DataByteArray ba = (DataByteArray) o; assertEquals(new DataByteArray("hello"), ba); o = after.get(9); assertTrue("isa String", o instanceof String); String s = (String) o; assertEquals("goodbye", s); } file.delete(); }
76. TestBinInterSedes#testSerTuple()
Project: pig
File: TestBinInterSedes.java
File: TestBinInterSedes.java
private void testSerTuple(Tuple t, byte[] expected) throws Exception { ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutput out = new DataOutputStream(baos); bis.writeDatum(out, t); Tuple t2 = (Tuple) bis.readDatum(new DataInputStream(new ByteArrayInputStream(baos.toByteArray()))); assertEquals(t, t2); }
77. TestSchemaTuple#testSerDe()
Project: pig
File: TestSchemaTuple.java
File: TestSchemaTuple.java
public void testSerDe(SchemaTupleFactory tf) throws IOException { int sz = 4096; List<Tuple> written = new ArrayList<Tuple>(sz); File temp = File.createTempFile("tmp", "tmp"); temp.deleteOnExit(); FileOutputStream fos = new FileOutputStream(temp); DataOutput dos = new DataOutputStream(fos); for (int i = 0; i < sz; i++) { SchemaTuple<?> st = (SchemaTuple<?>) tf.newTuple(); fillWithData(st); bis.writeDatum(dos, st); written.add(st); } fos.close(); assertEquals(sz, written.size()); FileInputStream fis = new FileInputStream(temp); DataInput din = new DataInputStream(fis); for (int i = 0; i < sz; i++) { SchemaTuple<?> st = (SchemaTuple<?>) bis.readDatum(din); assertEquals(written.get(i), st); } fis.close(); }
78. TestPigBytesRawComparator#serializeAsNullableBytesWritable()
Project: pig
File: TestPigBytesRawComparator.java
File: TestPigBytesRawComparator.java
// Wrap the passed object with NullableBytesWritable and return the serialized // form. private byte[] serializeAsNullableBytesWritable(Object obj) throws Exception { NullableBytesWritable nbw = new NullableBytesWritable(obj); bas.reset(); DataOutput dout = new DataOutputStream(bas); nbw.write(dout); return bas.toByteArray(); }
79. PigSplit#write()
Project: pig
File: PigSplit.java
File: PigSplit.java
@Override @SuppressWarnings("unchecked") public void write(DataOutput os) throws IOException { os.writeBoolean(disableCounter); os.writeBoolean(isMultiInputs); os.writeInt(totalSplits); os.writeInt(splitIndex); os.writeInt(inputIndex); writeObject(targetOps, os); os.writeInt(wrappedSplits.length); Set<String> splitClassNameSet = new HashSet<String>(); //first get the distinct split class name set for (int i = 0; i < wrappedSplits.length; i++) { splitClassNameSet.add(wrappedSplits[i].getClass().getName()); } List<String> distinctSplitClassList = new ArrayList<String>(); distinctSplitClassList.addAll(splitClassNameSet); boolean nonFileSplit = distinctSplitClassList.size() > 1 || (!distinctSplitClassList.contains(FILESPLIT_CLASSNAME)); //write the distinct number of split class name os.writeInt(distinctSplitClassList.size()); //write each classname once for (int i = 0; i < distinctSplitClassList.size(); i++) { os.writeUTF(distinctSplitClassList.get(i)); } SerializationFactory sf = new SerializationFactory(conf); if (wrappedSplits.length <= 0) { return; } boolean compress = nonFileSplit && conf.getBoolean(PigConfiguration.PIG_COMPRESS_INPUT_SPLITS, PigConfiguration.PIG_COMPRESS_INPUT_SPLITS_DEFAULT); WritableByteArray byteStream = null; Deflater deflater = null; DataOutputStream dos = null; if (compress) { byteStream = new WritableByteArray(16384); deflater = new Deflater(Deflater.BEST_COMPRESSION); dos = new DataOutputStream(new DeflaterOutputStream(byteStream, deflater)); } DataOutput dataOut = compress ? dos : os; for (int i = 0; i < wrappedSplits.length; i++) { //find out the index of the split class name int index = distinctSplitClassList.indexOf(wrappedSplits[i].getClass().getName()); dataOut.writeInt(index); Serializer s = sf.getSerializer(wrappedSplits[i].getClass()); //Checks if Serializer is NULL or not before calling open() method on it. if (s == null) { throw new IllegalArgumentException("Could not find Serializer for class " + wrappedSplits[i].getClass() + ". InputSplits must implement Writable."); } s.open((OutputStream) dataOut); // The correct call sequence for Serializer is, we shall open, then serialize, but we shall not close s.serialize(wrappedSplits[i]); } if (compress) { //Get the compressed serialized bytes and write them dos.close(); os.writeInt(byteStream.getLength()); os.write(byteStream.getData(), 0, byteStream.getLength()); deflater.end(); } }
80. TestDefaultHCatRecord#testRYW()
Project: hcatalog
File: TestDefaultHCatRecord.java
File: TestDefaultHCatRecord.java
public void testRYW() throws IOException { File f = new File("binary.dat"); f.delete(); f.createNewFile(); f.deleteOnExit(); OutputStream fileOutStream = new FileOutputStream(f); DataOutput outStream = new DataOutputStream(fileOutStream); HCatRecord[] recs = getHCatRecords(); for (int i = 0; i < recs.length; i++) { recs[i].write(outStream); } fileOutStream.flush(); fileOutStream.close(); InputStream fInStream = new FileInputStream(f); DataInput inpStream = new DataInputStream(fInStream); for (int i = 0; i < recs.length; i++) { HCatRecord rec = new DefaultHCatRecord(); rec.readFields(inpStream); Assert.assertEquals(recs[i], rec); } Assert.assertEquals(fInStream.available(), 0); fInStream.close(); }
81. TypedBytesInputWriter#initialize()
Project: hadoop-mapreduce
File: TypedBytesInputWriter.java
File: TypedBytesInputWriter.java
@Override public void initialize(PipeMapRed pipeMapRed) throws IOException { super.initialize(pipeMapRed); DataOutput clientOut = pipeMapRed.getClientOutput(); tbOut = new TypedBytesOutput(clientOut); tbwOut = new TypedBytesWritableOutput(clientOut); }
82. TypedBytesInputWriter#initialize()
Project: hadoop-common
File: TypedBytesInputWriter.java
File: TypedBytesInputWriter.java
@Override public void initialize(PipeMapRed pipeMapRed) throws IOException { super.initialize(pipeMapRed); DataOutput clientOut = pipeMapRed.getClientOutput(); tbOut = new TypedBytesOutput(clientOut); tbwOut = new TypedBytesWritableOutput(clientOut); }
83. TestPartitionRequest#testPartitionRequest()
Project: goldenorb
File: TestPartitionRequest.java
File: TestPartitionRequest.java
/* * Start of user / non-generated code -- any code written outside of this block will be * removed in subsequent code generations. */ /* End of user / non-generated code */ @Before public void testPartitionRequest() throws IOException { /* * Start of user / non-generated code -- any code written outside of this block will be * removed in subsequent code generations. */ /* End of user / non-generated code */ partitionRequest = new PartitionRequest(); partitionRequest.setReservedPartitions(INT_RESERVEDPARTITIONS_VALUE); partitionRequest.setActivePartitions(INT_ACTIVEPARTITIONS_VALUE); partitionRequest.setJobID(STRING_JOBID_VALUE); partitionRequest.setBasePartitionID(INT_BASEPARTITIONID_VALUE); partitionRequest.setJobConf(new OrbConfiguration(true)); ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutput out = new DataOutputStream(baos); partitionRequest.write(out); DataInput in = new DataInputStream(new ByteArrayInputStream(baos.toByteArray())); partitionRequestOut = new PartitionRequest(); partitionRequestOut.readFields(in); /* * Start of user / non-generated code -- any code written outside of this block will be * removed in subsequent code generations. */ /* End of user / non-generated code */ }
84. TestOrbTrackerMember#testOrbTrackerMember()
Project: goldenorb
File: TestOrbTrackerMember.java
File: TestOrbTrackerMember.java
/* * Start of user / non-generated code -- any code written outside of this block will be * removed in subsequent code generations. */ /* End of user / non-generated code */ @Before public void testOrbTrackerMember() throws IOException { /* * Start of user / non-generated code -- any code written outside of this block will be * removed in subsequent code generations. */ /* End of user / non-generated code */ orbTrackerMember = new OrbTrackerMember(); orbTrackerMember.setPartitionCapacity(INT_PARTITIONCAPACITY_VALUE); orbTrackerMember.setAvailablePartitions(INT_AVAILABLEPARTITIONS_VALUE); orbTrackerMember.setReservedPartitions(INT_RESERVEDPARTITIONS_VALUE); orbTrackerMember.setInUsePartitions(INT_INUSEPARTITIONS_VALUE); orbTrackerMember.setHostname(STRING_HOSTNAME_VALUE); orbTrackerMember.setLeader(BOOLEAN_LEADER_VALUE); orbTrackerMember.setPort(INT_PORT_VALUE); ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutput out = new DataOutputStream(baos); orbTrackerMember.write(out); DataInput in = new DataInputStream(new ByteArrayInputStream(baos.toByteArray())); orbTrackerMemberOut = new OrbTrackerMember(); orbTrackerMemberOut.readFields(in); /* * Start of user / non-generated code -- any code written outside of this block will be * removed in subsequent code generations. */ /* End of user / non-generated code */ }
85. TestOrbPartitionMember#testOrbPartitionMember()
Project: goldenorb
File: TestOrbPartitionMember.java
File: TestOrbPartitionMember.java
/* * Start of user / non-generated code -- any code written outside of this block will be * removed in subsequent code generations. */ /* End of user / non-generated code */ @Before public void testOrbPartitionMember() throws IOException { /* * Start of user / non-generated code -- any code written outside of this block will be * removed in subsequent code generations. */ /* End of user / non-generated code */ orbPartitionMember = new OrbPartitionMember(); orbPartitionMember.setPartitionID(INT_PARTITIONID_VALUE); orbPartitionMember.setNumberOfVertices(INT_NUMBEROFVERTICES_VALUE); orbPartitionMember.setSuperStep(INT_SUPERSTEP_VALUE); orbPartitionMember.setMessagesSent(INT_MESSAGESSENT_VALUE); orbPartitionMember.setPercentComplete(FLOAT_PERCENTCOMPLETE_VALUE); orbPartitionMember.setHostname(STRING_HOSTNAME_VALUE); orbPartitionMember.setLeader(BOOLEAN_LEADER_VALUE); orbPartitionMember.setPort(INT_PORT_VALUE); ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutput out = new DataOutputStream(baos); orbPartitionMember.write(out); DataInput in = new DataInputStream(new ByteArrayInputStream(baos.toByteArray())); orbPartitionMemberOut = new OrbPartitionMember(); orbPartitionMemberOut.readFields(in); /* * Start of user / non-generated code -- any code written outside of this block will be * removed in subsequent code generations. */ /* End of user / non-generated code */ }
86. ZookeeperUtils#writableToByteArray()
Project: goldenorb
File: ZookeeperUtils.java
File: ZookeeperUtils.java
/** * * @param w * - Writable * @returns byte[] */ public static byte[] writableToByteArray(Writable w) throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutput out = new DataOutputStream(baos); w.write(out); return baos.toByteArray(); }
87. BspServiceWorker#storeCheckpoint()
Project: giraph
File: BspServiceWorker.java
File: BspServiceWorker.java
@Override public void storeCheckpoint() throws IOException { LoggerUtils.setStatusAndLog(getContext(), LOG, Level.INFO, "storeCheckpoint: Starting checkpoint " + getGraphTaskManager().getGraphFunctions().toString() + " - Attempt=" + getApplicationAttempt() + ", Superstep=" + getSuperstep()); // Algorithm: // For each partition, dump vertices and messages Path metadataFilePath = new Path(getCheckpointBasePath(getSuperstep()) + "." + getHostnamePartitionId() + CHECKPOINT_METADATA_POSTFIX); Path verticesFilePath = new Path(getCheckpointBasePath(getSuperstep()) + "." + getHostnamePartitionId() + CHECKPOINT_VERTICES_POSTFIX); Path validFilePath = new Path(getCheckpointBasePath(getSuperstep()) + "." + getHostnamePartitionId() + CHECKPOINT_VALID_POSTFIX); // of previous failure of this worker) if (getFs().delete(validFilePath, false)) { LOG.warn("storeCheckpoint: Removed valid file " + validFilePath); } if (getFs().delete(metadataFilePath, false)) { LOG.warn("storeCheckpoint: Removed metadata file " + metadataFilePath); } if (getFs().delete(verticesFilePath, false)) { LOG.warn("storeCheckpoint: Removed file " + verticesFilePath); } FSDataOutputStream verticesOutputStream = getFs().create(verticesFilePath); ByteArrayOutputStream metadataByteStream = new ByteArrayOutputStream(); DataOutput metadataOutput = new DataOutputStream(metadataByteStream); for (Integer partitionId : getPartitionStore().getPartitionIds()) { Partition<I, V, E, M> partition = getPartitionStore().getPartition(partitionId); long startPos = verticesOutputStream.getPos(); partition.write(verticesOutputStream); // write messages getServerData().getCurrentMessageStore().writePartition(verticesOutputStream, partition.getId()); // Write the metadata for this partition // Format: // <index count> // <index 0 start pos><partition id> // <index 1 start pos><partition id> metadataOutput.writeLong(startPos); metadataOutput.writeInt(partition.getId()); if (LOG.isDebugEnabled()) { LOG.debug("storeCheckpoint: Vertex file starting " + "offset = " + startPos + ", length = " + (verticesOutputStream.getPos() - startPos) + ", partition = " + partition.toString()); } getPartitionStore().putPartition(partition); getContext().progress(); } // Metadata is buffered and written at the end since it's small and // needs to know how many partitions this worker owns FSDataOutputStream metadataOutputStream = getFs().create(metadataFilePath); metadataOutputStream.writeInt(getPartitionStore().getNumPartitions()); metadataOutputStream.write(metadataByteStream.toByteArray()); metadataOutputStream.close(); verticesOutputStream.close(); if (LOG.isInfoEnabled()) { LOG.info("storeCheckpoint: Finished metadata (" + metadataFilePath + ") and vertices (" + verticesFilePath + ")."); } getFs().createNewFile(validFilePath); // Notify master that checkpoint is stored String workerWroteCheckpoint = getWorkerWroteCheckpointPath(getApplicationAttempt(), getSuperstep()) + "/" + getHostnamePartitionId(); try { getZkExt().createExt(workerWroteCheckpoint, new byte[0], Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT, true); } catch (KeeperException.NodeExistsException e) { LOG.warn("storeCheckpoint: wrote checkpoint worker path " + workerWroteCheckpoint + " already exists!"); } catch (KeeperException e) { throw new IllegalStateException("Creating " + workerWroteCheckpoint + " failed with KeeperException", e); } catch (InterruptedException e) { throw new IllegalStateException("Creating " + workerWroteCheckpoint + " failed with InterruptedException", e); } }
88. VertexToFaunusBinaryTest#testConversion()
Project: faunus
File: VertexToFaunusBinaryTest.java
File: VertexToFaunusBinaryTest.java
public void testConversion() throws IOException { Graph graph = new TinkerGraph(); Vertex marko = graph.addVertex(1); marko.setProperty("name", "marko"); marko.setProperty("age", 32); Vertex stephen = graph.addVertex(3); stephen.setProperty("name", "stephen"); stephen.setProperty("weight", 160.42); stephen.setProperty("male", true); Edge e = graph.addEdge(null, marko, stephen, "knows"); e.setProperty("weight", 0.2); e.setProperty("type", "coworker"); ByteArrayOutputStream bytes1 = new ByteArrayOutputStream(); DataOutput out = new DataOutputStream(bytes1); VertexToFaunusBinary.write(marko, out); VertexToFaunusBinary.write(stephen, out); DataInput in = new DataInputStream(new ByteArrayInputStream(bytes1.toByteArray())); FaunusVertex markoFaunus = new FaunusVertex(in); assertEquals(markoFaunus.getProperty("name"), "marko"); assertEquals(markoFaunus.getProperty("age"), 32); assertEquals(markoFaunus.getPropertyKeys().size(), 2); assertEquals(asList(markoFaunus.getEdges(Direction.OUT)).size(), 1); assertFalse(markoFaunus.getEdges(Direction.IN).iterator().hasNext()); assertTrue(markoFaunus.getEdges(Direction.OUT, "knows").iterator().hasNext()); assertFalse(markoFaunus.getEdges(Direction.OUT, "blah").iterator().hasNext()); FaunusEdge edge = (FaunusEdge) markoFaunus.getEdges(Direction.OUT).iterator().next(); assertEquals(edge.getLabel(), "knows"); assertEquals(edge.getProperty("weight"), 0.2); assertEquals(edge.getProperty("type"), "coworker"); assertEquals(edge.getPropertyKeys().size(), 2); assertEquals(edge.getVertex(Direction.IN).getId(), 3l); assertEquals(edge.getVertex(Direction.OUT).getId(), 1l); FaunusVertex stephenFaunus = new FaunusVertex(in); assertEquals(stephenFaunus.getProperty("name"), "stephen"); assertEquals(stephenFaunus.getProperty("weight"), 160.42); assertTrue((Boolean) stephenFaunus.getProperty("male")); assertEquals(stephenFaunus.getPropertyKeys().size(), 3); assertEquals(asList(stephenFaunus.getEdges(Direction.IN)).size(), 1); assertFalse(stephenFaunus.getEdges(Direction.OUT).iterator().hasNext()); assertTrue(stephenFaunus.getEdges(Direction.IN, "knows").iterator().hasNext()); assertFalse(stephenFaunus.getEdges(Direction.IN, "blah").iterator().hasNext()); edge = (FaunusEdge) stephenFaunus.getEdges(Direction.IN).iterator().next(); assertEquals(edge.getLabel(), "knows"); assertEquals(edge.getProperty("weight"), 0.2); assertEquals(edge.getProperty("type"), "coworker"); assertEquals(edge.getPropertyKeys().size(), 2); assertEquals(edge.getVertex(Direction.IN).getId(), 3l); assertEquals(edge.getVertex(Direction.OUT).getId(), 1l); }
89. Stat#writeToDataOutput()
Project: commons
File: Stat.java
File: Stat.java
/** * Write the data to the output steam so it can be streamed to an * other process, wire or storage median in a format that another Stats * object can read. * * @param out java.io.OutputStream to write to. * @throws IOException */ public void writeToDataOutput(OutputStream out) throws IOException { DataOutput dout = new DataOutputStream(out); writeToDataOutput(dout); return; }
90. SparkBatchSourceFactory#serialize()
Project: cdap
File: SparkBatchSourceFactory.java
File: SparkBatchSourceFactory.java
public void serialize(OutputStream outputStream) throws IOException { DataOutput output = new DataOutputStream(outputStream); Serializations.serializeMap(streamBatchReadables, new Serializations.ObjectWriter<StreamBatchReadable>() { @Override public void write(StreamBatchReadable streamBatchReadable, DataOutput output) throws IOException { output.writeUTF(streamBatchReadable.toURI().toString()); } }, output); Serializations.serializeMap(inputFormatProviders, new Serializations.ObjectWriter<InputFormatProvider>() { @Override public void write(InputFormatProvider inputFormatProvider, DataOutput output) throws IOException { output.writeUTF(inputFormatProvider.getInputFormatClassName()); Serializations.serializeMap(inputFormatProvider.getInputFormatConfiguration(), Serializations.createStringObjectWriter(), output); } }, output); Serializations.serializeMap(datasetInfos, new Serializations.ObjectWriter<DatasetInfo>() { @Override public void write(DatasetInfo datasetInfo, DataOutput output) throws IOException { datasetInfo.serialize(output); } }, output); Serializations.serializeMap(sourceInputs, Serializations.createStringSetObjectWriter(), output); }
91. DataOutputTest#testWrite()
Project: cassandra
File: DataOutputTest.java
File: DataOutputTest.java
private DataInput testWrite(DataOutputPlus test) throws IOException { final ByteArrayOutputStream bos = new ByteArrayOutputStream(); final DataOutput canon = new DataOutputStream(bos); Random rnd = ThreadLocalRandom.current(); int size = 50; byte[] bytes = new byte[size]; rnd.nextBytes(bytes); ByteBufferUtil.writeWithLength(bytes, test); ByteBufferUtil.writeWithLength(bytes, canon); bytes = new byte[size]; rnd.nextBytes(bytes); ByteBufferUtil.writeWithLength(wrap(bytes, false), test); ByteBufferUtil.writeWithLength(bytes, canon); bytes = new byte[size]; rnd.nextBytes(bytes); ByteBufferUtil.writeWithLength(wrap(bytes, true), test); ByteBufferUtil.writeWithLength(bytes, canon); bytes = new byte[size]; rnd.nextBytes(bytes); ByteBufferUtil.writeWithShortLength(bytes, test); ByteBufferUtil.writeWithShortLength(bytes, canon); bytes = new byte[size]; rnd.nextBytes(bytes); ByteBufferUtil.writeWithShortLength(wrap(bytes, false), test); ByteBufferUtil.writeWithShortLength(bytes, canon); bytes = new byte[size]; rnd.nextBytes(bytes); ByteBufferUtil.writeWithShortLength(wrap(bytes, true), test); ByteBufferUtil.writeWithShortLength(bytes, canon); // 318 { long v = rnd.nextLong(); test.writeLong(v); canon.writeLong(v); } { int v = rnd.nextInt(); test.writeInt(v); canon.writeInt(v); } { short v = (short) rnd.nextInt(); test.writeShort(v); canon.writeShort(v); } { byte v = (byte) rnd.nextInt(); test.write(v); canon.write(v); } { double v = rnd.nextDouble(); test.writeDouble(v); canon.writeDouble(v); } { float v = (float) rnd.nextDouble(); test.writeFloat(v); canon.writeFloat(v); } // 27 return new DataInputStream(new ByteArrayInputStream(bos.toByteArray())); }
92. DecimalOptionTest#stress_write()
Project: asakusafw
File: DecimalOptionTest.java
File: DecimalOptionTest.java
/** * heavy write. * @throws Exception if failed */ @Test public void stress_write() throws Exception { int count = 10000000; DecimalOption option = new DecimalOption(new BigDecimal("3.14")); DataOutput out = new DataOutputStream(new NullOutputStream()); for (int i = 0; i < count; i++) { option.write(out); } }