Here are the examples of the java api org.apache.avro.Schema.getType() taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.
238 Examples
18
Source : AvroRecordReader.java
with Apache License 2.0
from zpochen
with Apache License 2.0
from zpochen
private void processRecord(final GenericContainer container, final Schema schema) {
final Schema.Type type = schema.getType();
switch(type) {
case RECORD:
process(container, schema, null, new MapOrListWriterImpl(writer.rootAsMap()), fieldSelection);
break;
default:
throw new DrillRuntimeException("Root object must be record type. Found: " + type);
}
}
18
Source : AvroRowDecoderFactory.java
with Apache License 2.0
from trinodb
with Apache License 2.0
from trinodb
@Override
public RowDecoder create(Map<String, String> decoderParams, Set<DecoderColumnHandle> columns) {
requireNonNull(columns, "columns is null");
if (columns.isEmpty()) {
// For select count(*)
return DummyRowDecoderFactory.DECODER_INSTANCE;
}
String dataSchema = requireNonNull(decoderParams.get(DATA_SCHEMA), format("%s cannot be null", DATA_SCHEMA));
Schema parsedSchema = (new Schema.Parser()).parse(dataSchema);
if (parsedSchema.getType().equals(Schema.Type.RECORD)) {
AvroReaderSupplier<GenericRecord> avroReaderSupplier = avroReaderSupplierFactory.create(parsedSchema);
AvroDeserializer<GenericRecord> dataDecoder = avroDeserializerFactory.create(avroReaderSupplier);
return new GenericRecordRowDecoder(dataDecoder, columns);
} else {
AvroReaderSupplier<Object> avroReaderSupplier = avroReaderSupplierFactory.create(parsedSchema);
AvroDeserializer<Object> dataDecoder = avroDeserializerFactory.create(avroReaderSupplier);
return new SingleValueRowDecoder(dataDecoder, getOnlyElement(columns));
}
}
18
Source : AvroSchema.java
with Apache License 2.0
from Talend
with Apache License 2.0
from Talend
private Type doMapType(final Schema schema) {
switch(schema.getType()) {
case LONG:
if (Boolean.parseBoolean(readProp(schema, Type.DATETIME.name())) || LogicalTypes.timestampMillis().equals(LogicalTypes.fromSchemaIgnoreInvalid(schema))) {
return Type.DATETIME;
}
return Type.LONG;
default:
return Type.valueOf(schema.getType().name());
}
}
18
Source : AvroSchemas.java
with Apache License 2.0
from Talend
with Apache License 2.0
from Talend
public static Schema unwrapUnion(final Schema schema) {
switch(schema.getType()) {
case UNION:
return schema.getTypes().stream().filter(it -> it.getType() != NULL).findFirst().orElse(null);
default:
return schema;
}
}
18
Source : SchemaAssistant.java
with Apache License 2.0
from RTBHOUSE
with Apache License 2.0
from RTBHOUSE
/* Complex type here means type that it have to handle other types inside itself. */
public static boolean isComplexType(Schema schema) {
switch(schema.getType()) {
case MAP:
case RECORD:
case ARRAY:
case UNION:
return true;
default:
return false;
}
}
18
Source : SchemaAssistant.java
with Apache License 2.0
from RTBHOUSE
with Apache License 2.0
from RTBHOUSE
public static boolean isNamedType(Schema schema) {
switch(schema.getType()) {
case RECORD:
case ENUM:
case FIXED:
return true;
default:
return false;
}
}
18
Source : FastSerializerGeneratorBase.java
with Apache License 2.0
from RTBHOUSE
with Apache License 2.0
from RTBHOUSE
public static String getClreplacedName(Schema schema, String description) {
final Integer schemaId = Math.abs(getSchemaId(schema));
if (Schema.Type.RECORD.equals(schema.getType())) {
return schema.getName() + description + "Serializer" + "_" + schemaId;
} else if (Schema.Type.ARRAY.equals(schema.getType())) {
return "Array" + description + "Serializer" + "_" + schemaId;
} else if (Schema.Type.MAP.equals(schema.getType())) {
return "Map" + description + "Serializer" + "_" + schemaId;
}
throw new FastSerializerGeneratorException("Unsupported return type: " + schema.getType());
}
18
Source : FastDeserializerGeneratorBase.java
with Apache License 2.0
from RTBHOUSE
with Apache License 2.0
from RTBHOUSE
public static String getClreplacedName(Schema writerSchema, Schema readerSchema, String description) {
Integer writerSchemaId = Math.abs(getSchemaId(writerSchema));
Integer readerSchemaId = Math.abs(getSchemaId(readerSchema));
if (Schema.Type.RECORD.equals(readerSchema.getType())) {
return readerSchema.getName() + description + "Deserializer" + writerSchemaId + "_" + readerSchemaId;
} else if (Schema.Type.ARRAY.equals(readerSchema.getType())) {
return "Array" + description + "Deserializer" + writerSchemaId + "_" + readerSchemaId;
} else if (Schema.Type.MAP.equals(readerSchema.getType())) {
return "Map" + description + "Deserializer" + writerSchemaId + "_" + readerSchemaId;
}
throw new FastDeserializerGeneratorException("Unsupported return type: " + readerSchema.getType());
}
18
Source : RecordBuilder.java
with Apache License 2.0
from rdblue
with Apache License 2.0
from rdblue
/**
* Returns a the value as the first matching schema type or null.
*
* Note that if the value may be null even if the schema does not allow the
* value to be null.
*
* @param string a String representation of the value
* @param schema a Schema
* @return the string coerced to the correct type from the schema or null
*/
private static Object makeValue(String string, Schema schema) {
if (string == null) {
return null;
}
try {
switch(schema.getType()) {
case BOOLEAN:
return Boolean.valueOf(string);
case STRING:
return string;
case FLOAT:
return Float.valueOf(string);
case DOUBLE:
return Double.valueOf(string);
case INT:
return Integer.valueOf(string);
case LONG:
return Long.valueOf(string);
case ENUM:
// TODO: translate to enum clreplaced
if (schema.hasEnumSymbol(string)) {
return string;
} else {
try {
return schema.getEnumSymbols().get(Integer.parseInt(string));
} catch (IndexOutOfBoundsException ex) {
return null;
}
}
case UNION:
Object value = null;
for (Schema possible : schema.getTypes()) {
value = makeValue(string, possible);
if (value != null) {
return value;
}
}
return null;
case NULL:
return null;
default:
// FIXED, BYTES, MAP, ARRAY, RECORD are not supported
throw new RecordException("Unsupported field type:" + schema.getType());
}
} catch (NumberFormatException e) {
// empty string is considered null for numeric types
if (string.isEmpty()) {
return null;
} else {
throw e;
}
}
}
18
Source : AvroSchemaUtil.java
with MIT License
from orfeon
with MIT License
from orfeon
public static boolean isSqlTypeGeography(Schema schema) {
if (schema.getType().equals(Schema.Type.UNION)) {
Schema childSchema = unnestUnion(schema);
return isSqlTypeGeography(childSchema);
}
final String sqlType = schema.getProp("sqlType");
return "GEOGRAPHY".equals(sqlType);
}
18
Source : AvroSchemaUtil.java
with MIT License
from orfeon
with MIT License
from orfeon
public static boolean isSqlTypeDatetime(Schema schema) {
if (schema.getType().equals(Schema.Type.UNION)) {
Schema childSchema = unnestUnion(schema);
return isSqlTypeDatetime(childSchema);
}
final String sqlType = schema.getProp("sqlType");
return "DATETIME".equals(sqlType);
}
18
Source : AvroSchemaUtil.java
with MIT License
from mercari
with MIT License
from mercari
/**
* Check Avro {@link Schema} is sql datetime type or not.
*
* @param schema Avro Schema object.
* @return true if schema is sql datetime type.
*/
public static boolean isSqlTypeDatetime(Schema schema) {
if (schema.getType().equals(Schema.Type.UNION)) {
Schema childSchema = unnestUnion(schema);
return isSqlTypeDatetime(childSchema);
}
final String sqlType = schema.getProp("sqlType");
return "DATETIME".equals(sqlType);
}
18
Source : AvroSchemaUtil.java
with MIT License
from mercari
with MIT License
from mercari
/**
* Check Avro {@link Schema} is sql geography type or not.
*
* @param schema Avro Schema object.
* @return true if schema is sql geography type.
*/
public static boolean isSqlTypeGeography(Schema schema) {
if (schema.getType().equals(Schema.Type.UNION)) {
Schema childSchema = unnestUnion(schema);
return isSqlTypeGeography(childSchema);
}
final String sqlType = schema.getProp("sqlType");
return "GEOGRAPHY".equals(sqlType);
}
18
Source : AvroSchemaUtil.java
with MIT License
from mercari
with MIT License
from mercari
public static byte[] getBytes(final GenericRecord record, final String fieldName) {
final Schema.Field field = record.getSchema().getField(fieldName);
if (field == null) {
return null;
}
final Object value = record.get(fieldName);
if (value == null) {
return null;
}
final Schema schema = unnestUnion(field.schema());
switch(schema.getType()) {
case BYTES:
return ((ByteBuffer) value).array();
default:
return null;
}
}
18
Source : JdbcUtil.java
with MIT License
from mercari
with MIT License
from mercari
private static boolean isValidColumnType(final Schema fieldType) {
switch(fieldType.getType()) {
case MAP:
case RECORD:
return false;
case ARRAY:
if (!isValidColumnType(fieldType.getElementType())) {
return false;
}
return true;
default:
return true;
}
}
18
Source : AvroWrapper.java
with BSD 2-Clause "Simplified" License
from linkedin
with BSD 2-Clause "Simplified" License
from linkedin
public static StdData createStdData(Object avroData, Schema avroSchema) {
switch(avroSchema.getType()) {
case INT:
return new AvroInteger((Integer) avroData);
case LONG:
return new AvroLong((Long) avroData);
case BOOLEAN:
return new AvroBoolean((Boolean) avroData);
case ENUM:
{
if (avroData == null) {
return new AvroString(null);
}
if (avroData instanceof String) {
return new AvroString(new Utf8((String) avroData));
} else if (avroData instanceof GenericEnumSymbol) {
return new AvroString(new Utf8(((GenericEnumSymbol) avroData).toString()));
}
throw new IllegalArgumentException("Unsupported type for Avro enum: " + avroData.getClreplaced());
}
case STRING:
{
if (avroData == null) {
return new AvroString(null);
}
if (avroData instanceof Utf8) {
return new AvroString((Utf8) avroData);
} else if (avroData instanceof String) {
return new AvroString(new Utf8((String) avroData));
}
throw new IllegalArgumentException("Unsupported type for Avro string: " + avroData.getClreplaced());
}
case FLOAT:
return new AvroFloat((Float) avroData);
case DOUBLE:
return new AvroDouble((Double) avroData);
case BYTES:
return new AvroBinary((ByteBuffer) avroData);
case ARRAY:
return new AvroArray((GenericArray<Object>) avroData, avroSchema);
case MAP:
return new AvroMap((Map<Object, Object>) avroData, avroSchema);
case RECORD:
return new AvroStruct((GenericRecord) avroData, avroSchema);
case UNION:
{
Schema nonNullableType = getNonNullComponent(avroSchema);
if (avroData == null) {
return null;
}
return createStdData(avroData, nonNullableType);
}
case NULL:
return null;
default:
throw new RuntimeException("Unrecognized Avro Schema: " + avroSchema.getClreplaced());
}
}
18
Source : AvroWrapper.java
with BSD 2-Clause "Simplified" License
from linkedin
with BSD 2-Clause "Simplified" License
from linkedin
public static StdType createStdType(Schema avroSchema) {
switch(avroSchema.getType()) {
case INT:
return new AvroIntegerType(avroSchema);
case LONG:
return new AvroLongType(avroSchema);
case BOOLEAN:
return new AvroBooleanType(avroSchema);
case STRING:
return new AvroStringType(avroSchema);
case FLOAT:
return new AvroFloatType(avroSchema);
case DOUBLE:
return new AvroDoubleType(avroSchema);
case BYTES:
return new AvroBinaryType(avroSchema);
case ARRAY:
return new AvroArrayType(avroSchema);
case MAP:
return new AvroMapType(avroSchema);
case RECORD:
return new AvroStructType(avroSchema);
case UNION:
{
Schema nonNullableType = getNonNullComponent(avroSchema);
return createStdType(nonNullableType);
}
default:
throw new RuntimeException("Unrecognized Avro Schema: " + avroSchema.getClreplaced());
}
}
18
Source : SchemaUtilities.java
with BSD 2-Clause "Simplified" License
from linkedin
with BSD 2-Clause "Simplified" License
from linkedin
private static boolean isUnionSchemaCompatible(@Nonnull Schema leftSchema, @Nonnull Schema rightSchema, boolean strictMode) {
Preconditions.checkNotNull(leftSchema);
Preconditions.checkNotNull(rightSchema);
switch(leftSchema.getType()) {
case BOOLEAN:
case BYTES:
case DOUBLE:
case FLOAT:
case INT:
case LONG:
case STRING:
return leftSchema.getType() == rightSchema.getType();
case FIXED:
boolean isSameType = leftSchema.getType() == rightSchema.getType();
boolean isSameNamespace = Objects.equals(leftSchema.getNamespace(), rightSchema.getNamespace());
return isSameType && (!strictMode || isSameNamespace);
case ENUM:
boolean isSameEnumType = (leftSchema.getType() == rightSchema.getType());
boolean isSameSymbolSize = (leftSchema.getEnumSymbols().size() == rightSchema.getEnumSymbols().size());
boolean isSameEnumNamespace = Objects.equals(leftSchema.getNamespace(), rightSchema.getNamespace());
return isSameEnumType && isSameSymbolSize && (!strictMode || isSameEnumNamespace);
case RECORD:
return leftSchema.getType() == rightSchema.getType() && isUnionRecordSchemaCompatible(leftSchema, rightSchema, strictMode);
case MAP:
return leftSchema.getType() == rightSchema.getType() && isUnionSchemaCompatible(leftSchema.getValueType(), rightSchema.getValueType(), strictMode);
case ARRAY:
return leftSchema.getType() == rightSchema.getType() && isUnionSchemaCompatible(leftSchema.getElementType(), rightSchema.getElementType(), strictMode);
case UNION:
boolean isSameUnionType = (leftSchema.getType() == rightSchema.getType());
boolean isBothNullableType = AvroSerdeUtils.isNullableType(leftSchema) && AvroSerdeUtils.isNullableType(rightSchema);
Schema leftOtherType = AvroSerdeUtils.getOtherTypeFromNullableType(leftSchema);
Schema rightOtherType = AvroSerdeUtils.getOtherTypeFromNullableType(rightSchema);
boolean isOtherTypeUnionCompatible = isUnionSchemaCompatible(leftOtherType, rightOtherType, strictMode);
return isSameUnionType && isBothNullableType && isOtherTypeUnionCompatible;
default:
throw new IllegalArgumentException("Unsupported Avro type " + leftSchema.getType() + " in schema: " + leftSchema.toString(true));
}
}
18
Source : SchemaAssistant.java
with BSD 2-Clause "Simplified" License
from linkedin
with BSD 2-Clause "Simplified" License
from linkedin
/**
* Infer String clreplaced for the preplaceded schema.
* If the current Avro lib is avro-1.4, this function will return default String type directly.
* Otherwise, according to Avro specification, this function only supports the following property to specify
* Stringable clreplaced while using Specific deserializer:
* 1. {@link Schema.Type#STRING} : {@link #CLreplaced_PROP};
* 2. {@link Schema.Type#MAP} : {@link #KEY_CLreplaced_PROP};
*
* If the above property is absent for Specific deserializer or the current deserializer is generic,
* this function will try to look at {@link #STRING_PROP} to decide whether it should use {@link String}
* or the default String type.
* @param schema a schema who's string impl clreplaced we want to infer
* @return the string clreplaced used by given schema
*/
public JClreplaced findStringClreplaced(Schema schema) {
Schema.Type schemaType = schema.getType();
if (!schemaType.equals(Schema.Type.STRING) && !schemaType.equals(Schema.Type.MAP)) {
throw new SchemareplacedistantException("Function `findStringClreplaced` only supports schema types: " + Schema.Type.STRING + " and " + Schema.Type.MAP);
}
JClreplaced outputClreplaced = defaultStringType();
/**
* When generating fast-serializer in generic mode, those java string property doesn't have any effect.
*/
if (useGenericTypes && isForSerializer) {
return outputClreplaced;
}
if (Utils.isAbleToSupportJavaStrings()) {
String stringClreplacedProp;
if (schemaType.equals(Schema.Type.STRING)) {
stringClreplacedProp = schema.getProp(CLreplaced_PROP);
} else {
stringClreplacedProp = schema.getProp(KEY_CLreplaced_PROP);
}
if (!useGenericTypes && null != stringClreplacedProp) {
outputClreplaced = codeModel.ref(stringClreplacedProp);
extendExceptionsFromStringable(schema.getProp(CLreplaced_PROP));
} else {
String stringProp = schema.getProp(STRING_PROP);
if (null != stringProp && stringProp.equals(STRING_TYPE_STRING)) {
outputClreplaced = codeModel.ref(String.clreplaced);
}
}
if (!Utils.isAbleToSupportStringableProps() && !outputClreplaced.equals(codeModel.ref(String.clreplaced)) && !outputClreplaced.equals(codeModel.ref(Utf8.clreplaced))) {
// This case could happen in Avro 1.6 when using a non-Utf8 and non-String clreplaced (e.g. URI, etc.), in which
// case Avro 1.6 ignores the unsupported clreplaced, and fast-avro should be compatible with that behavior.
outputClreplaced = defaultStringType();
}
}
return outputClreplaced;
}
18
Source : SchemaAssistant.java
with BSD 2-Clause "Simplified" License
from linkedin
with BSD 2-Clause "Simplified" License
from linkedin
public static String getTypeName(Schema schema) {
Schema.Type schemaType = schema.getType();
if (Schema.Type.RECORD.equals(schemaType)) {
return schema.getName();
} else if (Schema.Type.ARRAY.equals(schemaType)) {
return "Array_of_" + getTypeName(schema.getElementType());
} else if (Schema.Type.MAP.equals(schemaType)) {
return "Map_of_" + getTypeName(schema.getValueType());
} else {
return schema.getType().name();
}
}
18
Source : SchemaAssistant.java
with BSD 2-Clause "Simplified" License
from linkedin
with BSD 2-Clause "Simplified" License
from linkedin
/**
* Determines if a data type is capable of reuse
*
* @param schema of the data type in question
* @return true if that data type is reusable (by mutating it), false otherwise
*/
public static boolean isCapableOfReuse(Schema schema) {
// TODO: Add special handling for Strings... {@link org.apache.avro.util.Utf8} is reusable, but {@link String} is not.
switch(schema.getType()) {
case BOOLEAN:
case DOUBLE:
case ENUM:
case FLOAT:
case INT:
case LONG:
return false;
default:
return true;
}
}
18
Source : SchemaAssistant.java
with BSD 2-Clause "Simplified" License
from linkedin
with BSD 2-Clause "Simplified" License
from linkedin
public static boolean isPrimitive(Schema schema) {
switch(schema.getType()) {
case BOOLEAN:
case DOUBLE:
case FLOAT:
case INT:
case LONG:
return true;
default:
return false;
}
}
18
Source : SchemaAssistant.java
with BSD 2-Clause "Simplified" License
from linkedin
with BSD 2-Clause "Simplified" License
from linkedin
/**
* @return true if it is valid to call both {@link Schema#getName()} and {@link GenericContainer#getSchema()} on this data type
*/
public static boolean isNamedTypeWithSchema(Schema schema) {
switch(schema.getType()) {
case RECORD:
return true;
case ENUM:
case FIXED:
/**
* This is used to avoid `getSchema` call since in Avro-1.4, `getSchema` method is not available for Enum and Fixed.
*/
return Utils.isAvro14() ? false : true;
default:
return false;
}
}
18
Source : SchemaAssistant.java
with BSD 2-Clause "Simplified" License
from linkedin
with BSD 2-Clause "Simplified" License
from linkedin
/**
* @return true if it is valid to call {@link Schema#getName()} on this data type
*/
public static boolean isNamedType(Schema schema) {
switch(schema.getType()) {
case RECORD:
case ENUM:
case FIXED:
return true;
default:
return false;
}
}
18
Source : AvroExtractor.java
with Mozilla Public License 2.0
from corunet
with Mozilla Public License 2.0
from corunet
private String extractTypeName(Schema schema) {
return schema.getType().getName();
}
18
Source : MercifulJsonConverter.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
private static Object convertJsonToAvroField(Object value, String name, Schema schema) {
if (isOptional(schema)) {
if (value == null) {
return null;
} else {
schema = getNonNull(schema);
}
} else if (value == null) {
// Always fail on null for non-nullable schemas
throw new HoodieJsonToAvroConversionException(null, name, schema);
}
JsonToAvroFieldProcessor processor = FIELD_TYPE_PROCESSORS.get(schema.getType());
if (null != processor) {
return processor.convertToAvro(value, name, schema);
}
throw new IllegalArgumentException("JsonConverter cannot handle type: " + schema.getType());
}
17
Source : SchemaMerge.java
with MIT License
from typemeta
with MIT License
from typemeta
public static Schema merge(Schema lhs, Schema rhs) {
switch(lhs.getType()) {
case STRING:
case BYTES:
case INT:
case LONG:
case FLOAT:
case DOUBLE:
case BOOLEAN:
case NULL:
if (lhs.getType() == rhs.getType()) {
return lhs;
} else {
break;
}
case ENUM:
case FIXED:
case RECORD:
if (lhs.equals(rhs)) {
return lhs;
}
break;
case ARRAY:
case MAP:
if (lhs.getValueType().equals(rhs.getValueType())) {
return lhs;
} else {
break;
}
case UNION:
final Set<Schema> subSchemas = new HashSet<>(lhs.getTypes());
if (rhs.getType() == Schema.Type.UNION) {
subSchemas.addAll(rhs.getTypes());
} else {
subSchemas.add(rhs);
}
return Schema.createUnion(new ArrayList<>(subSchemas));
default:
throw new CodecException("Unexpected schema type - " + lhs.getType());
}
return Schema.createUnion(lhs, rhs);
}
17
Source : AvroSchemaConverter.java
with Apache License 2.0
from trinodb
with Apache License 2.0
from trinodb
public List<Type> convertAvroSchema(Schema schema) {
requireNonNull(schema, "schema is null");
List<Type> types;
if (schema.getType().equals(RECORD)) {
types = convertRecordSchema(schema);
} else {
types = convertSimpleSchema(schema);
}
checkState(!types.isEmpty(), "Schema has no valid fields: '%s'", schema);
return types;
}
17
Source : FastDeserializerGenerator.java
with Apache License 2.0
from RTBHOUSE
with Apache License 2.0
from RTBHOUSE
private void processSimpleType(Schema schema, JBlock methodBody, FieldAction action, BiConsumer<JBlock, JExpression> putExpressionIntoParent) {
switch(schema.getType()) {
case ENUM:
processEnum(schema, methodBody, action, putExpressionIntoParent);
break;
case FIXED:
processFixed(schema, methodBody, action, putExpressionIntoParent);
break;
default:
processPrimitive(schema, methodBody, action, putExpressionIntoParent);
}
}
17
Source : RecordToEntityConverter.java
with MIT License
from orfeon
with MIT License
from orfeon
private static Schema.Type getType(final Schema schema) {
switch(schema.getType()) {
case UNION:
Schema childSchema = schema.getTypes().stream().filter(s -> !s.getType().equals(Schema.Type.NULL)).findAny().orElseThrow(() -> new IllegalArgumentException("UNION does not have another schema."));
return getType(childSchema);
default:
return schema.getType();
}
}
17
Source : AvroSchemaUtil.java
with MIT License
from orfeon
with MIT License
from orfeon
public static Schema unnestUnion(Schema schema) {
if (schema.getType().equals(Schema.Type.UNION)) {
return schema.getTypes().stream().filter(s -> !s.getType().equals(Schema.Type.NULL)).findAny().orElseThrow(() -> new IllegalArgumentException("UNION does not have another schema."));
}
return schema;
}
17
Source : AvroSchemaUtil.java
with MIT License
from orfeon
with MIT License
from orfeon
public static LogicalTypes.Decimal getLogicalTypeDecimal(Schema schema) {
if (schema.getType().equals(Schema.Type.UNION)) {
Schema childSchema = unnestUnion(schema);
return getLogicalTypeDecimal(childSchema);
}
final int precision = schema.getObjectProp("precision") != null ? Integer.valueOf(schema.getObjectProp("precision").toString()) : 0;
final int scale = schema.getObjectProp("scale") != null ? Integer.valueOf(schema.getObjectProp("scale").toString()) : 0;
return LogicalTypes.decimal(precision, scale);
}
17
Source : SpannerTablePrepareDoFn.java
with MIT License
from orfeon
with MIT License
from orfeon
private static boolean isValidColumnType(final Schema schema) {
switch(schema.getType()) {
case RECORD:
case MAP:
case NULL:
return false;
case ARRAY:
return isValidColumnType(schema.getElementType());
case UNION:
final Schema childSchema = schema.getTypes().stream().filter(s -> !s.getType().equals(Schema.Type.NULL)).findAny().orElseThrow(() -> new IllegalArgumentException("UNION does not have another schema."));
return isValidColumnType(childSchema);
default:
return true;
}
}
17
Source : AvroLoader.java
with Apache License 2.0
from nhl
with Apache License 2.0
from nhl
protected Acreplacedulator<?> mapColumn(Schema columnSchema) {
switch(columnSchema.getType()) {
// Raw numeric and boolean types can be loaded as primitives,
// as numeric nullable types are declared as unions and will fall under the "default" case
case INT:
return new IntAcreplacedulator();
case DOUBLE:
return new DoubleAcreplacedulator();
case LONG:
return new LongAcreplacedulator();
case BOOLEAN:
return new BooleanAcreplacedulator();
case STRING:
case BYTES:
case ENUM:
case NULL:
return new ObjectAcreplacedulator<>();
case UNION:
return mapUnionColumn(columnSchema.getTypes());
default:
throw new UnsupportedOperationException("(Yet) unsupported Avro schema type: " + columnSchema.getType());
}
}
17
Source : AvroSchemaUtil.java
with MIT License
from mercari
with MIT License
from mercari
/**
* Extract logical type decimal from {@link Schema}.
*
* @param schema Avro Schema object.
* @return true if schema is logical decimal type.
*/
public static LogicalTypes.Decimal getLogicalTypeDecimal(Schema schema) {
if (schema.getType().equals(Schema.Type.UNION)) {
Schema childSchema = unnestUnion(schema);
return getLogicalTypeDecimal(childSchema);
}
final int precision = schema.getObjectProp("precision") != null ? Integer.valueOf(schema.getObjectProp("precision").toString()) : 0;
final int scale = schema.getObjectProp("scale") != null ? Integer.valueOf(schema.getObjectProp("scale").toString()) : 0;
return LogicalTypes.decimal(precision, scale);
}
17
Source : AvroSchemaUtil.java
with MIT License
from mercari
with MIT License
from mercari
/**
* Extract child Avro schema from nullable union Avro {@link Schema}.
*
* @param schema Avro Schema object.
* @return Child Avro schema or input schema if not union schema.
*/
public static Schema unnestUnion(Schema schema) {
if (schema.getType().equals(Schema.Type.UNION)) {
return schema.getTypes().stream().filter(s -> !s.getType().equals(Schema.Type.NULL)).findAny().orElseThrow(() -> new IllegalArgumentException("UNION does not have another schema."));
}
return schema;
}
17
Source : AvroSchemaUtil.java
with MIT License
from mercari
with MIT License
from mercari
public static boolean isNullable(final Schema schema) {
if (schema.getType().equals(Schema.Type.UNION)) {
return schema.getTypes().stream().map(Schema::getType).anyMatch(Schema.Type.NULL::equals);
}
return false;
}
17
Source : ResolvingGrammarGenerator.java
with BSD 2-Clause "Simplified" License
from linkedin
with BSD 2-Clause "Simplified" License
from linkedin
private static int bestBranch(Schema r, Schema w) {
Schema.Type vt = w.getType();
// first scan for exact match
int j = 0;
for (Schema b : r.getTypes()) {
if (vt == b.getType())
if (vt == Schema.Type.RECORD) {
String vname = w.getName();
if (vname == null || vname.equals(b.getName()))
return j;
} else
return j;
j++;
}
// then scan match via numeric promotion
j = 0;
for (Schema b : r.getTypes()) {
switch(vt) {
case INT:
switch(b.getType()) {
case LONG:
case DOUBLE:
return j;
}
break;
case LONG:
case FLOAT:
switch(b.getType()) {
case DOUBLE:
return j;
}
break;
}
j++;
}
return -1;
}
17
Source : AvroSchemaUtil.java
with BSD 2-Clause "Simplified" License
from linkedin
with BSD 2-Clause "Simplified" License
from linkedin
private static void traverseSchema(Schema schema, SchemaVisitor visitor, IdenreplacedyHashMap<Object, Boolean> visited) {
if (visited.put(schema, Boolean.TRUE) != null) {
// been there, done that
return;
}
visitor.visitSchema(schema);
switch(schema.getType()) {
case UNION:
for (Schema unionBranch : schema.getTypes()) {
traverseSchema(unionBranch, visitor, visited);
}
return;
case ARRAY:
traverseSchema(schema.getElementType(), visitor, visited);
return;
case MAP:
traverseSchema(schema.getValueType(), visitor, visited);
return;
case RECORD:
for (Schema.Field field : schema.getFields()) {
visitor.visitField(schema, field);
traverseSchema(field.schema(), visitor, visited);
}
break;
default:
}
}
17
Source : RecordBuilderBase.java
with BSD 2-Clause "Simplified" License
from linkedin
with BSD 2-Clause "Simplified" License
from linkedin
protected static boolean isValidValue(Field f, Object value) {
if (value != null) {
return true;
} else {
Schema schema = f.schema();
Type type = schema.getType();
if (type == Type.NULL) {
return true;
} else {
if (type == Type.UNION) {
Iterator i$ = schema.getTypes().iterator();
while (i$.hasNext()) {
Schema s = (Schema) i$.next();
if (s.getType() == Type.NULL) {
return true;
}
}
}
return false;
}
}
}
17
Source : AvroRandomDataGenerator.java
with BSD 2-Clause "Simplified" License
from linkedin
with BSD 2-Clause "Simplified" License
from linkedin
private Object generateObject(Schema schema, Map propertiesProp) {
switch(schema.getType()) {
case ARRAY:
return generateArray(schema, propertiesProp);
case BOOLEAN:
return generateBoolean(propertiesProp);
case BYTES:
return generateBytes(propertiesProp);
case DOUBLE:
return generateDouble(propertiesProp);
case ENUM:
return generateEnumSymbol(schema);
case FIXED:
return generateFixed(schema);
case FLOAT:
return generateFloat(propertiesProp);
case INT:
return generateInt(propertiesProp);
case LONG:
return generateLong(propertiesProp);
case MAP:
return generateMap(schema, propertiesProp);
case NULL:
return generateNull();
case RECORD:
return generateRecord(schema, propertiesProp);
case STRING:
return generateString(propertiesProp);
case UNION:
return generateUnion(schema, propertiesProp);
default:
throw new RuntimeException("Unrecognized schema type: " + schema.getType());
}
}
17
Source : AvroSchemaRegistrySqsAsyncClient.java
with MIT License
from JaidenAshmore
with MIT License
from JaidenAshmore
private Map<Clreplaced<?>, RegisteredSchema> parseAvailableSchemas(final List<Resource> schemaImports, final List<Resource> schemaLocations) {
final Schema.Parser schemaParser = new Schema.Parser();
return Stream.of(schemaImports, schemaLocations).filter(arr -> !ObjectUtils.isEmpty(arr)).distinct().flatMap(List::stream).flatMap(resource -> {
final Schema schema;
try {
schema = schemaParser.parse(resource.getInputStream());
} catch (IOException ioException) {
throw new RuntimeException("Error parsing schema: " + resource.getFilename(), ioException);
}
if (schema.getType().equals(Schema.Type.UNION)) {
return schema.getTypes().stream();
} else {
return Stream.of(schema);
}
}).map(this::createObjectMapping).collect(toMap(Map.Entry::getKey, Map.Entry::getValue));
}
17
Source : DefaultAvroSerDesHandler.java
with Apache License 2.0
from hortonworks
with Apache License 2.0
from hortonworks
@Override
public Object handlePayloadDeserialization(InputStream payloadInputStream, Schema writerSchema, Schema readerSchema, boolean useSpecificAvroReader) {
Object deserializedObj;
Schema.Type writerSchemaType = writerSchema.getType();
try {
if (Schema.Type.BYTES.equals(writerSchemaType)) {
// serializer writes byte array directly without going through avro encoder layers.
deserializedObj = IOUtils.toByteArray(payloadInputStream);
} else if (Schema.Type.STRING.equals(writerSchemaType)) {
// generate UTF-8 string object from the received bytes.
deserializedObj = new String(IOUtils.toByteArray(payloadInputStream), AvroUtils.UTF_8);
} else {
DatumReader datumReader = getDatumReader(writerSchema, readerSchema, useSpecificAvroReader);
deserializedObj = datumReader.read(null, DecoderFactory.get().binaryDecoder(payloadInputStream, null));
}
} catch (IOException e) {
throw new AvroRetryableException(e);
} catch (Exception e) {
throw new AvroException(e);
}
return deserializedObj;
}
17
Source : ConfluentAvroSerDesHandler.java
with Apache License 2.0
from hortonworks
with Apache License 2.0
from hortonworks
@Override
public Object handlePayloadDeserialization(InputStream payloadInputStream, Schema writerSchema, Schema readerSchema, boolean useSpecificAvroReader) {
Object deserializedObj;
try {
if (Schema.Type.BYTES.equals(writerSchema.getType())) {
// serializer writes byte array directly without going through avro encoder layers.
deserializedObj = IOUtils.toByteArray(payloadInputStream);
} else {
DatumReader datumReader = getDatumReader(writerSchema, readerSchema, useSpecificAvroReader);
deserializedObj = datumReader.read(null, DecoderFactory.get().binaryDecoder(payloadInputStream, null));
}
} catch (IOException e) {
throw new AvroRetryableException("Error deserializing Avro message for id " + writerSchema, e);
} catch (RuntimeException e) {
// avro deserialization may throw AvroRuntimeException, NullPointerException, etc
throw new AvroException("Error deserializing Avro message for id " + writerSchema, e);
}
return deserializedObj;
}
17
Source : AvroFieldsGenerator.java
with Apache License 2.0
from hortonworks
with Apache License 2.0
from hortonworks
private void parseField(Schema.Field field, List<SchemaFieldInfo> schemaFieldInfos, Set<String> visitedRecords) {
Schema schema = field.schema();
Schema.Type type = schema.getType();
String name = field.name();
LOG.debug("Visiting field: [{}]", field);
String namespace = null;
try {
namespace = schema.getNamespace();
} catch (Exception e) {
// ignore.
}
schemaFieldInfos.add(new SchemaFieldInfo(namespace, name, type.name()));
// todo check whether fields should be mapped to the root schema.
parseSchema(schema, schemaFieldInfos, visitedRecords);
}
17
Source : AvroFileAccessorTest.java
with Apache License 2.0
from greenplum-db
with Apache License 2.0
from greenplum-db
/**
* Helper method for testing schema
*
* @param schema the schema
* @param name the name
*/
private static void verifySchema(Schema schema, String name) {
replacedertNotNull(schema);
replacedertEquals(Schema.Type.RECORD, schema.getType());
replacedertEquals(name, schema.getName());
Map<String, String> fieldToType = new HashMap<String, String>() {
{
put("id", "long");
put("username", "string");
put("followers", "array");
}
};
for (String key : fieldToType.keySet()) {
replacedertEquals(fieldToType.get(key), schema.getField(key).schema().getType().getName());
}
}
17
Source : AvroUtilitiesTest.java
with Apache License 2.0
from greenplum-db
with Apache License 2.0
from greenplum-db
/**
* Helper method for testing generated schema
*
* @param schema the schema
*/
private static void verifyGeneratedSchema(Schema schema) {
replacedertNotNull(schema);
replacedertEquals(schema.getType(), Schema.Type.RECORD);
Map<String, String> fieldToType = new HashMap<String, String>() {
{
put("id", "union");
put("username", "union");
put("followers", "union");
}
};
Map<String, String> unionToInnerType = new HashMap<String, String>() {
{
put("id", "long");
put("username", "string");
// arrays become strings
put("followers", "string");
}
};
for (String key : fieldToType.keySet()) {
replacedertEquals(fieldToType.get(key), schema.getField(key).schema().getType().getName());
// check the union's inner types
replacedertEquals("null", schema.getField(key).schema().getTypes().get(0).getName());
replacedertEquals(unionToInnerType.get(key), schema.getField(key).schema().getTypes().get(1).getName());
}
}
17
Source : AvroExtractor.java
with Mozilla Public License 2.0
from corunet
with Mozilla Public License 2.0
from corunet
private boolean checkIfMap(Schema innerSchema) {
return MAP.equals(innerSchema.getType());
}
17
Source : AvroExtractor.java
with Mozilla Public License 2.0
from corunet
with Mozilla Public License 2.0
from corunet
private boolean checkIfRecord(Schema innerSchema) {
return RECORD.equals(innerSchema.getType());
}
17
Source : ValueReaders.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
public static ValueReader<byte[]> decimalBytesReader(Schema schema) {
switch(schema.getType()) {
case FIXED:
return ValueReaders.fixed(schema.getFixedSize());
case BYTES:
return ValueReaders.bytes();
default:
throw new IllegalArgumentException("Invalid primitive type for decimal: " + schema.getType());
}
}
See More Examples