org.apache.avro.Schema.getType()

Here are the examples of the java api org.apache.avro.Schema.getType() taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.

238 Examples 7

18 Source : AvroRecordReader.java
with Apache License 2.0
from zpochen

private void processRecord(final GenericContainer container, final Schema schema) {
    final Schema.Type type = schema.getType();
    switch(type) {
        case RECORD:
            process(container, schema, null, new MapOrListWriterImpl(writer.rootAsMap()), fieldSelection);
            break;
        default:
            throw new DrillRuntimeException("Root object must be record type. Found: " + type);
    }
}

18 Source : AvroRowDecoderFactory.java
with Apache License 2.0
from trinodb

@Override
public RowDecoder create(Map<String, String> decoderParams, Set<DecoderColumnHandle> columns) {
    requireNonNull(columns, "columns is null");
    if (columns.isEmpty()) {
        // For select count(*)
        return DummyRowDecoderFactory.DECODER_INSTANCE;
    }
    String dataSchema = requireNonNull(decoderParams.get(DATA_SCHEMA), format("%s cannot be null", DATA_SCHEMA));
    Schema parsedSchema = (new Schema.Parser()).parse(dataSchema);
    if (parsedSchema.getType().equals(Schema.Type.RECORD)) {
        AvroReaderSupplier<GenericRecord> avroReaderSupplier = avroReaderSupplierFactory.create(parsedSchema);
        AvroDeserializer<GenericRecord> dataDecoder = avroDeserializerFactory.create(avroReaderSupplier);
        return new GenericRecordRowDecoder(dataDecoder, columns);
    } else {
        AvroReaderSupplier<Object> avroReaderSupplier = avroReaderSupplierFactory.create(parsedSchema);
        AvroDeserializer<Object> dataDecoder = avroDeserializerFactory.create(avroReaderSupplier);
        return new SingleValueRowDecoder(dataDecoder, getOnlyElement(columns));
    }
}

18 Source : AvroSchema.java
with Apache License 2.0
from Talend

private Type doMapType(final Schema schema) {
    switch(schema.getType()) {
        case LONG:
            if (Boolean.parseBoolean(readProp(schema, Type.DATETIME.name())) || LogicalTypes.timestampMillis().equals(LogicalTypes.fromSchemaIgnoreInvalid(schema))) {
                return Type.DATETIME;
            }
            return Type.LONG;
        default:
            return Type.valueOf(schema.getType().name());
    }
}

18 Source : AvroSchemas.java
with Apache License 2.0
from Talend

public static Schema unwrapUnion(final Schema schema) {
    switch(schema.getType()) {
        case UNION:
            return schema.getTypes().stream().filter(it -> it.getType() != NULL).findFirst().orElse(null);
        default:
            return schema;
    }
}

18 Source : SchemaAssistant.java
with Apache License 2.0
from RTBHOUSE

/* Complex type here means type that it have to handle other types inside itself. */
public static boolean isComplexType(Schema schema) {
    switch(schema.getType()) {
        case MAP:
        case RECORD:
        case ARRAY:
        case UNION:
            return true;
        default:
            return false;
    }
}

18 Source : SchemaAssistant.java
with Apache License 2.0
from RTBHOUSE

public static boolean isNamedType(Schema schema) {
    switch(schema.getType()) {
        case RECORD:
        case ENUM:
        case FIXED:
            return true;
        default:
            return false;
    }
}

18 Source : FastSerializerGeneratorBase.java
with Apache License 2.0
from RTBHOUSE

public static String getClreplacedName(Schema schema, String description) {
    final Integer schemaId = Math.abs(getSchemaId(schema));
    if (Schema.Type.RECORD.equals(schema.getType())) {
        return schema.getName() + description + "Serializer" + "_" + schemaId;
    } else if (Schema.Type.ARRAY.equals(schema.getType())) {
        return "Array" + description + "Serializer" + "_" + schemaId;
    } else if (Schema.Type.MAP.equals(schema.getType())) {
        return "Map" + description + "Serializer" + "_" + schemaId;
    }
    throw new FastSerializerGeneratorException("Unsupported return type: " + schema.getType());
}

18 Source : FastDeserializerGeneratorBase.java
with Apache License 2.0
from RTBHOUSE

public static String getClreplacedName(Schema writerSchema, Schema readerSchema, String description) {
    Integer writerSchemaId = Math.abs(getSchemaId(writerSchema));
    Integer readerSchemaId = Math.abs(getSchemaId(readerSchema));
    if (Schema.Type.RECORD.equals(readerSchema.getType())) {
        return readerSchema.getName() + description + "Deserializer" + writerSchemaId + "_" + readerSchemaId;
    } else if (Schema.Type.ARRAY.equals(readerSchema.getType())) {
        return "Array" + description + "Deserializer" + writerSchemaId + "_" + readerSchemaId;
    } else if (Schema.Type.MAP.equals(readerSchema.getType())) {
        return "Map" + description + "Deserializer" + writerSchemaId + "_" + readerSchemaId;
    }
    throw new FastDeserializerGeneratorException("Unsupported return type: " + readerSchema.getType());
}

18 Source : RecordBuilder.java
with Apache License 2.0
from rdblue

/**
 * Returns a the value as the first matching schema type or null.
 *
 * Note that if the value may be null even if the schema does not allow the
 * value to be null.
 *
 * @param string a String representation of the value
 * @param schema a Schema
 * @return the string coerced to the correct type from the schema or null
 */
private static Object makeValue(String string, Schema schema) {
    if (string == null) {
        return null;
    }
    try {
        switch(schema.getType()) {
            case BOOLEAN:
                return Boolean.valueOf(string);
            case STRING:
                return string;
            case FLOAT:
                return Float.valueOf(string);
            case DOUBLE:
                return Double.valueOf(string);
            case INT:
                return Integer.valueOf(string);
            case LONG:
                return Long.valueOf(string);
            case ENUM:
                // TODO: translate to enum clreplaced
                if (schema.hasEnumSymbol(string)) {
                    return string;
                } else {
                    try {
                        return schema.getEnumSymbols().get(Integer.parseInt(string));
                    } catch (IndexOutOfBoundsException ex) {
                        return null;
                    }
                }
            case UNION:
                Object value = null;
                for (Schema possible : schema.getTypes()) {
                    value = makeValue(string, possible);
                    if (value != null) {
                        return value;
                    }
                }
                return null;
            case NULL:
                return null;
            default:
                // FIXED, BYTES, MAP, ARRAY, RECORD are not supported
                throw new RecordException("Unsupported field type:" + schema.getType());
        }
    } catch (NumberFormatException e) {
        // empty string is considered null for numeric types
        if (string.isEmpty()) {
            return null;
        } else {
            throw e;
        }
    }
}

18 Source : AvroSchemaUtil.java
with MIT License
from orfeon

public static boolean isSqlTypeGeography(Schema schema) {
    if (schema.getType().equals(Schema.Type.UNION)) {
        Schema childSchema = unnestUnion(schema);
        return isSqlTypeGeography(childSchema);
    }
    final String sqlType = schema.getProp("sqlType");
    return "GEOGRAPHY".equals(sqlType);
}

18 Source : AvroSchemaUtil.java
with MIT License
from orfeon

public static boolean isSqlTypeDatetime(Schema schema) {
    if (schema.getType().equals(Schema.Type.UNION)) {
        Schema childSchema = unnestUnion(schema);
        return isSqlTypeDatetime(childSchema);
    }
    final String sqlType = schema.getProp("sqlType");
    return "DATETIME".equals(sqlType);
}

18 Source : AvroSchemaUtil.java
with MIT License
from mercari

/**
 * Check Avro {@link Schema} is sql datetime type or not.
 *
 * @param schema Avro Schema object.
 * @return true if schema is sql datetime type.
 */
public static boolean isSqlTypeDatetime(Schema schema) {
    if (schema.getType().equals(Schema.Type.UNION)) {
        Schema childSchema = unnestUnion(schema);
        return isSqlTypeDatetime(childSchema);
    }
    final String sqlType = schema.getProp("sqlType");
    return "DATETIME".equals(sqlType);
}

18 Source : AvroSchemaUtil.java
with MIT License
from mercari

/**
 * Check Avro {@link Schema} is sql geography type or not.
 *
 * @param schema Avro Schema object.
 * @return true if schema is sql geography type.
 */
public static boolean isSqlTypeGeography(Schema schema) {
    if (schema.getType().equals(Schema.Type.UNION)) {
        Schema childSchema = unnestUnion(schema);
        return isSqlTypeGeography(childSchema);
    }
    final String sqlType = schema.getProp("sqlType");
    return "GEOGRAPHY".equals(sqlType);
}

18 Source : AvroSchemaUtil.java
with MIT License
from mercari

public static byte[] getBytes(final GenericRecord record, final String fieldName) {
    final Schema.Field field = record.getSchema().getField(fieldName);
    if (field == null) {
        return null;
    }
    final Object value = record.get(fieldName);
    if (value == null) {
        return null;
    }
    final Schema schema = unnestUnion(field.schema());
    switch(schema.getType()) {
        case BYTES:
            return ((ByteBuffer) value).array();
        default:
            return null;
    }
}

18 Source : JdbcUtil.java
with MIT License
from mercari

private static boolean isValidColumnType(final Schema fieldType) {
    switch(fieldType.getType()) {
        case MAP:
        case RECORD:
            return false;
        case ARRAY:
            if (!isValidColumnType(fieldType.getElementType())) {
                return false;
            }
            return true;
        default:
            return true;
    }
}

18 Source : AvroWrapper.java
with BSD 2-Clause "Simplified" License
from linkedin

public static StdData createStdData(Object avroData, Schema avroSchema) {
    switch(avroSchema.getType()) {
        case INT:
            return new AvroInteger((Integer) avroData);
        case LONG:
            return new AvroLong((Long) avroData);
        case BOOLEAN:
            return new AvroBoolean((Boolean) avroData);
        case ENUM:
            {
                if (avroData == null) {
                    return new AvroString(null);
                }
                if (avroData instanceof String) {
                    return new AvroString(new Utf8((String) avroData));
                } else if (avroData instanceof GenericEnumSymbol) {
                    return new AvroString(new Utf8(((GenericEnumSymbol) avroData).toString()));
                }
                throw new IllegalArgumentException("Unsupported type for Avro enum: " + avroData.getClreplaced());
            }
        case STRING:
            {
                if (avroData == null) {
                    return new AvroString(null);
                }
                if (avroData instanceof Utf8) {
                    return new AvroString((Utf8) avroData);
                } else if (avroData instanceof String) {
                    return new AvroString(new Utf8((String) avroData));
                }
                throw new IllegalArgumentException("Unsupported type for Avro string: " + avroData.getClreplaced());
            }
        case FLOAT:
            return new AvroFloat((Float) avroData);
        case DOUBLE:
            return new AvroDouble((Double) avroData);
        case BYTES:
            return new AvroBinary((ByteBuffer) avroData);
        case ARRAY:
            return new AvroArray((GenericArray<Object>) avroData, avroSchema);
        case MAP:
            return new AvroMap((Map<Object, Object>) avroData, avroSchema);
        case RECORD:
            return new AvroStruct((GenericRecord) avroData, avroSchema);
        case UNION:
            {
                Schema nonNullableType = getNonNullComponent(avroSchema);
                if (avroData == null) {
                    return null;
                }
                return createStdData(avroData, nonNullableType);
            }
        case NULL:
            return null;
        default:
            throw new RuntimeException("Unrecognized Avro Schema: " + avroSchema.getClreplaced());
    }
}

18 Source : AvroWrapper.java
with BSD 2-Clause "Simplified" License
from linkedin

public static StdType createStdType(Schema avroSchema) {
    switch(avroSchema.getType()) {
        case INT:
            return new AvroIntegerType(avroSchema);
        case LONG:
            return new AvroLongType(avroSchema);
        case BOOLEAN:
            return new AvroBooleanType(avroSchema);
        case STRING:
            return new AvroStringType(avroSchema);
        case FLOAT:
            return new AvroFloatType(avroSchema);
        case DOUBLE:
            return new AvroDoubleType(avroSchema);
        case BYTES:
            return new AvroBinaryType(avroSchema);
        case ARRAY:
            return new AvroArrayType(avroSchema);
        case MAP:
            return new AvroMapType(avroSchema);
        case RECORD:
            return new AvroStructType(avroSchema);
        case UNION:
            {
                Schema nonNullableType = getNonNullComponent(avroSchema);
                return createStdType(nonNullableType);
            }
        default:
            throw new RuntimeException("Unrecognized Avro Schema: " + avroSchema.getClreplaced());
    }
}

18 Source : SchemaUtilities.java
with BSD 2-Clause "Simplified" License
from linkedin

private static boolean isUnionSchemaCompatible(@Nonnull Schema leftSchema, @Nonnull Schema rightSchema, boolean strictMode) {
    Preconditions.checkNotNull(leftSchema);
    Preconditions.checkNotNull(rightSchema);
    switch(leftSchema.getType()) {
        case BOOLEAN:
        case BYTES:
        case DOUBLE:
        case FLOAT:
        case INT:
        case LONG:
        case STRING:
            return leftSchema.getType() == rightSchema.getType();
        case FIXED:
            boolean isSameType = leftSchema.getType() == rightSchema.getType();
            boolean isSameNamespace = Objects.equals(leftSchema.getNamespace(), rightSchema.getNamespace());
            return isSameType && (!strictMode || isSameNamespace);
        case ENUM:
            boolean isSameEnumType = (leftSchema.getType() == rightSchema.getType());
            boolean isSameSymbolSize = (leftSchema.getEnumSymbols().size() == rightSchema.getEnumSymbols().size());
            boolean isSameEnumNamespace = Objects.equals(leftSchema.getNamespace(), rightSchema.getNamespace());
            return isSameEnumType && isSameSymbolSize && (!strictMode || isSameEnumNamespace);
        case RECORD:
            return leftSchema.getType() == rightSchema.getType() && isUnionRecordSchemaCompatible(leftSchema, rightSchema, strictMode);
        case MAP:
            return leftSchema.getType() == rightSchema.getType() && isUnionSchemaCompatible(leftSchema.getValueType(), rightSchema.getValueType(), strictMode);
        case ARRAY:
            return leftSchema.getType() == rightSchema.getType() && isUnionSchemaCompatible(leftSchema.getElementType(), rightSchema.getElementType(), strictMode);
        case UNION:
            boolean isSameUnionType = (leftSchema.getType() == rightSchema.getType());
            boolean isBothNullableType = AvroSerdeUtils.isNullableType(leftSchema) && AvroSerdeUtils.isNullableType(rightSchema);
            Schema leftOtherType = AvroSerdeUtils.getOtherTypeFromNullableType(leftSchema);
            Schema rightOtherType = AvroSerdeUtils.getOtherTypeFromNullableType(rightSchema);
            boolean isOtherTypeUnionCompatible = isUnionSchemaCompatible(leftOtherType, rightOtherType, strictMode);
            return isSameUnionType && isBothNullableType && isOtherTypeUnionCompatible;
        default:
            throw new IllegalArgumentException("Unsupported Avro type " + leftSchema.getType() + " in schema: " + leftSchema.toString(true));
    }
}

18 Source : SchemaAssistant.java
with BSD 2-Clause "Simplified" License
from linkedin

/**
 * Infer String clreplaced for the preplaceded schema.
 * If the current Avro lib is avro-1.4, this function will return default String type directly.
 * Otherwise, according to Avro specification, this function only supports the following property to specify
 * Stringable clreplaced while using Specific deserializer:
 * 1. {@link Schema.Type#STRING} : {@link #CLreplaced_PROP};
 * 2. {@link Schema.Type#MAP} :  {@link #KEY_CLreplaced_PROP};
 *
 * If the above property is absent for Specific deserializer or the current deserializer is generic,
 * this function will try to look at {@link #STRING_PROP} to decide whether it should use {@link String}
 * or the default String type.
 * @param schema a schema who's string impl clreplaced we want to infer
 * @return the string clreplaced used by given schema
 */
public JClreplaced findStringClreplaced(Schema schema) {
    Schema.Type schemaType = schema.getType();
    if (!schemaType.equals(Schema.Type.STRING) && !schemaType.equals(Schema.Type.MAP)) {
        throw new SchemareplacedistantException("Function `findStringClreplaced` only supports schema types: " + Schema.Type.STRING + " and " + Schema.Type.MAP);
    }
    JClreplaced outputClreplaced = defaultStringType();
    /**
     * When generating fast-serializer in generic mode, those java string property doesn't have any effect.
     */
    if (useGenericTypes && isForSerializer) {
        return outputClreplaced;
    }
    if (Utils.isAbleToSupportJavaStrings()) {
        String stringClreplacedProp;
        if (schemaType.equals(Schema.Type.STRING)) {
            stringClreplacedProp = schema.getProp(CLreplaced_PROP);
        } else {
            stringClreplacedProp = schema.getProp(KEY_CLreplaced_PROP);
        }
        if (!useGenericTypes && null != stringClreplacedProp) {
            outputClreplaced = codeModel.ref(stringClreplacedProp);
            extendExceptionsFromStringable(schema.getProp(CLreplaced_PROP));
        } else {
            String stringProp = schema.getProp(STRING_PROP);
            if (null != stringProp && stringProp.equals(STRING_TYPE_STRING)) {
                outputClreplaced = codeModel.ref(String.clreplaced);
            }
        }
        if (!Utils.isAbleToSupportStringableProps() && !outputClreplaced.equals(codeModel.ref(String.clreplaced)) && !outputClreplaced.equals(codeModel.ref(Utf8.clreplaced))) {
            // This case could happen in Avro 1.6 when using a non-Utf8 and non-String clreplaced (e.g. URI, etc.), in which
            // case  Avro 1.6 ignores the unsupported clreplaced, and fast-avro should be compatible with that behavior.
            outputClreplaced = defaultStringType();
        }
    }
    return outputClreplaced;
}

18 Source : SchemaAssistant.java
with BSD 2-Clause "Simplified" License
from linkedin

public static String getTypeName(Schema schema) {
    Schema.Type schemaType = schema.getType();
    if (Schema.Type.RECORD.equals(schemaType)) {
        return schema.getName();
    } else if (Schema.Type.ARRAY.equals(schemaType)) {
        return "Array_of_" + getTypeName(schema.getElementType());
    } else if (Schema.Type.MAP.equals(schemaType)) {
        return "Map_of_" + getTypeName(schema.getValueType());
    } else {
        return schema.getType().name();
    }
}

18 Source : SchemaAssistant.java
with BSD 2-Clause "Simplified" License
from linkedin

/**
 * Determines if a data type is capable of reuse
 *
 * @param schema of the data type in question
 * @return true if that data type is reusable (by mutating it), false otherwise
 */
public static boolean isCapableOfReuse(Schema schema) {
    // TODO: Add special handling for Strings... {@link org.apache.avro.util.Utf8} is reusable, but {@link String} is not.
    switch(schema.getType()) {
        case BOOLEAN:
        case DOUBLE:
        case ENUM:
        case FLOAT:
        case INT:
        case LONG:
            return false;
        default:
            return true;
    }
}

18 Source : SchemaAssistant.java
with BSD 2-Clause "Simplified" License
from linkedin

public static boolean isPrimitive(Schema schema) {
    switch(schema.getType()) {
        case BOOLEAN:
        case DOUBLE:
        case FLOAT:
        case INT:
        case LONG:
            return true;
        default:
            return false;
    }
}

18 Source : SchemaAssistant.java
with BSD 2-Clause "Simplified" License
from linkedin

/**
 * @return true if it is valid to call both {@link Schema#getName()} and {@link GenericContainer#getSchema()} on this data type
 */
public static boolean isNamedTypeWithSchema(Schema schema) {
    switch(schema.getType()) {
        case RECORD:
            return true;
        case ENUM:
        case FIXED:
            /**
             * This is used to avoid `getSchema` call since in Avro-1.4, `getSchema` method is not available for Enum and Fixed.
             */
            return Utils.isAvro14() ? false : true;
        default:
            return false;
    }
}

18 Source : SchemaAssistant.java
with BSD 2-Clause "Simplified" License
from linkedin

/**
 * @return true if it is valid to call {@link Schema#getName()} on this data type
 */
public static boolean isNamedType(Schema schema) {
    switch(schema.getType()) {
        case RECORD:
        case ENUM:
        case FIXED:
            return true;
        default:
            return false;
    }
}

18 Source : AvroExtractor.java
with Mozilla Public License 2.0
from corunet

private String extractTypeName(Schema schema) {
    return schema.getType().getName();
}

18 Source : MercifulJsonConverter.java
with Apache License 2.0
from apache

private static Object convertJsonToAvroField(Object value, String name, Schema schema) {
    if (isOptional(schema)) {
        if (value == null) {
            return null;
        } else {
            schema = getNonNull(schema);
        }
    } else if (value == null) {
        // Always fail on null for non-nullable schemas
        throw new HoodieJsonToAvroConversionException(null, name, schema);
    }
    JsonToAvroFieldProcessor processor = FIELD_TYPE_PROCESSORS.get(schema.getType());
    if (null != processor) {
        return processor.convertToAvro(value, name, schema);
    }
    throw new IllegalArgumentException("JsonConverter cannot handle type: " + schema.getType());
}

17 Source : SchemaMerge.java
with MIT License
from typemeta

public static Schema merge(Schema lhs, Schema rhs) {
    switch(lhs.getType()) {
        case STRING:
        case BYTES:
        case INT:
        case LONG:
        case FLOAT:
        case DOUBLE:
        case BOOLEAN:
        case NULL:
            if (lhs.getType() == rhs.getType()) {
                return lhs;
            } else {
                break;
            }
        case ENUM:
        case FIXED:
        case RECORD:
            if (lhs.equals(rhs)) {
                return lhs;
            }
            break;
        case ARRAY:
        case MAP:
            if (lhs.getValueType().equals(rhs.getValueType())) {
                return lhs;
            } else {
                break;
            }
        case UNION:
            final Set<Schema> subSchemas = new HashSet<>(lhs.getTypes());
            if (rhs.getType() == Schema.Type.UNION) {
                subSchemas.addAll(rhs.getTypes());
            } else {
                subSchemas.add(rhs);
            }
            return Schema.createUnion(new ArrayList<>(subSchemas));
        default:
            throw new CodecException("Unexpected schema type - " + lhs.getType());
    }
    return Schema.createUnion(lhs, rhs);
}

17 Source : AvroSchemaConverter.java
with Apache License 2.0
from trinodb

public List<Type> convertAvroSchema(Schema schema) {
    requireNonNull(schema, "schema is null");
    List<Type> types;
    if (schema.getType().equals(RECORD)) {
        types = convertRecordSchema(schema);
    } else {
        types = convertSimpleSchema(schema);
    }
    checkState(!types.isEmpty(), "Schema has no valid fields: '%s'", schema);
    return types;
}

17 Source : FastDeserializerGenerator.java
with Apache License 2.0
from RTBHOUSE

private void processSimpleType(Schema schema, JBlock methodBody, FieldAction action, BiConsumer<JBlock, JExpression> putExpressionIntoParent) {
    switch(schema.getType()) {
        case ENUM:
            processEnum(schema, methodBody, action, putExpressionIntoParent);
            break;
        case FIXED:
            processFixed(schema, methodBody, action, putExpressionIntoParent);
            break;
        default:
            processPrimitive(schema, methodBody, action, putExpressionIntoParent);
    }
}

17 Source : RecordToEntityConverter.java
with MIT License
from orfeon

private static Schema.Type getType(final Schema schema) {
    switch(schema.getType()) {
        case UNION:
            Schema childSchema = schema.getTypes().stream().filter(s -> !s.getType().equals(Schema.Type.NULL)).findAny().orElseThrow(() -> new IllegalArgumentException("UNION does not have another schema."));
            return getType(childSchema);
        default:
            return schema.getType();
    }
}

17 Source : AvroSchemaUtil.java
with MIT License
from orfeon

public static Schema unnestUnion(Schema schema) {
    if (schema.getType().equals(Schema.Type.UNION)) {
        return schema.getTypes().stream().filter(s -> !s.getType().equals(Schema.Type.NULL)).findAny().orElseThrow(() -> new IllegalArgumentException("UNION does not have another schema."));
    }
    return schema;
}

17 Source : AvroSchemaUtil.java
with MIT License
from orfeon

public static LogicalTypes.Decimal getLogicalTypeDecimal(Schema schema) {
    if (schema.getType().equals(Schema.Type.UNION)) {
        Schema childSchema = unnestUnion(schema);
        return getLogicalTypeDecimal(childSchema);
    }
    final int precision = schema.getObjectProp("precision") != null ? Integer.valueOf(schema.getObjectProp("precision").toString()) : 0;
    final int scale = schema.getObjectProp("scale") != null ? Integer.valueOf(schema.getObjectProp("scale").toString()) : 0;
    return LogicalTypes.decimal(precision, scale);
}

17 Source : SpannerTablePrepareDoFn.java
with MIT License
from orfeon

private static boolean isValidColumnType(final Schema schema) {
    switch(schema.getType()) {
        case RECORD:
        case MAP:
        case NULL:
            return false;
        case ARRAY:
            return isValidColumnType(schema.getElementType());
        case UNION:
            final Schema childSchema = schema.getTypes().stream().filter(s -> !s.getType().equals(Schema.Type.NULL)).findAny().orElseThrow(() -> new IllegalArgumentException("UNION does not have another schema."));
            return isValidColumnType(childSchema);
        default:
            return true;
    }
}

17 Source : AvroLoader.java
with Apache License 2.0
from nhl

protected Acreplacedulator<?> mapColumn(Schema columnSchema) {
    switch(columnSchema.getType()) {
        // Raw numeric and boolean types can be loaded as primitives,
        // as numeric nullable types are declared as unions and will fall under the "default" case
        case INT:
            return new IntAcreplacedulator();
        case DOUBLE:
            return new DoubleAcreplacedulator();
        case LONG:
            return new LongAcreplacedulator();
        case BOOLEAN:
            return new BooleanAcreplacedulator();
        case STRING:
        case BYTES:
        case ENUM:
        case NULL:
            return new ObjectAcreplacedulator<>();
        case UNION:
            return mapUnionColumn(columnSchema.getTypes());
        default:
            throw new UnsupportedOperationException("(Yet) unsupported Avro schema type: " + columnSchema.getType());
    }
}

17 Source : AvroSchemaUtil.java
with MIT License
from mercari

/**
 * Extract logical type decimal from {@link Schema}.
 *
 * @param schema Avro Schema object.
 * @return true if schema is logical decimal type.
 */
public static LogicalTypes.Decimal getLogicalTypeDecimal(Schema schema) {
    if (schema.getType().equals(Schema.Type.UNION)) {
        Schema childSchema = unnestUnion(schema);
        return getLogicalTypeDecimal(childSchema);
    }
    final int precision = schema.getObjectProp("precision") != null ? Integer.valueOf(schema.getObjectProp("precision").toString()) : 0;
    final int scale = schema.getObjectProp("scale") != null ? Integer.valueOf(schema.getObjectProp("scale").toString()) : 0;
    return LogicalTypes.decimal(precision, scale);
}

17 Source : AvroSchemaUtil.java
with MIT License
from mercari

/**
 * Extract child Avro schema from nullable union Avro {@link Schema}.
 *
 * @param schema Avro Schema object.
 * @return Child Avro schema or input schema if not union schema.
 */
public static Schema unnestUnion(Schema schema) {
    if (schema.getType().equals(Schema.Type.UNION)) {
        return schema.getTypes().stream().filter(s -> !s.getType().equals(Schema.Type.NULL)).findAny().orElseThrow(() -> new IllegalArgumentException("UNION does not have another schema."));
    }
    return schema;
}

17 Source : AvroSchemaUtil.java
with MIT License
from mercari

public static boolean isNullable(final Schema schema) {
    if (schema.getType().equals(Schema.Type.UNION)) {
        return schema.getTypes().stream().map(Schema::getType).anyMatch(Schema.Type.NULL::equals);
    }
    return false;
}

17 Source : ResolvingGrammarGenerator.java
with BSD 2-Clause "Simplified" License
from linkedin

private static int bestBranch(Schema r, Schema w) {
    Schema.Type vt = w.getType();
    // first scan for exact match
    int j = 0;
    for (Schema b : r.getTypes()) {
        if (vt == b.getType())
            if (vt == Schema.Type.RECORD) {
                String vname = w.getName();
                if (vname == null || vname.equals(b.getName()))
                    return j;
            } else
                return j;
        j++;
    }
    // then scan match via numeric promotion
    j = 0;
    for (Schema b : r.getTypes()) {
        switch(vt) {
            case INT:
                switch(b.getType()) {
                    case LONG:
                    case DOUBLE:
                        return j;
                }
                break;
            case LONG:
            case FLOAT:
                switch(b.getType()) {
                    case DOUBLE:
                        return j;
                }
                break;
        }
        j++;
    }
    return -1;
}

17 Source : AvroSchemaUtil.java
with BSD 2-Clause "Simplified" License
from linkedin

private static void traverseSchema(Schema schema, SchemaVisitor visitor, IdenreplacedyHashMap<Object, Boolean> visited) {
    if (visited.put(schema, Boolean.TRUE) != null) {
        // been there, done that
        return;
    }
    visitor.visitSchema(schema);
    switch(schema.getType()) {
        case UNION:
            for (Schema unionBranch : schema.getTypes()) {
                traverseSchema(unionBranch, visitor, visited);
            }
            return;
        case ARRAY:
            traverseSchema(schema.getElementType(), visitor, visited);
            return;
        case MAP:
            traverseSchema(schema.getValueType(), visitor, visited);
            return;
        case RECORD:
            for (Schema.Field field : schema.getFields()) {
                visitor.visitField(schema, field);
                traverseSchema(field.schema(), visitor, visited);
            }
            break;
        default:
    }
}

17 Source : RecordBuilderBase.java
with BSD 2-Clause "Simplified" License
from linkedin

protected static boolean isValidValue(Field f, Object value) {
    if (value != null) {
        return true;
    } else {
        Schema schema = f.schema();
        Type type = schema.getType();
        if (type == Type.NULL) {
            return true;
        } else {
            if (type == Type.UNION) {
                Iterator i$ = schema.getTypes().iterator();
                while (i$.hasNext()) {
                    Schema s = (Schema) i$.next();
                    if (s.getType() == Type.NULL) {
                        return true;
                    }
                }
            }
            return false;
        }
    }
}

17 Source : AvroRandomDataGenerator.java
with BSD 2-Clause "Simplified" License
from linkedin

private Object generateObject(Schema schema, Map propertiesProp) {
    switch(schema.getType()) {
        case ARRAY:
            return generateArray(schema, propertiesProp);
        case BOOLEAN:
            return generateBoolean(propertiesProp);
        case BYTES:
            return generateBytes(propertiesProp);
        case DOUBLE:
            return generateDouble(propertiesProp);
        case ENUM:
            return generateEnumSymbol(schema);
        case FIXED:
            return generateFixed(schema);
        case FLOAT:
            return generateFloat(propertiesProp);
        case INT:
            return generateInt(propertiesProp);
        case LONG:
            return generateLong(propertiesProp);
        case MAP:
            return generateMap(schema, propertiesProp);
        case NULL:
            return generateNull();
        case RECORD:
            return generateRecord(schema, propertiesProp);
        case STRING:
            return generateString(propertiesProp);
        case UNION:
            return generateUnion(schema, propertiesProp);
        default:
            throw new RuntimeException("Unrecognized schema type: " + schema.getType());
    }
}

17 Source : AvroSchemaRegistrySqsAsyncClient.java
with MIT License
from JaidenAshmore

private Map<Clreplaced<?>, RegisteredSchema> parseAvailableSchemas(final List<Resource> schemaImports, final List<Resource> schemaLocations) {
    final Schema.Parser schemaParser = new Schema.Parser();
    return Stream.of(schemaImports, schemaLocations).filter(arr -> !ObjectUtils.isEmpty(arr)).distinct().flatMap(List::stream).flatMap(resource -> {
        final Schema schema;
        try {
            schema = schemaParser.parse(resource.getInputStream());
        } catch (IOException ioException) {
            throw new RuntimeException("Error parsing schema: " + resource.getFilename(), ioException);
        }
        if (schema.getType().equals(Schema.Type.UNION)) {
            return schema.getTypes().stream();
        } else {
            return Stream.of(schema);
        }
    }).map(this::createObjectMapping).collect(toMap(Map.Entry::getKey, Map.Entry::getValue));
}

17 Source : DefaultAvroSerDesHandler.java
with Apache License 2.0
from hortonworks

@Override
public Object handlePayloadDeserialization(InputStream payloadInputStream, Schema writerSchema, Schema readerSchema, boolean useSpecificAvroReader) {
    Object deserializedObj;
    Schema.Type writerSchemaType = writerSchema.getType();
    try {
        if (Schema.Type.BYTES.equals(writerSchemaType)) {
            // serializer writes byte array directly without going through avro encoder layers.
            deserializedObj = IOUtils.toByteArray(payloadInputStream);
        } else if (Schema.Type.STRING.equals(writerSchemaType)) {
            // generate UTF-8 string object from the received bytes.
            deserializedObj = new String(IOUtils.toByteArray(payloadInputStream), AvroUtils.UTF_8);
        } else {
            DatumReader datumReader = getDatumReader(writerSchema, readerSchema, useSpecificAvroReader);
            deserializedObj = datumReader.read(null, DecoderFactory.get().binaryDecoder(payloadInputStream, null));
        }
    } catch (IOException e) {
        throw new AvroRetryableException(e);
    } catch (Exception e) {
        throw new AvroException(e);
    }
    return deserializedObj;
}

17 Source : ConfluentAvroSerDesHandler.java
with Apache License 2.0
from hortonworks

@Override
public Object handlePayloadDeserialization(InputStream payloadInputStream, Schema writerSchema, Schema readerSchema, boolean useSpecificAvroReader) {
    Object deserializedObj;
    try {
        if (Schema.Type.BYTES.equals(writerSchema.getType())) {
            // serializer writes byte array directly without going through avro encoder layers.
            deserializedObj = IOUtils.toByteArray(payloadInputStream);
        } else {
            DatumReader datumReader = getDatumReader(writerSchema, readerSchema, useSpecificAvroReader);
            deserializedObj = datumReader.read(null, DecoderFactory.get().binaryDecoder(payloadInputStream, null));
        }
    } catch (IOException e) {
        throw new AvroRetryableException("Error deserializing Avro message for id " + writerSchema, e);
    } catch (RuntimeException e) {
        // avro deserialization may throw AvroRuntimeException, NullPointerException, etc
        throw new AvroException("Error deserializing Avro message for id " + writerSchema, e);
    }
    return deserializedObj;
}

17 Source : AvroFieldsGenerator.java
with Apache License 2.0
from hortonworks

private void parseField(Schema.Field field, List<SchemaFieldInfo> schemaFieldInfos, Set<String> visitedRecords) {
    Schema schema = field.schema();
    Schema.Type type = schema.getType();
    String name = field.name();
    LOG.debug("Visiting field: [{}]", field);
    String namespace = null;
    try {
        namespace = schema.getNamespace();
    } catch (Exception e) {
    // ignore.
    }
    schemaFieldInfos.add(new SchemaFieldInfo(namespace, name, type.name()));
    // todo check whether fields should be mapped to the root schema.
    parseSchema(schema, schemaFieldInfos, visitedRecords);
}

17 Source : AvroFileAccessorTest.java
with Apache License 2.0
from greenplum-db

/**
 * Helper method for testing schema
 *
 * @param schema the schema
 * @param name   the name
 */
private static void verifySchema(Schema schema, String name) {
    replacedertNotNull(schema);
    replacedertEquals(Schema.Type.RECORD, schema.getType());
    replacedertEquals(name, schema.getName());
    Map<String, String> fieldToType = new HashMap<String, String>() {

        {
            put("id", "long");
            put("username", "string");
            put("followers", "array");
        }
    };
    for (String key : fieldToType.keySet()) {
        replacedertEquals(fieldToType.get(key), schema.getField(key).schema().getType().getName());
    }
}

17 Source : AvroUtilitiesTest.java
with Apache License 2.0
from greenplum-db

/**
 * Helper method for testing generated schema
 *
 * @param schema the schema
 */
private static void verifyGeneratedSchema(Schema schema) {
    replacedertNotNull(schema);
    replacedertEquals(schema.getType(), Schema.Type.RECORD);
    Map<String, String> fieldToType = new HashMap<String, String>() {

        {
            put("id", "union");
            put("username", "union");
            put("followers", "union");
        }
    };
    Map<String, String> unionToInnerType = new HashMap<String, String>() {

        {
            put("id", "long");
            put("username", "string");
            // arrays become strings
            put("followers", "string");
        }
    };
    for (String key : fieldToType.keySet()) {
        replacedertEquals(fieldToType.get(key), schema.getField(key).schema().getType().getName());
        // check the union's inner types
        replacedertEquals("null", schema.getField(key).schema().getTypes().get(0).getName());
        replacedertEquals(unionToInnerType.get(key), schema.getField(key).schema().getTypes().get(1).getName());
    }
}

17 Source : AvroExtractor.java
with Mozilla Public License 2.0
from corunet

private boolean checkIfMap(Schema innerSchema) {
    return MAP.equals(innerSchema.getType());
}

17 Source : AvroExtractor.java
with Mozilla Public License 2.0
from corunet

private boolean checkIfRecord(Schema innerSchema) {
    return RECORD.equals(innerSchema.getType());
}

17 Source : ValueReaders.java
with Apache License 2.0
from apache

public static ValueReader<byte[]> decimalBytesReader(Schema schema) {
    switch(schema.getType()) {
        case FIXED:
            return ValueReaders.fixed(schema.getFixedSize());
        case BYTES:
            return ValueReaders.bytes();
        default:
            throw new IllegalArgumentException("Invalid primitive type for decimal: " + schema.getType());
    }
}

See More Examples