Load .avdl to erlavro internal format

Now it's possible to use basic .avdl to encode/decode avro

Author: seriyps

src/avro.erl

@@ -154,7 +154,8 @@ set_json_provider(Module) -> avro_json_compat:set_provider(Module).
 get_json_provider() -> avro_json_compat:get_provider().
 
 %% @doc Decode JSON format avro schema into `erlavro' internals.
--spec decode_schema(binary()) -> avro_type().
+%% @param JSON: JSON binary or erlang `map()' json representation
+-spec decode_schema(binary() | map() | [map()]) -> avro_type().
 decode_schema(JSON) -> avro_json_decoder:decode_schema(JSON).
 
 %% @doc Make type lookup function from type definition.
@@ -187,7 +188,8 @@ make_lkup_fun(AssignedName, Type) ->
 %%  * allow_type_redefine: `boolean()'
 %%     This option is to allow one type being defined more than once.
 %% @end
--spec decode_schema(binary(), proplists:proplist()) -> avro_type().
+-spec decode_schema(binary() | map() | [map()], proplists:proplist()) ->
+                       avro_type().
 decode_schema(JSON, Options) ->
   avro_json_decoder:decode_schema(JSON, Options).
 
@@ -280,7 +282,8 @@ make_decoder(Schema, Options) ->
 %% takes only one `binary()' input arg.
 -spec make_simple_decoder(avro_type() | binary(), codec_options()) ->
         simple_decoder().
-make_simple_decoder(JSON, Options) when is_binary(JSON) ->
+make_simple_decoder(JSON, Options) when is_binary(JSON);
+                                        is_map(JSON) ->
   make_simple_decoder(decode_schema(JSON), Options);
 make_simple_decoder(Type, Options) when ?IS_TYPE_RECORD(Type) ->
   Lkup = make_lkup_fun(Type),

src/avro_idl.erl

@@ -4,9 +4,10 @@
 %%% as create Avro encoders and decoders.
 %%% @end
 %%% @reference See [https://avro.apache.org/docs/current/idl.html]
-%%% @author Sergey Prokhhorov <me@seriyps.ru>
+%%% @author Sergey Prokhorov <me@seriyps.ru>
 -module(avro_idl).
 
+-export([decode_schema/2]).
 -export([new_context/1,
          str_to_avpr/2,
          protocol_to_avpr/2,
@@ -16,6 +17,23 @@
 
 -record(st, {cwd}).
 
+decode_schema(SchemaStr, Cwd) ->
+    Protocol = str_to_avpr(SchemaStr, Cwd),
+    #{<<"types">> := Types0} = Protocol,
+    Types1 = lists:filter(
+              fun(#{<<"type">> := TName}) ->TName =/= <<"error">> end, Types0),
+    Ns = maps:get(<<"namespace">>, Protocol, ?AVRO_NS_GLOBAL),
+    Types = lists:map(
+        fun(T) ->
+                case maps:is_key(<<"namespace">>, T) of
+                    false ->
+                        T#{<<"namespace">> => Ns};
+                    true ->
+                        T
+                end
+        end, Types1),
+    avro:decode_schema(Types, [{ignore_bad_default_values, true}]).
+
 new_context(Cwd) ->
     #st{cwd = Cwd}.
 
@@ -64,7 +82,7 @@ typedecl_to_avsc(#enum{name = Name, meta = Meta, variants = Vars}, _St) ->
     meta(
       #{<<"type">> => ?AVRO_ENUM,
         <<"name">> => b(Name),
-        <<"variants">> => lists:map(fun b/1, Vars)
+        <<"symbols">> => lists:map(fun b/1, Vars)
        },
       Meta);
 typedecl_to_avsc(#fixed{name = Name, meta = Meta, size = Size}, _St) ->

src/avro_idl_lexer.xrl

@@ -1,7 +1,7 @@
 %% @doc Avro IDL lexer
 %% @end
 %% @reference See [https://avro.apache.org/docs/current/idl.html]
-%% @author Sergey Prokhhorov <me@seriyps.ru>
+%% @author Sergey Prokhorov <me@seriyps.ru>
 
 Definitions.
 

src/avro_idl_parser.yrl

@@ -3,7 +3,7 @@ Header "%%% @doc Avro IDL parser
 %%% XXX: all `comment_v' tokens should be filtered-out before parsing!
 %%% @end
 %%% @reference See [https://avro.apache.org/docs/current/idl.html]
-%%% @author Sergey Prokhhorov <me@seriyps.ru>".
+%%% @author Sergey Prokhorov <me@seriyps.ru>".
 
 Terminals id ns_id null string_v doc_v float_v integer_v bool_v annotation_v
     primitive_t logical_t decimal_t
@@ -120,7 +120,7 @@ import_file_type -> schema_k : schema.
 
 %% -- Enum typedef
 enum ->
-    enum_t id '{' id enum_variants :
+    enum_t id '{' id enum_variants :            % TODO: add support for default
         #enum{name = value_of('$2'), variants = [value_of('$4') | '$5']}.
 enum ->
     meta enum :

src/avro_json_decoder.erl

@@ -47,7 +47,7 @@
 %%%_* APIs =====================================================================
 
 %% @doc Decode JSON format avro schema into erlavro internals.
--spec decode_schema(binary()) -> avro_type().
+-spec decode_schema(binary() | map() | [map()]) -> avro_type().
 decode_schema(JSON) ->
   decode_schema(JSON, _Opts = []).
 
@@ -66,7 +66,7 @@ decode_schema(JSON) ->
 %%  * allow_type_redefine: `boolean()'
 %%     This option is to allow one type being defined more than once.
 %% @end
--spec decode_schema(binary(), sc_opts()) -> avro_type().
+-spec decode_schema(binary() | map() | [map()], sc_opts()) -> avro_type().
 decode_schema(JSON, Opts) when is_list(Opts) ->
   %% Parse JSON first
   Type = parse_schema(decode_json(JSON)),
@@ -513,9 +513,21 @@ do_parse_union_ex(ValueTypeName, Value, UnionType,
 %% 'map' is a better option, but we have to keep it backward compatible.
 %% 'proplist' is not an option because otherwise there is no way to tell
 %% apart 'object' and 'array'.
--spec decode_json(binary()) -> json_value().
+-spec decode_json(binary() | map() | [map()]) -> json_value().
+decode_json(Parsed) when is_map(Parsed);
+                         is_list(Parsed) ->
+  map_to_tuple(Parsed);
 decode_json(JSON) -> avro_json_compat:decode(JSON, [{object_format, tuple}]).
 
+%% recursively convert map to json-tuple format
+map_to_tuple(Map) when is_map(Map) ->
+  {[{K, map_to_tuple(V)}
+    || {K, V} <- maps:to_list(Map)]};
+map_to_tuple(Array) when is_list(Array) ->
+  lists:map(fun map_to_tuple/1, Array);
+map_to_tuple(Other) ->
+  Other.
+
 %% Filter out non-custom properties.
 -spec filter_custom_props([{binary(), json_value()}], [name()]) ->
         [custom_prop()].

test/avro_idl_parse_tests.erl

@@ -1,6 +1,6 @@
 %% @doc Tests for IDL lexer + parser
 %% @end
-%% @author Sergey Prokhhorov <me@seriyps.ru>
+%% @author Sergey Prokhorov <me@seriyps.ru>
 -module(avro_idl_parse_tests).
 
 -include("../src/idl.hrl").

test/avro_idl_tests.erl

@@ -1,6 +1,6 @@
 %% @doc Tests for IDL converter / loader
 %% @end
-%% @author Sergey Prokhhorov <me@seriyps.ru>
+%% @author Sergey Prokhorov <me@seriyps.ru>
 -module(avro_idl_tests).
 
 -include("../src/idl.hrl").
@@ -34,7 +34,7 @@ annotations_avpr_test() ->
                <<"namespace">> => <<"enums">>,
                <<"type">> => ?AVRO_ENUM,
                <<"name">> => <<"MyEnum">>,
-               <<"variants">> => [<<"A">>, <<"B">>, <<"C">>]},
+               <<"symbols">> => [<<"A">>, <<"B">>, <<"C">>]},
              #{<<"doc">> => <<"My Fixed">>,
                <<"namespace">> => <<"fixeds">>,
                <<"type">> => ?AVRO_FIXED,
@@ -88,7 +88,7 @@ protocol_with_typedefs_avpr_test() ->
        [#{<<"name">> := <<"MyEnum1">>},
         #{<<"name">> := <<"MyEnum2">>,
           <<"type">> := ?AVRO_ENUM,
-          <<"variants">> := [<<"VAR21">>, <<"VAR22">>, <<"VAR23">>]},
+          <<"symbols">> := [<<"VAR21">>, <<"VAR22">>, <<"VAR23">>]},
         #{<<"name">> := <<"MyFix">>,
           <<"type">> := ?AVRO_FIXED,
           <<"size">> := 10},
@@ -137,14 +137,14 @@ protocol_with_typedefs_avpr_test() ->
        Messages).
 
 
-duplicate_annotation_test() ->
+duplicate_annotation_avpr_test() ->
     ?assertError(
        {duplicate_annotation, "my_decorator", _, _},
        avro_idl:str_to_avpr(
          "@my_decorator(\"a\") @my_decorator(\"b\") protocol MyProto{}", "")
       ).
 
-nested_complex_types_test() ->
+nested_complex_types_avr_test() ->
     ?assertEqual(
        #{<<"protocol">> => <<"P">>,
          <<"messages">> => [],
@@ -165,9 +165,19 @@ nested_complex_types_test() ->
          "protocol P { record R { array<map<union{null, ns.T}>> f; }}", "")
       ).
 
+full_protocol_load_test() ->
+    Schema = read_schema("full_protocol"),
+    DecSchema = avro_idl:decode_schema(Schema, ""),
+    _EncSchema = avro:encode_schema(DecSchema).
+    %% ?debugFmt("~n~p~n~s", [DecSchema, EncSchema]).
+
 %% Helpers
 
-idl_to_avpr(Name) ->
+read_schema(Name) ->
     File = "test/data/" ++ Name ++ ".avdl",
     {ok, B} = file:read_file(File),
-    avro_idl:str_to_avpr(binary_to_list(B), "").
+    binary_to_list(B).
+
+idl_to_avpr(Name) ->
+    Schema = read_schema(Name),
+    avro_idl:str_to_avpr(Schema, "").