Skip to content

Commit d011692

Browse files
seriypsseriyps
committed
Use binaries for avpr JSON-map representation
1 parent 1ee357e commit d011692

File tree

6 files changed

+201
-157
lines changed

6 files changed

+201
-157
lines changed

‎include/erlavro.hrl‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@
3737
-define(AVRO_MAP, <<"map">>).
3838
-define(AVRO_UNION, <<"union">>).
3939
-define(AVRO_FIXED, <<"fixed">>).
40+
-define(AVRO_ERROR, <<"error">>). % idl
4041

4142
-define(IS_AVRO_PRIMITIVE_NAME(N),
4243
(N =:= ?AVRO_NULL orelse

‎src/avro_idl.erl‎

Lines changed: 63 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,18 @@
11
%%% @doc APIs to work with Avro IDL format
22
%%%
3-
%%% See [https://avro.apache.org/docs/1.9.2/idl.html]
3+
%%% This module allows to convert .avdl format to .avpr and .avsc as well
4+
%%% as create Avro encoders and decoders.
5+
%%% @end
6+
%%% @reference See [https://avro.apache.org/docs/current/idl.html]
7+
%%% @author Sergey Prokhhorov <me@seriyps.ru>
48
-module(avro_idl).
59

610
-export([new_context/1,
711
str_to_avpr/2,
812
protocol_to_avpr/2,
913
typedecl_to_avsc/2]).
1014
-include("idl.hrl").
15+
-include("erlavro.hrl").
1116

1217
-record(st, {cwd}).
1318

@@ -32,13 +37,13 @@ protocol_to_avpr(#protocol{name = Name,
3237
(_) -> true
3338
end, Defs),
3439
Protocol0 =
35-
#{protocol => Name,
36-
types =>
40+
#{<<"protocol">> => b(Name),
41+
<<"types">> =>
3742
lists:map(
3843
fun(Type) ->
3944
typedecl_to_avsc(Type, St)
4045
end, Types),
41-
messages =>
46+
<<"messages">> =>
4247
lists:map(
4348
fun(Message) ->
4449
message_to_avsc(Message, St)
@@ -57,36 +62,36 @@ process_imports(Defs, _St) ->
5762

5863
typedecl_to_avsc(#enum{name = Name, meta = Meta, variants = Vars}, _St) ->
5964
meta(
60-
#{type => enum,
61-
name => Name,
62-
variants => Vars
65+
#{<<"type">> => ?AVRO_ENUM,
66+
<<"name">> => b(Name),
67+
<<"variants">> => lists:map(fun b/1, Vars)
6368
},
6469
Meta);
6570
typedecl_to_avsc(#fixed{name = Name, meta = Meta, size = Size}, _St) ->
6671
meta(
67-
#{type => fixed,
68-
name => Name,
69-
size => Size},
72+
#{<<"type">> => ?AVRO_FIXED,
73+
<<"name">> => b(Name),
74+
<<"size">> => Size},
7075
Meta);
7176
typedecl_to_avsc(#error{name = Name, meta = Meta, fields = Fields}, St) ->
7277
meta(
73-
#{type => error,
74-
name => Name,
75-
fields => [field_to_avsc(Field, St) || Field <- Fields]},
78+
#{<<"type">> => ?AVRO_ERROR,
79+
<<"name">> => b(Name),
80+
<<"fields">> => [field_to_avsc(Field, St) || Field <- Fields]},
7681
Meta);
7782
typedecl_to_avsc(#record{name = Name, meta = Meta, fields = Fields}, St) ->
7883
meta(
79-
#{type => record,
80-
name => Name,
81-
fields => [field_to_avsc(Field, St) || Field <- Fields]},
84+
#{<<"type">> => ?AVRO_RECORD,
85+
<<"name">> => b(Name),
86+
<<"fields">> => [field_to_avsc(Field, St) || Field <- Fields]},
8287
Meta).
8388

8489
field_to_avsc(#field{name = Name, meta = Meta,
8590
type = Type, default = Default}, St) ->
8691
meta(
8792
default(
88-
#{name => Name,
89-
type => type_to_avsc(Type, St)},
93+
#{<<"name">> => b(Name),
94+
<<"type">> => type_to_avsc(Type, St)},
9095
Default), % TODO: maybe validate default matches type
9196
Meta).
9297

@@ -96,62 +101,60 @@ message_to_avsc(#function{name = Name, meta = Meta,
96101
%% TODO: arguments can just reuse `#field{}`
97102
ArgsSchema =
98103
[default(
99-
#{name => ArgName,
100-
type => type_to_avsc(Type, St)},
104+
#{<<"name">> => b(ArgName),
105+
<<"type">> => type_to_avsc(Type, St)},
101106
Default)
102107
|| {arg, ArgName, Type, Default} <- Args],
103108
Schema0 =
104-
#{name => Name,
105-
request => ArgsSchema,
106-
response => type_to_avsc(Return, St)},
109+
#{<<"name">> => b(Name),
110+
<<"request">> => ArgsSchema,
111+
<<"response">> => type_to_avsc(Return, St)},
107112
Schema1 = case Extra of
108113
undefined -> Schema0;
109114
oneway ->
110-
Schema0#{'one-way' => true};
115+
Schema0#{<<"one-way">> => true};
111116
{throws, ThrowsTypes} ->
112-
%% Throws = [type_to_avsc(TType, St)
113-
%% || TType <- ThrowsTypes],
114-
Schema0#{error => ThrowsTypes}
117+
Schema0#{<<"error">> => lists:map(fun b/1, ThrowsTypes)}
115118
end,
116119
meta(Schema1, Meta).
117120

118121

119122
type_to_avsc(void, _St) ->
120-
null;
123+
?AVRO_NULL;
121124
type_to_avsc(null, _St) ->
122-
null;
125+
?AVRO_NULL;
123126
type_to_avsc(T, _St) when T == int;
124127
T == long;
125128
T == string;
126129
T == boolean;
127130
T == float;
128131
T == double;
129132
T == bytes ->
130-
T;
133+
atom_to_binary(T, utf8);
131134
type_to_avsc({decimal, Precision, Scale}, _St) ->
132-
#{type => bytes,
133-
'logicalType' => "decimal",
134-
precision => Precision,
135-
scale => Scale};
135+
#{<<"type">> => ?AVRO_BYTES,
136+
<<"logicalType">> => <<"decimal">>,
137+
<<"precision">> => Precision,
138+
<<"scale">> => Scale};
136139
type_to_avsc(date, _St) ->
137-
#{type => int,
138-
'logicalType' => "date"};
140+
#{<<"type">> => ?AVRO_INT,
141+
<<"logicalType">> => <<"date">>};
139142
type_to_avsc(time_ms, _St) ->
140-
#{type => int,
141-
'logicalType' => "time-millis"};
143+
#{<<"type">> => ?AVRO_INT,
144+
<<"logicalType">> => <<"time-millis">>};
142145
type_to_avsc(timestamp_ms, _St) ->
143-
#{type => long,
144-
'logicalType' => "timestamp-millis"};
146+
#{<<"type">> => ?AVRO_LONG,
147+
<<"logicalType">> => <<"timestamp-millis">>};
145148
type_to_avsc({custom, Id}, _St) ->
146-
Id;
149+
b(Id);
147150
type_to_avsc({union, Types}, St) ->
148151
[type_to_avsc(Type, St) || Type <- Types];
149152
type_to_avsc({array, Of}, St) ->
150-
#{type => array,
151-
items => type_to_avsc(Of, St)};
153+
#{<<"type">> => ?AVRO_ARRAY,
154+
<<"items">> => type_to_avsc(Of, St)};
152155
type_to_avsc({map, ValType}, St) ->
153-
#{type => map,
154-
values => type_to_avsc(ValType, St)}.
156+
#{<<"type">> => ?AVRO_MAP,
157+
<<"values">> => type_to_avsc(ValType, St)}.
155158

156159
meta(Schema, Meta) ->
157160
{Docs, Annotations} =
@@ -163,17 +166,27 @@ meta(Schema, Meta) ->
163166
[] -> Schema;
164167
_ ->
165168
DocStrings = [S || {doc, S} <- Docs],
166-
Schema#{"doc" => lists:flatten(lists:join(
167-
"\n", DocStrings))}
169+
Schema#{<<"doc">> => b(lists:join(
170+
"\n", DocStrings))}
168171
end,
169172
lists:foldl(
170173
fun(#annotation{name = Name, value = Value}, Schema2) ->
171-
maps:is_key(Name, Schema2) andalso
174+
BName = b(Name),
175+
BVal = case Value of
176+
[] -> <<>>;
177+
[C | _] when is_integer(C) -> b(Value);
178+
_ ->
179+
[b(Str) || Str <- Value]
180+
end,
181+
maps:is_key(BName, Schema2) andalso
172182
error({duplicate_annotation, Name, Value, Schema2}),
173-
Schema2#{Name => Value}
183+
Schema2#{BName => BVal}
174184
end, Schema1, Annotations).
175185

176186
default(Obj, undefined) ->
177187
Obj;
178188
default(Obj, Default) ->
179-
Obj#{default => Default}.
189+
Obj#{<<"default">> => Default}.
190+
191+
b(Str) when is_list(Str) ->
192+
unicode:characters_to_binary(Str).

‎src/avro_idl_lexer.xrl‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
%% @doc Avro IDL lexer
2-
%% https://avro.apache.org/docs/current/idl.html
2+
%% @end
3+
%% @reference See [https://avro.apache.org/docs/current/idl.html]
4+
%% @author Sergey Prokhhorov <me@seriyps.ru>
35

46
Definitions.
57

‎src/avro_idl_parser.yrl‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
1-
%% @doc Avro IDL parser
2-
%% https://avro.apache.org/docs/1.9.2/idl.html
3-
%% XXX: all `comment_v` tokens should be filtered-out before parsing!
4-
5-
Header "%% Hello".
1+
Header "%%% @doc Avro IDL parser
2+
%%%
3+
%%% XXX: all `comment_v' tokens should be filtered-out before parsing!
4+
%%% @end
5+
%%% @reference See [https://avro.apache.org/docs/current/idl.html]
6+
%%% @author Sergey Prokhhorov <me@seriyps.ru>".
67

78
Terminals id ns_id null string_v doc_v float_v integer_v bool_v annotation_v
89
primitive_t logical_t decimal_t

‎test/avro_idl_parse_tests.erl‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
11
%% @doc Tests for IDL lexer + parser
2+
%% @end
3+
%% @author Sergey Prokhhorov <me@seriyps.ru>
24
-module(avro_idl_parse_tests).
35

46
-include("../src/idl.hrl").

0 commit comments

Comments
 (0)