Skip to content

Commit ec32200

Browse files
miloyipmiloyip
committed
Merge pull request #584 from ludocode/trailing-commas
Added optional support for trailing commas
2 parents ca07fe2 + 6821754 commit ec32200

File tree

3 files changed

+196
-2
lines changed

3 files changed

+196
-2
lines changed

‎doc/dom.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,7 @@ Parse flags | Meaning
116116
`kParseStopWhenDoneFlag` | After parsing a complete JSON root from stream, stop further processing the rest of stream. When this flag is used, parser will not generate `kParseErrorDocumentRootNotSingular` error. Using this flag for parsing multiple JSONs in the same stream.
117117
`kParseFullPrecisionFlag` | Parse number in full precision (slower). If this flag is not set, the normal precision (faster) is used. Normal precision has maximum 3 [ULP](http://en.wikipedia.org/wiki/Unit_in_the_last_place) error.
118118
`kParseCommentsFlag` | Allow one-line `// ...` and multi-line `/* ... */` comments (relaxed JSON syntax).
119+
`kParseTrailingCommasFlag` | Allow trailing commas at the end of objects and arrays (relaxed JSON syntax).
119120
120121
By using a non-type template parameter, instead of a function parameter, C++ compiler can generate code which is optimized for specified combinations, improving speed, and reducing code size (if only using a single specialization). The downside is the flags needed to be determined in compile-time.
121122

‎include/rapidjson/reader.h‎

Lines changed: 34 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -149,6 +149,7 @@ enum ParseFlag {
149149
kParseFullPrecisionFlag = 16, //!< Parse number in full precision (but slower).
150150
kParseCommentsFlag = 32, //!< Allow one-line (//) and multi-line (/**/) comments.
151151
kParseNumbersAsStringsFlag = 64, //!< Parse all numbers (ints/doubles) as strings.
152+
kParseTrailingCommasFlag = 128, //!< Allow trailing commas at the end of objects and arrays.
152153
kParseDefaultFlags = RAPIDJSON_PARSE_DEFAULT_FLAGS //!< Default parse flags. Can be customized by defining RAPIDJSON_PARSE_DEFAULT_FLAGS
153154
};
154155

@@ -636,6 +637,15 @@ class GenericReader {
636637
RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissCommaOrCurlyBracket, is.Tell());
637638
break;
638639
}
640+
641+
if (parseFlags & kParseTrailingCommasFlag) {
642+
if (is.Peek() == '}') {
643+
if (RAPIDJSON_UNLIKELY(!handler.EndObject(memberCount)))
644+
RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell());
645+
is.Take();
646+
return;
647+
}
648+
}
639649
}
640650
}
641651

@@ -676,6 +686,15 @@ class GenericReader {
676686
}
677687
else
678688
RAPIDJSON_PARSE_ERROR(kParseErrorArrayMissCommaOrSquareBracket, is.Tell());
689+
690+
if (parseFlags & kParseTrailingCommasFlag) {
691+
if (is.Peek() == ']') {
692+
if (RAPIDJSON_UNLIKELY(!handler.EndArray(elementCount)))
693+
RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell());
694+
is.Take();
695+
return;
696+
}
697+
}
679698
}
680699
}
681700

@@ -1522,7 +1541,7 @@ class GenericReader {
15221541
IterativeParsingErrorState, // Left bracket
15231542
IterativeParsingErrorState, // Right bracket
15241543
IterativeParsingErrorState, // Left curly bracket
1525-
IterativeParsingErrorState, // Right curly bracket
1544+
IterativeParsingObjectFinishState, // Right curly bracket
15261545
IterativeParsingErrorState, // Comma
15271546
IterativeParsingErrorState, // Colon
15281547
IterativeParsingMemberKeyState, // String
@@ -1568,7 +1587,7 @@ class GenericReader {
15681587
// ElementDelimiter
15691588
{
15701589
IterativeParsingArrayInitialState, // Left bracket(push Element state)
1571-
IterativeParsingErrorState, // Right bracket
1590+
IterativeParsingArrayFinishState, // Right bracket
15721591
IterativeParsingObjectInitialState, // Left curly bracket(push Element state)
15731592
IterativeParsingErrorState, // Right curly bracket
15741593
IterativeParsingErrorState, // Comma
@@ -1670,6 +1689,11 @@ class GenericReader {
16701689

16711690
case IterativeParsingObjectFinishState:
16721691
{
1692+
// Transit from delimiter is only allowed when trailing commas are enabled
1693+
if (!(parseFlags & kParseTrailingCommasFlag) && src == IterativeParsingMemberDelimiterState) {
1694+
RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorObjectMissName, is.Tell());
1695+
return IterativeParsingErrorState;
1696+
}
16731697
// Get member count.
16741698
SizeType c = *stack_.template Pop<SizeType>(1);
16751699
// If the object is not empty, count the last member.
@@ -1695,6 +1719,11 @@ class GenericReader {
16951719

16961720
case IterativeParsingArrayFinishState:
16971721
{
1722+
// Transit from delimiter is only allowed when trailing commas are enabled
1723+
if (!(parseFlags & kParseTrailingCommasFlag) && src == IterativeParsingElementDelimiterState) {
1724+
RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorValueInvalid, is.Tell());
1725+
return IterativeParsingErrorState;
1726+
}
16981727
// Get element count.
16991728
SizeType c = *stack_.template Pop<SizeType>(1);
17001729
// If the array is not empty, count the last element.
@@ -1754,6 +1783,9 @@ class GenericReader {
17541783
case IterativeParsingMemberDelimiterState: RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissName, is.Tell()); return;
17551784
case IterativeParsingMemberKeyState: RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissColon, is.Tell()); return;
17561785
case IterativeParsingMemberValueState: RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissCommaOrCurlyBracket, is.Tell()); return;
1786+
case IterativeParsingKeyValueDelimiterState:
1787+
case IterativeParsingArrayInitialState:
1788+
case IterativeParsingElementDelimiterState: RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, is.Tell()); return;
17571789
case IterativeParsingElementState: RAPIDJSON_PARSE_ERROR(kParseErrorArrayMissCommaOrSquareBracket, is.Tell()); return;
17581790
default: RAPIDJSON_PARSE_ERROR(kParseErrorUnspecificSyntaxError, is.Tell()); return;
17591791
}

‎test/unittest/readertest.cpp‎

Lines changed: 161 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -778,6 +778,10 @@ TEST(Reader, ParseArray_Error) {
778778
TEST_ARRAY_ERROR(kParseErrorArrayMissCommaOrSquareBracket, "[1}", 2);
779779
TEST_ARRAY_ERROR(kParseErrorArrayMissCommaOrSquareBracket, "[1 2]", 3);
780780

781+
// Array cannot have a trailing comma (without kParseTrailingCommasFlag);
782+
// a value must follow a comma
783+
TEST_ARRAY_ERROR(kParseErrorValueInvalid, "[1,]", 3);
784+
781785
#undef TEST_ARRAY_ERROR
782786
}
783787

@@ -978,6 +982,10 @@ TEST(Reader, ParseObject_Error) {
978982
// Must be a comma or '}' after an object member
979983
TEST_ERROR(kParseErrorObjectMissCommaOrCurlyBracket, "{\"a\":1]", 6);
980984

985+
// Object cannot have a trailing comma (without kParseTrailingCommasFlag);
986+
// an object member name must follow a comma
987+
TEST_ERROR(kParseErrorObjectMissName, "{\"a\":1,}", 7);
988+
981989
// This tests that MemoryStream is checking the length in Peek().
982990
{
983991
MemoryStream ms("{\"a\"", 1);
@@ -1119,6 +1127,16 @@ TEST(Reader, IterativeParsing_ErrorHandling) {
11191127
TESTERRORHANDLING("{\"a\": 1", kParseErrorObjectMissCommaOrCurlyBracket, 7u);
11201128
TESTERRORHANDLING("[1 2 3]", kParseErrorArrayMissCommaOrSquareBracket, 3u);
11211129
TESTERRORHANDLING("{\"a: 1", kParseErrorStringMissQuotationMark, 6u);
1130+
TESTERRORHANDLING("{\"a\":}", kParseErrorValueInvalid, 5u);
1131+
TESTERRORHANDLING("{\"a\":]", kParseErrorValueInvalid, 5u);
1132+
TESTERRORHANDLING("[1,2,}", kParseErrorValueInvalid, 5u);
1133+
TESTERRORHANDLING("[}]", kParseErrorValueInvalid, 1u);
1134+
TESTERRORHANDLING("[,]", kParseErrorValueInvalid, 1u);
1135+
TESTERRORHANDLING("[1,,]", kParseErrorValueInvalid, 3u);
1136+
1137+
// Trailing commas are not allowed without kParseTrailingCommasFlag
1138+
TESTERRORHANDLING("{\"a\": 1,}", kParseErrorObjectMissName, 8u);
1139+
TESTERRORHANDLING("[1,2,3,]", kParseErrorValueInvalid, 7u);
11221140

11231141
// Any JSON value can be a valid root element in RFC7159.
11241142
TESTERRORHANDLING("\"ab", kParseErrorStringMissQuotationMark, 3u);
@@ -1552,6 +1570,149 @@ TEST(Reader, NumbersAsStrings) {
15521570
}
15531571
}
15541572

1573+
template <unsigned extraFlags>
1574+
void TestTrailingCommas() {
1575+
{
1576+
StringStream s("[1,2,3,]");
1577+
ParseArrayHandler<3> h;
1578+
Reader reader;
1579+
EXPECT_TRUE(reader.Parse<extraFlags|kParseTrailingCommasFlag>(s, h));
1580+
EXPECT_EQ(5u, h.step_);
1581+
}
1582+
{
1583+
const char* json = "{ \"hello\" : \"world\", \"t\" : true , \"f\" : false,"
1584+
"\"n\": null, \"i\":123, \"pi\": 3.1416, \"a\":[1, 2, 3],}";
1585+
StringStream s(json);
1586+
ParseObjectHandler h;
1587+
Reader reader;
1588+
EXPECT_TRUE(reader.Parse<extraFlags|kParseTrailingCommasFlag>(s, h));
1589+
EXPECT_EQ(20u, h.step_);
1590+
}
1591+
{
1592+
// whitespace around trailing commas
1593+
const char* json = "{ \"hello\" : \"world\", \"t\" : true , \"f\" : false,"
1594+
"\"n\": null, \"i\":123, \"pi\": 3.1416, \"a\":[1, 2, 3\n,\n]\n,\n} ";
1595+
StringStream s(json);
1596+
ParseObjectHandler h;
1597+
Reader reader;
1598+
EXPECT_TRUE(reader.Parse<extraFlags|kParseTrailingCommasFlag>(s, h));
1599+
EXPECT_EQ(20u, h.step_);
1600+
}
1601+
{
1602+
// comments around trailing commas
1603+
const char* json = "{ \"hello\" : \"world\", \"t\" : true , \"f\" : false, \"n\": null,"
1604+
"\"i\":123, \"pi\": 3.1416, \"a\":[1, 2, 3/*test*/,/*test*/]/*test*/,/*test*/}";
1605+
StringStream s(json);
1606+
ParseObjectHandler h;
1607+
Reader reader;
1608+
EXPECT_TRUE(reader.Parse<extraFlags|kParseTrailingCommasFlag|kParseCommentsFlag>(s, h));
1609+
EXPECT_EQ(20u, h.step_);
1610+
}
1611+
}
1612+
1613+
TEST(Reader, TrailingCommas) {
1614+
TestTrailingCommas<kParseNoFlags>();
1615+
}
1616+
1617+
TEST(Reader, TrailingCommasIterative) {
1618+
TestTrailingCommas<kParseIterativeFlag>();
1619+
}
1620+
1621+
template <unsigned extraFlags>
1622+
void TestMultipleTrailingCommaErrors() {
1623+
// only a single trailing comma is allowed.
1624+
{
1625+
StringStream s("[1,2,3,,]");
1626+
ParseArrayHandler<3> h;
1627+
Reader reader;
1628+
ParseResult r = reader.Parse<extraFlags|kParseTrailingCommasFlag>(s, h);
1629+
EXPECT_TRUE(reader.HasParseError());
1630+
EXPECT_EQ(kParseErrorValueInvalid, r.Code());
1631+
EXPECT_EQ(7u, r.Offset());
1632+
}
1633+
{
1634+
const char* json = "{ \"hello\" : \"world\", \"t\" : true , \"f\" : false,"
1635+
"\"n\": null, \"i\":123, \"pi\": 3.1416, \"a\":[1, 2, 3,],,}";
1636+
StringStream s(json);
1637+
ParseObjectHandler h;
1638+
Reader reader;
1639+
ParseResult r = reader.Parse<extraFlags|kParseTrailingCommasFlag>(s, h);
1640+
EXPECT_TRUE(reader.HasParseError());
1641+
EXPECT_EQ(kParseErrorObjectMissName, r.Code());
1642+
EXPECT_EQ(95u, r.Offset());
1643+
}
1644+
}
1645+
1646+
TEST(Reader, MultipleTrailingCommaErrors) {
1647+
TestMultipleTrailingCommaErrors<kParseNoFlags>();
1648+
}
1649+
1650+
TEST(Reader, MultipleTrailingCommaErrorsIterative) {
1651+
TestMultipleTrailingCommaErrors<kParseIterativeFlag>();
1652+
}
1653+
1654+
template <unsigned extraFlags>
1655+
void TestEmptyExceptForCommaErrors() {
1656+
// not allowed even with trailing commas enabled; the
1657+
// trailing comma must follow a value.
1658+
{
1659+
StringStream s("[,]");
1660+
ParseArrayHandler<3> h;
1661+
Reader reader;
1662+
ParseResult r = reader.Parse<extraFlags|kParseTrailingCommasFlag>(s, h);
1663+
EXPECT_TRUE(reader.HasParseError());
1664+
EXPECT_EQ(kParseErrorValueInvalid, r.Code());
1665+
EXPECT_EQ(1u, r.Offset());
1666+
}
1667+
{
1668+
StringStream s("{,}");
1669+
ParseObjectHandler h;
1670+
Reader reader;
1671+
ParseResult r = reader.Parse<extraFlags|kParseTrailingCommasFlag>(s, h);
1672+
EXPECT_TRUE(reader.HasParseError());
1673+
EXPECT_EQ(kParseErrorObjectMissName, r.Code());
1674+
EXPECT_EQ(1u, r.Offset());
1675+
}
1676+
}
1677+
1678+
TEST(Reader, EmptyExceptForCommaErrors) {
1679+
TestEmptyExceptForCommaErrors<kParseNoFlags>();
1680+
}
1681+
1682+
TEST(Reader, EmptyExceptForCommaErrorsIterative) {
1683+
TestEmptyExceptForCommaErrors<kParseIterativeFlag>();
1684+
}
1685+
1686+
template <unsigned extraFlags>
1687+
void TestTrailingCommaHandlerTermination() {
1688+
{
1689+
HandlerTerminateAtEndArray h;
1690+
Reader reader;
1691+
StringStream s("[1,2,3,]");
1692+
ParseResult r = reader.Parse<extraFlags|kParseTrailingCommasFlag>(s, h);
1693+
EXPECT_TRUE(reader.HasParseError());
1694+
EXPECT_EQ(kParseErrorTermination, r.Code());
1695+
EXPECT_EQ(7u, r.Offset());
1696+
}
1697+
{
1698+
HandlerTerminateAtEndObject h;
1699+
Reader reader;
1700+
StringStream s("{\"t\": true, \"f\": false,}");
1701+
ParseResult r = reader.Parse<extraFlags|kParseTrailingCommasFlag>(s, h);
1702+
EXPECT_TRUE(reader.HasParseError());
1703+
EXPECT_EQ(kParseErrorTermination, r.Code());
1704+
EXPECT_EQ(23u, r.Offset());
1705+
}
1706+
}
1707+
1708+
TEST(Reader, TrailingCommaHandlerTermination) {
1709+
TestTrailingCommaHandlerTermination<kParseNoFlags>();
1710+
}
1711+
1712+
TEST(Reader, TrailingCommaHandlerTerminationIterative) {
1713+
TestTrailingCommaHandlerTermination<kParseIterativeFlag>();
1714+
}
1715+
15551716
#ifdef __GNUC__
15561717
RAPIDJSON_DIAG_POP
15571718
#endif

0 commit comments

Comments
 (0)