Merge pull request #51 from pelletier/pelletier/fix-crlf-support

Fix support for CRLF line ending

Author: pelletier

example-crlf.toml

@@ -0,0 +1,29 @@
+# This is a TOML document. Boom.
+
+title = "TOML Example"
+
+[owner]
+name = "Tom Preston-Werner"
+organization = "GitHub"
+bio = "GitHub Cofounder & CEO\nLikes tater tots and beer."
+dob = 1979-05-27T07:32:00Z # First class dates? Why not?
+
+[database]
+server = "192.168.1.1"
+ports = [ 8001, 8001, 8002 ]
+connection_max = 5000
+enabled = true
+
+[servers]
+
+  # You can indent as you please. Tabs or spaces. TOML don't care.
+  [servers.alpha]
+  ip = "10.0.0.1"
+  dc = "eqdc10"
+
+  [servers.beta]
+  ip = "10.0.0.2"
+  dc = "eqdc10"
+
+[clients]
+data = [ ["gamma", "delta"], [1, 2] ] # just an update to make sure parsers support it

lexer.go

@@ -132,6 +132,8 @@ func (l *tomlLexer) lexVoid() tomlLexStateFn {
 			return l.lexComment
 		case '=':
 			return l.lexEqual
+		case '\r':
+			fallthrough
 		case '\n':
 			l.skip()
 			continue
@@ -185,6 +187,8 @@ func (l *tomlLexer) lexRvalue() tomlLexStateFn {
 			return l.lexLiteralString
 		case ',':
 			return l.lexComma
+		case '\r':
+			fallthrough
 		case '\n':
 			l.skip()
 			if l.depth == 0 {
@@ -277,7 +281,7 @@ func (l *tomlLexer) lexComma() tomlLexStateFn {
 
 func (l *tomlLexer) lexKey() tomlLexStateFn {
 	inQuotes := false
-	for r := l.peek(); isKeyChar(r) || r == '\n'; r = l.peek() {
+	for r := l.peek(); isKeyChar(r) || r == '\n' || r == '\r'; r = l.peek() {
 		if r == '"' {
 			inQuotes = !inQuotes
 		} else if r == '\n' {
@@ -295,6 +299,9 @@ func (l *tomlLexer) lexKey() tomlLexStateFn {
 
 func (l *tomlLexer) lexComment() tomlLexStateFn {
 	for next := l.peek(); next != '\n' && next != eof; next = l.peek() {
+		if (next == '\r' && l.follow("\r\n")) {
+			break
+		}
 		l.next()
 	}
 	l.ignore()
@@ -319,7 +326,10 @@ func (l *tomlLexer) lexLiteralString() tomlLexStateFn {
 		terminator = "'''"
 
 		// special case: discard leading newline
-		if l.peek() == '\n' {
+		if l.follow("\r\n") {
+			l.skip()
+			l.skip()
+		} else if l.peek() == '\n' {
 			l.skip()
 		}
 	}
@@ -355,7 +365,10 @@ func (l *tomlLexer) lexString() tomlLexStateFn {
 		terminator = "\"\"\""
 
 		// special case: discard leading newline
-		if l.peek() == '\n' {
+		if l.follow("\r\n") {
+			l.skip()
+			l.skip()
+		} else if l.peek() == '\n' {
 			l.skip()
 		}
 	}

lexer_test.go

@@ -87,6 +87,19 @@ func TestMultipleKeyGroupsComment(t *testing.T) {
 	})
 }
 
+
+func TestSimpleWindowsCRLF(t *testing.T) {
+	testFlow(t, "a=4\r\nb=2", []token{
+		token{Position{1, 1}, tokenKey, "a"},
+		token{Position{1, 2}, tokenEqual, "="},
+		token{Position{1, 3}, tokenInteger, "4"},
+		token{Position{2, 1}, tokenKey, "b"},
+		token{Position{2, 2}, tokenEqual, "="},
+		token{Position{2, 3}, tokenInteger, "2"},
+		token{Position{2, 4}, tokenEOF, ""},
+	})
+}
+
 func TestBasicKey(t *testing.T) {
 	testFlow(t, "hello", []token{
 		token{Position{1, 1}, tokenKey, "hello"},

parser_test.go

@@ -494,6 +494,42 @@ func TestParseFile(t *testing.T) {
 	})
 }
 
+func TestParseFileCRLF(t *testing.T) {
+	tree, err := LoadFile("example-crlf.toml")
+
+	assertTree(t, tree, err, map[string]interface{}{
+		"title": "TOML Example",
+		"owner": map[string]interface{}{
+			"name":         "Tom Preston-Werner",
+			"organization": "GitHub",
+			"bio":          "GitHub Cofounder & CEO\nLikes tater tots and beer.",
+			"dob":          time.Date(1979, time.May, 27, 7, 32, 0, 0, time.UTC),
+		},
+		"database": map[string]interface{}{
+			"server":         "192.168.1.1",
+			"ports":          []int64{8001, 8001, 8002},
+			"connection_max": 5000,
+			"enabled":        true,
+		},
+		"servers": map[string]interface{}{
+			"alpha": map[string]interface{}{
+				"ip": "10.0.0.1",
+				"dc": "eqdc10",
+			},
+			"beta": map[string]interface{}{
+				"ip": "10.0.0.2",
+				"dc": "eqdc10",
+			},
+		},
+		"clients": map[string]interface{}{
+			"data": []interface{}{
+				[]string{"gamma", "delta"},
+				[]int64{1, 2},
+			},
+		},
+	})
+}
+
 func TestParseKeyGroupArray(t *testing.T) {
 	tree, err := Load("[[foo.bar]] a = 42\n[[foo.bar]] a = 69")
 	assertTree(t, tree, err, map[string]interface{}{