Merge pull request globalsign#152 from globalsign/release/r2018.04.23

Release/r2018.04.23

Author: domodwyer

.gitignore

@@ -1,2 +1,2 @@
 _harness
-
+.vscode

.travis.yml

@@ -3,8 +3,8 @@ language: go
 go_import_path: github.com/globalsign/mgo
 
 go:
-    - 1.8.x
     - 1.9.x
+    - 1.10.x
 
 env:
     global:
@@ -29,7 +29,7 @@ install:
     - go get gopkg.in/check.v1
     - go get gopkg.in/yaml.v2
     - go get gopkg.in/tomb.v2
-    - go get github.com/golang/lint/golint
+    - go get github.com/golang/lint
 
 before_script:
     - golint  ./... | grep -v 'ID' | cat

README.md

@@ -3,10 +3,16 @@
 The MongoDB driver for Go
 -------------------------
 
-This fork has had a few improvements by ourselves as well as several PR's merged from the original mgo repo that are currently awaiting review. Changes are mostly geared towards performance improvements and bug fixes, though a few new features have been added.
+This fork has had a few improvements by ourselves as well as several PR's merged from the original mgo repo that are currently awaiting review.
+Changes are mostly geared towards performance improvements and bug fixes, though a few new features have been added.
 
 Further PR's (with tests) are welcome, but please maintain backwards compatibility.
 
+Detailed documentation of the API is available at
+[GoDoc](https://godoc.org/github.com/globalsign/mgo).
+
+A [sub-package](https://godoc.org/github.com/globalsign/mgo/bson) that implements the [BSON](http://bsonspec.org) specification is also included, and may be used independently of the driver.
+
 ## Changes
 * Fixes attempting to authenticate before every query ([details](https://github.com/go-mgo/mgo/issues/254))
 * Removes bulk update / delete batch size limitations ([details](https://github.com/go-mgo/mgo/issues/288))
@@ -15,13 +21,13 @@ Further PR's (with tests) are welcome, but please maintain backwards compatibili
 * Support majority read concerns ([details](https://github.com/globalsign/mgo/pull/2))
 * Improved connection handling ([details](https://github.com/globalsign/mgo/pull/5))
 * Hides SASL warnings ([details](https://github.com/globalsign/mgo/pull/7))
-* Support for partial indexes ([detials](https://github.com/domodwyer/mgo/commit/5efe8eccb028238d93c222828cae4806aeae9f51))
-* Fixes timezone handling ([details](https://github.com/go-mgo/mgo/pull/464)) 
+* Support for partial indexes ([details](https://github.com/domodwyer/mgo/commit/5efe8eccb028238d93c222828cae4806aeae9f51))
+* Fixes timezone handling ([details](https://github.com/go-mgo/mgo/pull/464))
 * Integration tests run against MongoDB 3.2 & 3.4 releases ([details](https://github.com/globalsign/mgo/pull/4), [more](https://github.com/globalsign/mgo/pull/24), [more](https://github.com/globalsign/mgo/pull/35))
 * Improved multi-document transaction performance ([details](https://github.com/globalsign/mgo/pull/10), [more](https://github.com/globalsign/mgo/pull/11), [more](https://github.com/globalsign/mgo/pull/16))
 * Fixes cursor timeouts ([details](https://jira.mongodb.org/browse/SERVER-24899))
 * Support index hints and timeouts for count queries ([details](https://github.com/globalsign/mgo/pull/17))
-* Don't panic when handling indexed `int64` fields ([detials](https://github.com/go-mgo/mgo/issues/475))
+* Don't panic when handling indexed `int64` fields ([details](https://github.com/go-mgo/mgo/issues/475))
 * Supports dropping all indexes on a collection ([details](https://github.com/globalsign/mgo/pull/25))
 * Annotates log entries/profiler output with optional appName on 3.4+ ([details](https://github.com/globalsign/mgo/pull/28))
 * Support for read-only [views](https://docs.mongodb.com/manual/core/views/) in 3.4+ ([details](https://github.com/globalsign/mgo/pull/33))
@@ -37,27 +43,39 @@ Further PR's (with tests) are welcome, but please maintain backwards compatibili
 * Use JSON tags when no explicit BSON are tags set ([details](https://github.com/globalsign/mgo/pull/91))
 * Support [$changeStream](https://docs.mongodb.com/manual/changeStreams/) tailing on 3.6+ ([details](https://github.com/globalsign/mgo/pull/97))
 * Fix deadlock in cluster synchronisation ([details](https://github.com/globalsign/mgo/issues/120))
+* Implement `maxIdleTimeout` for pooled connections ([details](https://github.com/globalsign/mgo/pull/116))
+* Connection pool waiting improvements ([details](https://github.com/globalsign/mgo/pull/115))
+* Fixes BSON encoding for `$in` and friends ([details](https://github.com/globalsign/mgo/pull/128))
+* Add BSON stream encoders ([details](https://github.com/globalsign/mgo/pull/127))
+* Add integer map key support in the BSON encoder ([details](https://github.com/globalsign/mgo/pull/140)) 
+* Support aggregation [collations](https://docs.mongodb.com/manual/reference/collation/) ([details](https://github.com/globalsign/mgo/pull/144))
 
 ---
 
 ### Thanks to
+* @aksentyev
 * @bachue
 * @bozaro
 * @BenLubar
+* @carldunham
 * @carter2000
 * @cezarsa
 * @drichelson
 * @dvic
 * @eaglerayp
 * @feliixx
 * @fmpwizard
+* @gazoon
+* @gnawux
 * @idy
 * @jameinel
+* @johnlawsharrison
 * @KJTsanaktsidis
-* @gazoon
 * @mapete94
+* @maxnoel
+* @mcspring
 * @peterdeka
 * @Reenjii
 * @smoya
 * @steve-gray
-* @wgallagher
+* @wgallagher

bson/README.md

@@ -0,0 +1,12 @@
+[![GoDoc](https://godoc.org/github.com/globalsign/mgo/bson?status.svg)](https://godoc.org/github.com/globalsign/mgo/bson)
+
+An Implementation of BSON for Go
+--------------------------------
+
+Package bson is an implementation of the [BSON specification](http://bsonspec.org) for Go.
+
+While the BSON package implements the BSON spec as faithfully as possible, there
+is some MongoDB specific behaviour (such as map keys `$in`, `$all`, etc) in the
+`bson` package. The priority is for backwards compatibility for the `mgo`
+driver, though fixes for obviously buggy behaviour is welcome (and features, etc
+behind feature flags).

bson/bson_test.go

@@ -34,9 +34,9 @@ import (
 	"errors"
 	"net/url"
 	"reflect"
+	"strings"
 	"testing"
 	"time"
-	"strings"
 
 	"github.com/globalsign/mgo/bson"
 	. "gopkg.in/check.v1"
@@ -111,6 +111,10 @@ var sampleItems = []testItemType{
 	{bson.M{"BSON": []interface{}{"awesome", float64(5.05), 1986}},
 		"1\x00\x00\x00\x04BSON\x00&\x00\x00\x00\x020\x00\x08\x00\x00\x00" +
 			"awesome\x00\x011\x00333333\x14@\x102\x00\xc2\x07\x00\x00\x00\x00"},
+	{bson.M{"slice": []uint8{1, 2}},
+		"\x13\x00\x00\x00\x05slice\x00\x02\x00\x00\x00\x00\x01\x02\x00"},
+	{bson.M{"slice": []byte{1, 2}},
+		"\x13\x00\x00\x00\x05slice\x00\x02\x00\x00\x00\x00\x01\x02\x00"},
 }
 
 func (s *S) TestMarshalSampleItems(c *C) {
@@ -343,6 +347,27 @@ func (s *S) TestOneWayMarshalItems(c *C) {
 	}
 }
 
+// --------------------------------------------------------------------------
+// Some ops marshaling operations which would encode []uint8 or []byte in array.
+
+var arrayOpsMarshalItems = []testItemType{
+	{bson.M{"_": bson.M{"$in": []uint8{1, 2}}},
+		"\x03_\x00\x1d\x00\x00\x00\x04$in\x00\x13\x00\x00\x00\x100\x00\x01\x00\x00\x00\x101\x00\x02\x00\x00\x00\x00\x00"},
+	{bson.M{"_": bson.M{"$nin": []uint8{1, 2}}},
+		"\x03_\x00\x1e\x00\x00\x00\x04$nin\x00\x13\x00\x00\x00\x100\x00\x01\x00\x00\x00\x101\x00\x02\x00\x00\x00\x00\x00"},
+	{bson.M{"_": bson.M{"$all": []uint8{1, 2}}},
+		"\x03_\x00\x1e\x00\x00\x00\x04$all\x00\x13\x00\x00\x00\x100\x00\x01\x00\x00\x00\x101\x00\x02\x00\x00\x00\x00\x00"},
+}
+
+func (s *S) TestArrayOpsMarshalItems(c *C) {
+	for i, item := range arrayOpsMarshalItems {
+		data, err := bson.Marshal(item.obj)
+		c.Assert(err, IsNil)
+		c.Assert(string(data), Equals, wrapInDoc(item.data),
+			Commentf("Failed on item %d", i))
+	}
+}
+
 // --------------------------------------------------------------------------
 // Two-way tests for user-defined structures using the samples
 // from bsonspec.org.
@@ -582,6 +607,8 @@ func (s *S) TestMarshalOneWayItems(c *C) {
 // --------------------------------------------------------------------------
 // One-way unmarshaling tests.
 
+type intAlias int
+
 var unmarshalItems = []testItemType{
 	// Field is private.  Should not attempt to unmarshal it.
 	{&struct{ priv byte }{},
@@ -636,6 +663,14 @@ var unmarshalItems = []testItemType{
 	// Decode a doc within a doc in to a slice within a doc; shouldn't error
 	{&struct{ Foo []string }{},
 		"\x03\x66\x6f\x6f\x00\x05\x00\x00\x00\x00"},
+
+	// int key maps
+	{map[int]string{10: "s"},
+		"\x0210\x00\x02\x00\x00\x00s\x00"},
+
+	//// event if type is alias to int
+	{map[intAlias]string{10: "s"},
+		"\x0210\x00\x02\x00\x00\x00s\x00"},
 }
 
 func (s *S) TestUnmarshalOneWayItems(c *C) {
@@ -713,11 +748,6 @@ var unmarshalErrorItems = []unmarshalErrorType{
 		"\x10name\x00\x08\x00\x00\x00",
 		"Duplicated key 'name' in struct bson_test.structWithDupKeys"},
 
-	// Non-string map key.
-	{map[int]interface{}{},
-		"\x10name\x00\x08\x00\x00\x00",
-		"BSON map must have string keys. Got: map\\[int\\]interface \\{\\}"},
-
 	{nil,
 		"\xEEname\x00",
 		"Unknown element kind \\(0xEE\\)"},
@@ -733,6 +763,11 @@ var unmarshalErrorItems = []unmarshalErrorType{
 	{nil,
 		"\x08\x62\x00\x02",
 		"encoded boolean must be 1 or 0, found 2"},
+
+	// Non-string and not numeric map key.
+	{map[bool]interface{}{true: 1},
+		"\x10true\x00\x01\x00\x00\x00",
+		"BSON map must have string or decimal keys. Got: map\\[bool\\]interface \\{\\}"},
 }
 
 func (s *S) TestUnmarshalErrorItems(c *C) {
@@ -1136,8 +1171,8 @@ type inlineBadKeyMap struct {
 	M map[int]int `bson:",inline"`
 }
 type inlineUnexported struct {
-	M          map[string]interface{} `bson:",inline"`
-	unexported `bson:",inline"`
+	M map[string]interface{} `bson:",inline"`
+	unexported               `bson:",inline"`
 }
 type unexported struct {
 	A int
@@ -1194,11 +1229,11 @@ func (s ifaceSlice) GetBSON() (interface{}, error) {
 
 type (
 	MyString string
-	MyBytes  []byte
-	MyBool   bool
-	MyD      []bson.DocElem
-	MyRawD   []bson.RawDocElem
-	MyM      map[string]interface{}
+	MyBytes []byte
+	MyBool bool
+	MyD []bson.DocElem
+	MyRawD []bson.RawDocElem
+	MyM map[string]interface{}
 )
 
 var (

bson/decode.go

@@ -176,9 +176,6 @@ func (d *decoder) readDocTo(out reflect.Value) {
 	switch outk {
 	case reflect.Map:
 		keyType = outt.Key()
-		if keyType.Kind() != reflect.String {
-			panic("BSON map must have string keys. Got: " + outt.String())
-		}
 		if keyType != typeString {
 			convertKey = true
 		}
@@ -240,7 +237,42 @@ func (d *decoder) readDocTo(out reflect.Value) {
 			if d.readElemTo(e, kind) {
 				k := reflect.ValueOf(name)
 				if convertKey {
-					k = k.Convert(keyType)
+					mapKeyType := out.Type().Key()
+					mapKeyKind := mapKeyType.Kind()
+
+					switch mapKeyKind {
+					case reflect.Int:
+						fallthrough
+					case reflect.Int8:
+						fallthrough
+					case reflect.Int16:
+						fallthrough
+					case reflect.Int32:
+						fallthrough
+					case reflect.Int64:
+						fallthrough
+					case reflect.Uint:
+						fallthrough
+					case reflect.Uint8:
+						fallthrough
+					case reflect.Uint16:
+						fallthrough
+					case reflect.Uint32:
+						fallthrough
+					case reflect.Uint64:
+						fallthrough
+					case reflect.Float32:
+						fallthrough
+					case reflect.Float64:
+						parsed := d.parseMapKeyAsFloat(k, mapKeyKind)
+						k = reflect.ValueOf(parsed)
+					case reflect.String:
+						mapKeyType = keyType
+					default:
+						panic("BSON map must have string or decimal keys. Got: " + outt.String())
+					}
+
+					k = k.Convert(mapKeyType)
 				}
 				out.SetMapIndex(k, e)
 			}
@@ -276,6 +308,16 @@ func (d *decoder) readDocTo(out reflect.Value) {
 	d.docType = docType
 }
 
+func (decoder) parseMapKeyAsFloat(k reflect.Value, mapKeyKind reflect.Kind) float64 {
+	parsed, err := strconv.ParseFloat(k.String(), 64)
+	if err != nil {
+		panic("Map key is defined to be a decimal type (" + mapKeyKind.String() + ") but got error " +
+			err.Error())
+	}
+
+	return parsed
+}
+
 func (d *decoder) readArrayDocTo(out reflect.Value) {
 	end := int(d.readInt32())
 	end += d.i - 4

bson/encode.go

@@ -60,6 +60,15 @@ var (
 	typeTimeDuration   = reflect.TypeOf(time.Duration(0))
 )
 
+var (
+	// spec for []uint8 or []byte encoding
+	arrayOps = map[string]bool{
+		"$in":  true,
+		"$nin": true,
+		"$all": true,
+	}
+)
+
 const itoaCacheSize = 32
 
 const (
@@ -194,7 +203,7 @@ func (e *encoder) addDoc(v reflect.Value) {
 
 func (e *encoder) addMap(v reflect.Value) {
 	for _, k := range v.MapKeys() {
-		e.addElem(k.String(), v.MapIndex(k), false)
+		e.addElem(fmt.Sprint(k), v.MapIndex(k), false)
 	}
 }
 
@@ -423,8 +432,13 @@ func (e *encoder) addElem(name string, v reflect.Value, minSize bool) {
 		vt := v.Type()
 		et := vt.Elem()
 		if et.Kind() == reflect.Uint8 {
-			e.addElemName(0x05, name)
-			e.addBinary(0x00, v.Bytes())
+			if arrayOps[name] {
+				e.addElemName(0x04, name)
+				e.addDoc(v)
+			} else {
+				e.addElemName(0x05, name)
+				e.addBinary(0x00, v.Bytes())
+			}
 		} else if et == typeDocElem || et == typeRawDocElem {
 			e.addElemName(0x03, name)
 			e.addDoc(v)
@@ -436,16 +450,21 @@ func (e *encoder) addElem(name string, v reflect.Value, minSize bool) {
 	case reflect.Array:
 		et := v.Type().Elem()
 		if et.Kind() == reflect.Uint8 {
-			e.addElemName(0x05, name)
-			if v.CanAddr() {
-				e.addBinary(0x00, v.Slice(0, v.Len()).Interface().([]byte))
+			if arrayOps[name] {
+				e.addElemName(0x04, name)
+				e.addDoc(v)
 			} else {
-				n := v.Len()
-				e.addInt32(int32(n))
-				e.addBytes(0x00)
-				for i := 0; i < n; i++ {
-					el := v.Index(i)
-					e.addBytes(byte(el.Uint()))
+				e.addElemName(0x05, name)
+				if v.CanAddr() {
+					e.addBinary(0x00, v.Slice(0, v.Len()).Interface().([]byte))
+				} else {
+					n := v.Len()
+					e.addInt32(int32(n))
+					e.addBytes(0x00)
+					for i := 0; i < n; i++ {
+						el := v.Index(i)
+						e.addBytes(byte(el.Uint()))
+					}
 				}
 			}
 		} else {