Skip to content

Commit 94ac941

Browse files
kshvakovkshvakov
committed
LowCardinality(Nullable(T))
1 parent d562eb4 commit 94ac941

File tree

8 files changed

+238
-57
lines changed

8 files changed

+238
-57
lines changed

‎README.md

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,6 @@ clickhouse://username:password@host1:9000,host2:9000/database?dial_timeout=200ms
5454

5555
## TODO
5656

57-
* [ ] LowCardinality
5857
* [ ] Bigint types
5958
* [ ] ZSTD
6059
* [ ] Geo
@@ -150,7 +149,7 @@ func example() error {
150149
return err
151150
}
152151

153-
rows, err := conn.Query(ctx, "SELECT Col1, Col2, Col3 FROM example")
152+
rows, err := conn.Query(ctx, "SELECT Col1, Col2, Col3 FROM example WHERE Col1 >= $1 AND Col2 <> $2", 0, "xxx")
154153
if err != nil {
155154
return err
156155
}
@@ -241,7 +240,7 @@ func example() error {
241240
if err := scope.Commit(); err != nil {
242241
return err
243242
}
244-
rows, err := conn.QueryContext(ctx, "SELECT Col1, Col2, Col3 FROM example")
243+
rows, err := conn.QueryContext(ctx, "SELECT Col1, Col2, Col3 FROM example WHERE Col1 >= $1 AND Col2 <> $2", 0, "xxx")
245244
if err != nil {
246245
return err
247246
}

‎lib/column/array.go

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -127,9 +127,10 @@ func (col *Array) AppendRow(v interface{}) error {
127127
from = fmt.Sprintf("%v", v)
128128
}
129129
return &ColumnConverterErr{
130-
op: "AppendRow",
131-
to: string(col.chType),
132-
from: from,
130+
op: "AppendRow",
131+
to: string(col.chType),
132+
from: from,
133+
advise: fmt.Sprintf("use %s", col.scanType),
133134
}
134135
}
135136
return col.append(elem, 0)

‎lib/column/lowcardinality.go

Lines changed: 30 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -42,8 +42,10 @@ type LowCardinality struct {
4242
keys32 UInt32
4343
keys64 UInt64
4444

45-
tmpIdx map[interface{}]int
46-
tmpKey []int
45+
tmpIdx map[interface{}]int
46+
tmpKey []int
47+
rows int
48+
nullable bool
4749
}
4850

4951
func (col *LowCardinality) parse(t Type) (_ *LowCardinality, err error) {
@@ -52,6 +54,9 @@ func (col *LowCardinality) parse(t Type) (_ *LowCardinality, err error) {
5254
if col.index, err = Type(t.params()).Column(); err != nil {
5355
return nil, err
5456
}
57+
if nullable, ok := col.index.(*Nullable); ok {
58+
col.nullable, nullable.enable = true, false
59+
}
5560
return col, nil
5661
}
5762

@@ -64,18 +69,23 @@ func (col *LowCardinality) ScanType() reflect.Type {
6469
}
6570

6671
func (col *LowCardinality) Rows() int {
67-
if len(col.tmpKey) != 0 {
68-
return len(col.tmpKey)
69-
}
70-
return col.keys().Rows()
72+
return col.rows
7173
}
7274

7375
func (col *LowCardinality) Row(i int, ptr bool) interface{} {
74-
return col.index.Row(col.indexRowNum(i), ptr)
76+
idx := col.indexRowNum(i)
77+
if idx == 0 && col.nullable {
78+
return nil
79+
}
80+
return col.index.Row(idx, ptr)
7581
}
7682

7783
func (col *LowCardinality) ScanRow(dest interface{}, row int) error {
78-
return col.index.ScanRow(dest, col.indexRowNum(row))
84+
idx := col.indexRowNum(row)
85+
if idx == 0 && col.nullable {
86+
return nil
87+
}
88+
return col.index.ScanRow(dest, idx)
7989
}
8090

8191
func (col *LowCardinality) Append(v interface{}) (nulls []uint8, err error) {
@@ -96,21 +106,27 @@ func (col *LowCardinality) Append(v interface{}) (nulls []uint8, err error) {
96106
}
97107

98108
func (col *LowCardinality) AppendRow(v interface{}) error {
109+
col.rows++
110+
if col.index.Rows() == 0 { //init
111+
if col.index.AppendRow(nil); col.nullable {
112+
col.index.AppendRow(nil)
113+
}
114+
}
115+
if v == nil {
116+
col.tmpKey = append(col.tmpKey, 0)
117+
return nil
118+
}
99119
switch x := v.(type) {
100120
case time.Time:
101121
v = x.Truncate(time.Second)
102122
}
103123
if _, found := col.tmpIdx[v]; !found {
104-
if v == nil {
105-
return fmt.Errorf("clickhouse: LowCardinality does not support NULL values")
106-
}
107124
if err := col.index.AppendRow(v); err != nil {
108125
return err
109126
}
110127
col.tmpIdx[v] = col.index.Rows() - 1
111128
}
112129
col.tmpKey = append(col.tmpKey, col.tmpIdx[v])
113-
114130
return nil
115131
}
116132

@@ -148,10 +164,8 @@ func (col *LowCardinality) Decode(decoder *binary.Decoder, _ int) error {
148164
if err != nil {
149165
return err
150166
}
151-
if err := col.keys().Decode(decoder, int(keysRows)); err != nil {
152-
return err
153-
}
154-
return nil
167+
col.rows = int(keysRows)
168+
return col.keys().Decode(decoder, col.rows)
155169
}
156170

157171
func (col *LowCardinality) Encode(encoder *binary.Encoder) error {

‎lib/column/nullable.go

Lines changed: 22 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -9,10 +9,12 @@ import (
99
type Nullable struct {
1010
base Interface
1111
nulls UInt8
12+
enable bool
1213
scanType reflect.Type
1314
}
1415

1516
func (col *Nullable) parse(t Type) (_ *Nullable, err error) {
17+
col.enable = true
1618
if col.base, err = Type(t.params()).Column(); err != nil {
1719
return nil, err
1820
}
@@ -38,19 +40,26 @@ func (col *Nullable) ScanType() reflect.Type {
3840
}
3941

4042
func (col *Nullable) Rows() int {
43+
if !col.enable {
44+
return col.base.Rows()
45+
}
4146
return len(col.nulls)
4247
}
4348

4449
func (col *Nullable) Row(i int, ptr bool) interface{} {
45-
if col.nulls[i] == 1 {
46-
return nil
50+
if col.enable {
51+
if col.nulls[i] == 1 {
52+
return nil
53+
}
4754
}
48-
return col.base.Row(i, ptr)
55+
return col.base.Row(i, true)
4956
}
5057

5158
func (col *Nullable) ScanRow(dest interface{}, row int) error {
52-
if col.nulls[row] == 1 {
53-
return nil
59+
if col.enable {
60+
if col.nulls[row] == 1 {
61+
return nil
62+
}
5463
}
5564
return col.base.ScanRow(dest, row)
5665
}
@@ -75,8 +84,10 @@ func (col *Nullable) AppendRow(v interface{}) error {
7584
}
7685

7786
func (col *Nullable) Decode(decoder *binary.Decoder, rows int) (err error) {
78-
if err := col.nulls.Decode(decoder, rows); err != nil {
79-
return err
87+
if col.enable {
88+
if err := col.nulls.Decode(decoder, rows); err != nil {
89+
return err
90+
}
8091
}
8192
if err := col.base.Decode(decoder, rows); err != nil {
8293
return err
@@ -85,8 +96,10 @@ func (col *Nullable) Decode(decoder *binary.Decoder, rows int) (err error) {
8596
}
8697

8798
func (col *Nullable) Encode(encoder *binary.Encoder) error {
88-
if err := col.nulls.Encode(encoder); err != nil {
89-
return err
99+
if col.enable {
100+
if err := col.nulls.Encode(encoder); err != nil {
101+
return err
102+
}
90103
}
91104
if err := col.base.Encode(encoder); err != nil {
92105
return err

‎lib/column/tuple.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,7 @@ func (col *Tuple) Rows() int {
8080
func (col *Tuple) Row(i int, ptr bool) interface{} {
8181
tuple := make([]interface{}, 0, len(col.columns))
8282
for _, c := range col.columns {
83-
tuple = append(tuple, c.Row(i, false))
83+
tuple = append(tuple, c.Row(i, ptr))
8484
}
8585
return tuple
8686
}

‎tests/lowcardinality_test.go

Lines changed: 51 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,8 @@ func TestLowCardinality(t *testing.T) {
4242
, Col4 LowCardinality(Int32)
4343
, Col5 Array(LowCardinality(String))
4444
, Col6 Array(Array(LowCardinality(String)))
45+
, Col7 LowCardinality(Nullable(String))
46+
, Col8 Array(Array(LowCardinality(Nullable(String))))
4547
) Engine Memory
4648
`
4749
if err := conn.Exec(ctx, "DROP TABLE IF EXISTS test_lowcardinality"); assert.NoError(t, err) {
@@ -62,34 +64,60 @@ func TestLowCardinality(t *testing.T) {
6264
[]string{"Q", "W", "E"},
6365
[]string{"R", "T", "Y"},
6466
}
67+
col7Data = &col2Data
68+
col8Data = [][]*string{
69+
[]*string{&col2Data, nil, &col2Data},
70+
[]*string{nil, &col2Data, nil},
71+
}
6572
)
66-
if err := batch.Append(col1Data, col2Data, col3Data, col4Data, col5Data, col6Data); !assert.NoError(t, err) {
67-
return
73+
if i%2 == 0 {
74+
if err := batch.Append(col1Data, col2Data, col3Data, col4Data, col5Data, col6Data, col7Data, col8Data); !assert.NoError(t, err) {
75+
return
76+
}
77+
} else {
78+
if err := batch.Append(col1Data, col2Data, col3Data, col4Data, col5Data, col6Data, nil, col8Data); !assert.NoError(t, err) {
79+
return
80+
}
6881
}
6982
}
7083
if assert.NoError(t, batch.Send()) {
7184
var count uint64
7285
if err := conn.QueryRow(ctx, "SELECT COUNT() FROM test_lowcardinality").Scan(&count); assert.NoError(t, err) {
7386
assert.Equal(t, uint64(10), count)
7487
}
75-
var (
76-
col1 string
77-
col2 string
78-
col3 time.Time
79-
col4 int32
80-
col5 []string
81-
col6 [][]string
82-
)
83-
if err := conn.QueryRow(ctx, "SELECT * FROM test_lowcardinality WHERE Col4 = $1", rnd+6).Scan(&col1, &col2, &col3, &col4, &col5, &col6); assert.NoError(t, err) {
84-
assert.Equal(t, timestamp.String(), col1)
85-
assert.Equal(t, "RU", col2)
86-
assert.Equal(t, timestamp.Add(time.Duration(6)*time.Minute).Truncate(time.Second), col3)
87-
assert.Equal(t, int32(rnd+6), col4)
88-
assert.Equal(t, []string{"A", "B", "C"}, col5)
89-
assert.Equal(t, [][]string{
90-
[]string{"Q", "W", "E"},
91-
[]string{"R", "T", "Y"},
92-
}, col6)
88+
for i := 0; i < 10; i++ {
89+
var (
90+
col1 string
91+
col2 string
92+
col3 time.Time
93+
col4 int32
94+
col5 []string
95+
col6 [][]string
96+
col7 *string
97+
col8 [][]*string
98+
)
99+
if err := conn.QueryRow(ctx, "SELECT * FROM test_lowcardinality WHERE Col4 = $1", rnd+int32(i)).Scan(&col1, &col2, &col3, &col4, &col5, &col6, &col7, &col8); assert.NoError(t, err) {
100+
assert.Equal(t, timestamp.String(), col1)
101+
assert.Equal(t, "RU", col2)
102+
assert.Equal(t, timestamp.Add(time.Duration(i)*time.Minute).Truncate(time.Second), col3)
103+
assert.Equal(t, rnd+int32(i), col4)
104+
assert.Equal(t, []string{"A", "B", "C"}, col5)
105+
assert.Equal(t, [][]string{
106+
[]string{"Q", "W", "E"},
107+
[]string{"R", "T", "Y"},
108+
}, col6)
109+
switch {
110+
case i%2 == 0:
111+
assert.Equal(t, &col2, col7)
112+
default:
113+
assert.Nil(t, col7)
114+
}
115+
col2Data := "RU"
116+
assert.Equal(t, [][]*string{
117+
[]*string{&col2Data, nil, &col2Data},
118+
[]*string{nil, &col2Data, nil},
119+
}, col8)
120+
}
93121
}
94122
}
95123
}
@@ -98,7 +126,6 @@ func TestLowCardinality(t *testing.T) {
98126
}
99127
}
100128

101-
/*
102129
func TestNullableLowCardinality(t *testing.T) {
103130
var (
104131
ctx = context.Background()
@@ -132,13 +159,15 @@ func TestNullableLowCardinality(t *testing.T) {
132159
if err := conn.Exec(ctx, "DROP TABLE IF EXISTS test_lowcardinality"); assert.NoError(t, err) {
133160
if err := conn.Exec(ctx, ddl); assert.NoError(t, err) {
134161
if batch, err := conn.PrepareBatch(ctx, "INSERT INTO test_lowcardinality"); assert.NoError(t, err) {
162+
batch.Append("X")
163+
batch.Append(nil)
135164
batch.Append("X")
136165
t.Log(batch.Send())
137166
}
138167
}
139168
}
140169
}
141-
}*/
170+
}
142171
func TestColmnarLowCardinality(t *testing.T) {
143172
var (
144173
ctx = context.Background()

0 commit comments

Comments
 (0)