Skip to content

Commit e0bc273

Browse files
sfc-gh-asawickisfc-gh-asawicki
authored
chore: Use common generator setup in SDK generator (#4048)
Continuation for: #4043 Use the common generator set up for the SDK generator: - Introduce the new logic for "generation parts" (all previous generators output a single file; SDK outputs several) - Part of the SDK is generated through a secondary generator (dto builders) - it is addressed in the subsequent PR (#4049) - Add filtering by generation part using env (similar to the existing object type env filtering example) - extracting it as a command line usable filter is addressed in the subsequent PR (#4050) - Move both filters to the generator commons - Change the test package to `genhelpers_test` to avoid the circular dependency - Use common preamble - Introduce `preprocessDefinition`, replacing the logic contained previously in the template executors - Extract temporary new fields to move the old template executors' logic to templates: - `PathToDtoBuilderGen` - pointing to the DTO builder gen - `StructsToGenerate` - containing all structs to be generated (unique logic was handled differently before; it was the easiest way) - `DtosToGenerate` - as above - `ObjectIdMethod` is a model to generate the ID() method for an SDK object; replaces the old logic - `ObjectIdMethod` is a model to generate the ObjectType() method for an SDK object; replaces the old logic - Revert whitespace handling changed in the recent semantic views change - Describe the proposed rework in a dedicated README - `sdk_definitions` file is needed for the compilation of the newly generated files Next PRs: - Rework dto builders generator - Add common filters to the main generator logic - TODO left in code, already addressed in the subsequent PR (#4050) - Update generators documentation - Extract a vararg constructor and use it in all generators (if this one is approved) - TODO left in code - Fix generation logic for the implementation mapping (check TODO in `streamlits_impl_gen.go`) - Use the new generator on all SDK-generated files (after DTO builder generator rework and filtering) - Remove `PathToDtoBuilderGen` - TODO left in code - Remove SDK template executors (after full regeneration) - multiple TODOs left in code - Move the current rework to the `example` directory - Decide if the object type is needed in the object assert definition (it looks unused) - TODO left in code - Update schema parameters definition and use in tests (result of #4043 (comment)) - TODO left in code - The templates may not have been updated with `conversionErrorWrapped` during the `convert` method signature change - TODO left in code - In SDK generator logic, split the definition from model (all other generators have this distinction, but in the SDK, the same object is used) - TODO left in code
1 parent 8853d9a commit e0bc273

38 files changed

+2216
-80
lines changed

‎pkg/acceptance/bettertestspoc/assert/objectassert/gen/sdk_object_def.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ type SdkObjectDef struct {
1111
ObjectStruct any
1212
}
1313

14+
// TODO [SNOW-2324252]: remove object type?
1415
var allStructs = []SdkObjectDef{
1516
{
1617
IdType: "sdk.AccountObjectIdentifier",
@@ -155,7 +156,6 @@ func GetSdkObjectDetails() []genhelpers.SdkObjectDetails {
155156
structDetails := genhelpers.ExtractStructDetails(d.ObjectStruct)
156157
allSdkObjectsDetails[idx] = genhelpers.SdkObjectDetails{
157158
IdType: d.IdType,
158-
ObjectType: d.ObjectType,
159159
StructDetails: structDetails,
160160
}
161161
}

‎pkg/acceptance/bettertestspoc/assert/objectparametersassert/gen/object_parameters_def.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -346,6 +346,7 @@ var allObjectsParameters = []SnowflakeObjectParameters{
346346
{ParameterName: string(sdk.ProcedureParameterTraceLevel), ParameterType: "sdk.TraceLevel", DefaultValue: "sdk.TraceLevelOff", DefaultLevel: "sdk.ParameterTypeSnowflakeDefault"},
347347
},
348348
},
349+
// TODO [SNOW-1501905]: update this definition and use results in tests
349350
{
350351
Name: "Schema",
351352
IdType: "sdk.DatabaseObjectIdentifier",

‎pkg/internal/genhelpers/generation_part_filters.go‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
package genhelpers
2+
3+
import (
4+
"os"
5+
"slices"
6+
"strings"
7+
)
8+
9+
// TODO [SNOW-2324252]: Consider extracting this as a command line param
10+
func FilterGenerationPartByNameFromEnv[T ObjectNameProvider, M HasPreambleModel](part GenerationPart[T, M]) bool {
11+
allowedObjectNamesString := os.Getenv("SF_TF_GENERATOR_EXT_ALLOWED_GENERATION_PARTS_NAMES")
12+
if allowedObjectNamesString == "" {
13+
return true
14+
}
15+
allowedObjectNames := strings.Split(allowedObjectNamesString, ",")
16+
return slices.Contains(allowedObjectNames, part.GetName())
17+
}

‎pkg/internal/genhelpers/generator.go‎

Lines changed: 78 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -24,26 +24,61 @@ type Generator[T ObjectNameProvider, M HasPreambleModel] struct {
2424
filenameProvider func(T, M) string
2525
templates []*template.Template
2626

27+
generationParts []GenerationPart[T, M]
28+
2729
additionalObjectDebugLogProviders []func([]T)
2830
objectFilters []func(T) bool
31+
generationPartFilters []func(GenerationPart[T, M]) bool
2932

3033
preamble *PreambleModel
3134
}
3235

36+
type GenerationPart[T ObjectNameProvider, M HasPreambleModel] struct {
37+
name string
38+
filenameProvider func(T, M) string
39+
templates []*template.Template
40+
}
41+
42+
func (g *GenerationPart[_, _]) GetName() string {
43+
return g.name
44+
}
45+
46+
func NewGenerationPart[T ObjectNameProvider, M HasPreambleModel](name string, filenameProvider func(T, M) string, templates []*template.Template) GenerationPart[T, M] {
47+
return GenerationPart[T, M]{
48+
name: name,
49+
filenameProvider: filenameProvider,
50+
templates: templates,
51+
}
52+
}
53+
3354
func NewGenerator[T ObjectNameProvider, M HasPreambleModel](preamble *PreambleModel, objectsProvider func() []T, modelProvider func(T, *PreambleModel) M, filenameProvider func(T, M) string, templates []*template.Template) *Generator[T, M] {
55+
// TODO [SNOW-2324252]: handle vararg input
56+
parts := []GenerationPart[T, M]{
57+
// TODO [SNOW-2324252]: change default to name when changing to vararg
58+
NewGenerationPart("default", filenameProvider, templates),
59+
}
3460
return &Generator[T, M]{
3561
objectsProvider: objectsProvider,
3662
modelProvider: modelProvider,
3763
filenameProvider: filenameProvider,
3864
templates: templates,
3965

66+
generationParts: parts,
67+
4068
additionalObjectDebugLogProviders: make([]func([]T), 0),
4169
objectFilters: make([]func(T) bool, 0),
70+
generationPartFilters: make([]func(GenerationPart[T, M]) bool, 0),
4271

4372
preamble: preamble,
4473
}
4574
}
4675

76+
// TODO [SNOW-2324252]: Probably remove later when we have vararg support in the NewGenerator constructor
77+
func (g *Generator[T, M]) WithGenerationPart(partName string, filenameProvider func(T, M) string, templates []*template.Template) *Generator[T, M] {
78+
g.generationParts = append(g.generationParts, NewGenerationPart(partName, filenameProvider, templates))
79+
return g
80+
}
81+
4782
func (g *Generator[T, M]) WithAdditionalObjectsDebugLogs(objectLogsProvider func([]T)) *Generator[T, M] {
4883
g.additionalObjectDebugLogProviders = append(g.additionalObjectDebugLogProviders, objectLogsProvider)
4984
return g
@@ -54,7 +89,12 @@ func (g *Generator[T, M]) WithObjectFilter(objectFilter func(T) bool) *Generator
5489
return g
5590
}
5691

57-
func (g *Generator[T, _]) Run() error {
92+
func (g *Generator[T, M]) WithGenerationPartFilter(generationPartFilter func(GenerationPart[T, M]) bool) *Generator[T, M] {
93+
g.generationPartFilters = append(g.generationPartFilters, generationPartFilter)
94+
return g
95+
}
96+
97+
func (g *Generator[T, M]) Run() error {
5898
preprocessArgs()
5999

60100
file := os.Getenv("GOFILE")
@@ -65,7 +105,6 @@ func (g *Generator[T, _]) Run() error {
65105
flag.Parse()
66106

67107
objects := g.objectsProvider()
68-
69108
if len(g.objectFilters) > 0 {
70109
filteredObjects := make([]T, 0)
71110
for _, o := range objects {
@@ -80,18 +119,33 @@ func (g *Generator[T, _]) Run() error {
80119
objects = filteredObjects
81120
}
82121

122+
parts := g.generationParts
123+
if len(g.generationPartFilters) > 0 {
124+
filteredGenerationParts := make([]GenerationPart[T, M], 0)
125+
for _, p := range g.generationParts {
126+
matches := true
127+
for _, f := range g.generationPartFilters {
128+
matches = matches && f(p)
129+
}
130+
if matches {
131+
filteredGenerationParts = append(filteredGenerationParts, p)
132+
}
133+
}
134+
parts = filteredGenerationParts
135+
}
136+
83137
if *additionalLogs {
84138
for _, p := range g.additionalObjectDebugLogProviders {
85139
p(objects)
86140
}
87141
}
88142

89143
if *dryRun {
90-
if err := g.generateAndPrintForAllObjects(objects); err != nil {
144+
if err := g.generateAndPrint(objects, parts); err != nil {
91145
return err
92146
}
93147
} else {
94-
if err := g.generateAndSaveForAllObjects(objects); err != nil {
148+
if err := g.generateAndSave(objects, parts); err != nil {
95149
return err
96150
}
97151
}
@@ -116,33 +170,39 @@ func (g *Generator[_, _]) RunAndHandleOsReturn() {
116170
}
117171
}
118172

119-
func (g *Generator[T, _]) generateAndSaveForAllObjects(objects []T) error {
173+
func (g *Generator[T, M]) generateAndSave(objects []T, parts []GenerationPart[T, M]) error {
120174
var errs []error
121175
for _, s := range objects {
122-
buffer := bytes.Buffer{}
123176
model := g.modelProvider(s, g.preamble)
124-
if err := executeAllTemplates(model, &buffer, g.templates...); err != nil {
125-
errs = append(errs, fmt.Errorf("generating output for object %s failed with err: %w", s.ObjectName(), err))
126-
continue
127-
}
128-
filename := g.filenameProvider(s, model)
129-
if err := WriteCodeToFile(&buffer, filename); err != nil {
130-
errs = append(errs, fmt.Errorf("saving output for object %s to file %s failed with err: %w", s.ObjectName(), filename, err))
131-
continue
177+
178+
for _, p := range parts {
179+
buffer := bytes.Buffer{}
180+
181+
if err := executeAllTemplates(model, &buffer, p.templates...); err != nil {
182+
errs = append(errs, fmt.Errorf("generating output for object %s failed with err: %w", s.ObjectName(), err))
183+
continue
184+
}
185+
filename := p.filenameProvider(s, model)
186+
if err := WriteCodeToFile(&buffer, filename); err != nil {
187+
errs = append(errs, fmt.Errorf("saving output for object %s to file %s failed with err: %w", s.ObjectName(), filename, err))
188+
continue
189+
}
132190
}
133191
}
134192
return errors.Join(errs...)
135193
}
136194

137-
func (g *Generator[T, _]) generateAndPrintForAllObjects(objects []T) error {
195+
func (g *Generator[T, M]) generateAndPrint(objects []T, parts []GenerationPart[T, M]) error {
138196
var errs []error
139197
for _, s := range objects {
140198
fmt.Println("===========================")
141199
fmt.Printf("Generating for object %s\n", s.ObjectName())
142200
fmt.Println("===========================")
143-
if err := executeAllTemplates(g.modelProvider(s, g.preamble), os.Stdout, g.templates...); err != nil {
144-
errs = append(errs, fmt.Errorf("generating output for object %s failed with err: %w", s.ObjectName(), err))
145-
continue
201+
for _, p := range parts {
202+
if err := executeAllTemplates(g.modelProvider(s, g.preamble), os.Stdout, p.templates...); err != nil {
203+
errs = append(errs, fmt.Errorf("generating output for object %s failed with err: %w", s.ObjectName(), err))
204+
continue
205+
}
146206
}
147207
}
148208
return errors.Join(errs...)

‎pkg/internal/genhelpers/imports_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ import (
88
"golang.org/x/tools/imports"
99
)
1010

11-
// TODO [next PR - SNOW-2324252]: extract common setup, template for easier test writing
11+
// TODO [SNOW-2324252]: extract common setup, template for easier test writing
1212
func Test_ExperimentWithImportsProcess(t *testing.T) {
1313
t.Run("add standard imports", func(t *testing.T) {
1414
src := []byte(`package somepackagename

‎pkg/internal/genhelpers/object_filters.go‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
package genhelpers
2+
3+
import (
4+
"os"
5+
"slices"
6+
"strings"
7+
)
8+
9+
// TODO [SNOW-2324252]: Consider extracting this as a command line param
10+
func FilterObjectByNameFromEnv[T ObjectNameProvider](object T) bool {
11+
allowedObjectNamesString := os.Getenv("SF_TF_GENERATOR_EXT_ALLOWED_OBJECT_NAMES")
12+
if allowedObjectNamesString == "" {
13+
return true
14+
}
15+
allowedObjectNames := strings.Split(allowedObjectNamesString, ",")
16+
return slices.Contains(allowedObjectNames, object.ObjectName())
17+
}

‎pkg/internal/genhelpers/sdk_object_details.go‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,6 @@
11
package genhelpers
22

3-
import "github.com/Snowflake-Labs/terraform-provider-snowflake/pkg/sdk"
4-
53
type SdkObjectDetails struct {
6-
IdType string
7-
ObjectType sdk.ObjectType
4+
IdType string
85
StructDetails
96
}

‎pkg/internal/genhelpers/struct_details_extractor_test.go‎

Lines changed: 15 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,10 @@
1-
package genhelpers
1+
package genhelpers_test
22

33
import (
44
"testing"
55
"time"
66

7+
"github.com/Snowflake-Labs/terraform-provider-snowflake/pkg/internal/genhelpers"
78
"github.com/Snowflake-Labs/terraform-provider-snowflake/pkg/sdk"
89
"github.com/stretchr/testify/assert"
910
)
@@ -66,16 +67,16 @@ func Test_ExtractStructDetails(t *testing.T) {
6667
ExportedFloat64Ptr *float64
6768
}
6869

69-
assertFieldExtracted := func(field Field, expectedName string, expectedConcreteType string, expectedUnderlyingType string) {
70+
assertFieldExtracted := func(field genhelpers.Field, expectedName string, expectedConcreteType string, expectedUnderlyingType string) {
7071
assert.Equal(t, expectedName, field.Name)
7172
assert.Equal(t, expectedConcreteType, field.ConcreteType)
7273
assert.Equal(t, expectedUnderlyingType, field.UnderlyingType)
7374
}
7475

7576
t.Run("test struct details extraction", func(t *testing.T) {
76-
structDetails := ExtractStructDetails(testStruct{})
77+
structDetails := genhelpers.ExtractStructDetails(testStruct{})
7778

78-
assert.Equal(t, "genhelpers.testStruct", structDetails.Name)
79+
assert.Equal(t, "genhelpers_test.testStruct", structDetails.Name)
7980

8081
assertFieldExtracted(structDetails.Fields[0], "unexportedString", "string", "string")
8182
assertFieldExtracted(structDetails.Fields[1], "unexportedInt", "int", "int")
@@ -91,8 +92,8 @@ func Test_ExtractStructDetails(t *testing.T) {
9192

9293
assertFieldExtracted(structDetails.Fields[10], "unexportedStringEnum", "sdk.WarehouseType", "string")
9394
assertFieldExtracted(structDetails.Fields[11], "unexportedStringEnumPtr", "*sdk.WarehouseType", "*string")
94-
assertFieldExtracted(structDetails.Fields[12], "unexportedIntEnum", "genhelpers.testIntEnum", "int")
95-
assertFieldExtracted(structDetails.Fields[13], "unexportedIntEnumPtr", "*genhelpers.testIntEnum", "*int")
95+
assertFieldExtracted(structDetails.Fields[12], "unexportedIntEnum", "genhelpers_test.testIntEnum", "int")
96+
assertFieldExtracted(structDetails.Fields[13], "unexportedIntEnumPtr", "*genhelpers_test.testIntEnum", "*int")
9697

9798
assertFieldExtracted(structDetails.Fields[14], "unexportedAccountIdentifier", "sdk.AccountIdentifier", "struct")
9899
assertFieldExtracted(structDetails.Fields[15], "unexportedExternalObjectIdentifier", "sdk.ExternalObjectIdentifier", "struct")
@@ -119,43 +120,43 @@ func Test_ExtractStructDetails(t *testing.T) {
119120

120121
func Test_Field_IsSlice(t *testing.T) {
121122
t.Run("is a slice", func(t *testing.T) {
122-
field := Field{ConcreteType: "[]any_type"}
123+
field := genhelpers.Field{ConcreteType: "[]any_type"}
123124

124125
assert.True(t, field.IsSlice())
125126
})
126127

127128
t.Run("is not a slice", func(t *testing.T) {
128-
field := Field{ConcreteType: "*any_type"}
129+
field := genhelpers.Field{ConcreteType: "*any_type"}
129130

130131
assert.False(t, field.IsSlice())
131132

132-
field = Field{ConcreteType: "*[]any_type"}
133+
field = genhelpers.Field{ConcreteType: "*[]any_type"}
133134

134135
assert.False(t, field.IsSlice())
135136

136-
field = Field{ConcreteType: "any_type"}
137+
field = genhelpers.Field{ConcreteType: "any_type"}
137138

138139
assert.False(t, field.IsSlice())
139140
})
140141
}
141142

142143
func Test_Field_IsPointer(t *testing.T) {
143144
t.Run("is a pointer", func(t *testing.T) {
144-
field := Field{ConcreteType: "*any_type"}
145+
field := genhelpers.Field{ConcreteType: "*any_type"}
145146

146147
assert.True(t, field.IsPointer())
147148
})
148149

149150
t.Run("is not a pointer", func(t *testing.T) {
150-
field := Field{ConcreteType: "[]any_type"}
151+
field := genhelpers.Field{ConcreteType: "[]any_type"}
151152

152153
assert.False(t, field.IsPointer())
153154

154-
field = Field{ConcreteType: "[]*any_type"}
155+
field = genhelpers.Field{ConcreteType: "[]*any_type"}
155156

156157
assert.False(t, field.IsPointer())
157158

158-
field = Field{ConcreteType: "any_type"}
159+
field = genhelpers.Field{ConcreteType: "any_type"}
159160

160161
assert.False(t, field.IsPointer())
161162
})

‎pkg/schemas/gen/main/main.go‎

Lines changed: 1 addition & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,7 @@ package main
44

55
import (
66
"fmt"
7-
"os"
87
"slices"
9-
"strings"
108

119
"github.com/Snowflake-Labs/terraform-provider-snowflake/pkg/internal/genhelpers"
1210
"github.com/Snowflake-Labs/terraform-provider-snowflake/pkg/schemas/gen"
@@ -30,7 +28,7 @@ func main() {
3028
).
3129
WithAdditionalObjectsDebugLogs(printAllStructsFields).
3230
WithAdditionalObjectsDebugLogs(printUniqueTypes).
33-
WithObjectFilter(filterByNameFromEnv).
31+
WithObjectFilter(genhelpers.FilterObjectByNameFromEnv[genhelpers.StructDetails]).
3432
RunAndHandleOsReturn()
3533
}
3634

@@ -75,13 +73,3 @@ func printUniqueTypes(allStructs []genhelpers.StructDetails) {
7573
fmt.Println(k)
7674
}
7775
}
78-
79-
// TODO: move this filter to commons and consider extracting this as a command line param
80-
func filterByNameFromEnv(o genhelpers.StructDetails) bool {
81-
allowedObjectNamesString := os.Getenv("SF_TF_GENERATOR_EXT_ALLOWED_OBJECT_NAMES")
82-
if allowedObjectNamesString == "" {
83-
return true
84-
}
85-
allowedObjectNames := strings.Split(allowedObjectNamesString, ",")
86-
return slices.Contains(allowedObjectNames, o.ObjectName())
87-
}

0 commit comments

Comments
 (0)