diff --git a/go.mod b/go.mod index 375d9bb21..18e5bc7bf 100644 --- a/go.mod +++ b/go.mod @@ -90,7 +90,7 @@ require ( github.com/spf13/pflag v1.0.5 // indirect github.com/xlab/treeprint v1.2.0 // indirect go.opencensus.io v0.24.0 // indirect - go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect + go.starlark.net v0.0.0-20250804182900-3c9dc17c5f2e // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.26.0 // indirect golang.org/x/crypto v0.21.0 // indirect diff --git a/go.sum b/go.sum index dcbe326b0..2f348f093 100644 --- a/go.sum +++ b/go.sum @@ -673,8 +673,8 @@ go.opentelemetry.io/otel/sdk/export/metric v0.20.0/go.mod h1:h7RBNMsDJ5pmI1zExLi go.opentelemetry.io/otel/sdk/metric v0.20.0/go.mod h1:knxiS8Xd4E/N+ZqKmUPf3gTTZ4/0TjTXukfxjzSTpHE= go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= -go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 h1:+FNtrFTmVw0YZGpBGX56XDee331t6JAXeK2bcyhLOOc= -go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5/go.mod h1:nmDLcffg48OtT/PSW0Hg7FvpRQsQh5OSqIylirxKC7o= +go.starlark.net v0.0.0-20250804182900-3c9dc17c5f2e h1:0DI8mzcQzo8pkhUagHLRu9GmdXdjm0xRDzubkwIz36w= +go.starlark.net v0.0.0-20250804182900-3c9dc17c5f2e/go.mod h1:YKMCv9b1WrfWmeqdV5MAuEHWsu5iC+fe6kYl2sQjdI8= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= @@ -850,7 +850,6 @@ golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191002063906-3421d5a6bb1c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/vendor/go.starlark.net/internal/compile/compile.go b/vendor/go.starlark.net/internal/compile/compile.go index eb8e16279..b257d70d3 100644 --- a/vendor/go.starlark.net/internal/compile/compile.go +++ b/vendor/go.starlark.net/internal/compile/compile.go @@ -23,7 +23,6 @@ // // Operands, logically uint32s, are encoded using little-endian 7-bit // varints, the top bit indicating that more bytes follow. -// package compile // import "go.starlark.net/internal/compile" import ( @@ -33,6 +32,7 @@ import ( "os" "path/filepath" "strconv" + "strings" "sync" "go.starlark.net/resolve" @@ -46,7 +46,7 @@ var Disassemble = false const debug = false // make code generation verbose, for debugging the compiler // Increment this to force recompilation of saved bytecode files. -const Version = 10 +const Version = 14 type Opcode uint8 @@ -99,20 +99,19 @@ const ( FALSE // - FALSE False MANDATORY // - MANDATORY Mandatory [sentinel value for required kwonly args] - ITERPUSH // iterable ITERPUSH - [pushes the iterator stack] - ITERPOP // - ITERPOP - [pops the iterator stack] - NOT // value NOT bool - RETURN // value RETURN - - SETINDEX // a i new SETINDEX - - INDEX // a i INDEX elem - SETDICT // dict key value SETDICT - - SETDICTUNIQ // dict key value SETDICTUNIQ - - APPEND // list elem APPEND - - SLICE // x lo hi step SLICE slice - INPLACE_ADD // x y INPLACE_ADD z where z is x+y or x.extend(y) - MAKEDICT // - MAKEDICT dict - SETCELL // value cell SETCELL - - CELL // cell CELL value + ITERPUSH // iterable ITERPUSH - [pushes the iterator stack] + ITERPOP // - ITERPOP - [pops the iterator stack] + NOT // value NOT bool + RETURN // value RETURN - + SETINDEX // a i new SETINDEX - + INDEX // a i INDEX elem + SETDICT // dict key value SETDICT - + SETDICTUNIQ // dict key value SETDICTUNIQ - + APPEND // list elem APPEND - + SLICE // x lo hi step SLICE slice + INPLACE_ADD // x y INPLACE_ADD z where z is x+y or x.extend(y) + INPLACE_PIPE // x y INPLACE_PIPE z where z is x|y + MAKEDICT // - MAKEDICT dict // --- opcodes with an argument must go below this line --- @@ -122,21 +121,24 @@ const ( ITERJMP // - ITERJMP elem (and fall through) [acts on topmost iterator] // or: - ITERJMP - (and jump) - CONSTANT // - CONSTANT value - MAKETUPLE // x1 ... xn MAKETUPLE tuple - MAKELIST // x1 ... xn MAKELIST list - MAKEFUNC // defaults+freevars MAKEFUNC fn - LOAD // from1 ... fromN module LOAD v1 ... vN - SETLOCAL // value SETLOCAL - - SETGLOBAL // value SETGLOBAL - - LOCAL // - LOCAL value - FREE // - FREE cell - GLOBAL // - GLOBAL value - PREDECLARED // - PREDECLARED value - UNIVERSAL // - UNIVERSAL value - ATTR // x ATTR y y = x.name - SETFIELD // x y SETFIELD - x.name = y - UNPACK // iterable UNPACK vn ... v1 + CONSTANT // - CONSTANT value + MAKETUPLE // x1 ... xn MAKETUPLE tuple + MAKELIST // x1 ... xn MAKELIST list + MAKEFUNC // defaults+freevars MAKEFUNC fn + LOAD // from1 ... fromN module LOAD v1 ... vN + SETLOCAL // value SETLOCAL - + SETGLOBAL // value SETGLOBAL - + LOCAL // - LOCAL value + FREE // - FREE cell + FREECELL // - FREECELL value (content of FREE cell) + LOCALCELL // - LOCALCELL value (content of LOCAL cell) + SETLOCALCELL // value SETLOCALCELL - (set content of LOCAL cell) + GLOBAL // - GLOBAL value + PREDECLARED // - PREDECLARED value + UNIVERSAL // - UNIVERSAL value + ATTR // x ATTR y y = x.name + SETFIELD // x y SETFIELD - x.name = y + UNPACK // iterable UNPACK vn ... v1 // n>>8 is #positional args and n&0xff is #named args (pairs). CALL // fn positional named CALL result @@ -151,72 +153,74 @@ const ( // TODO(adonovan): add dynamic checks for missing opcodes in the tables below. var opcodeNames = [...]string{ - AMP: "amp", - APPEND: "append", - ATTR: "attr", - CALL: "call", - CALL_KW: "call_kw ", - CALL_VAR: "call_var", - CALL_VAR_KW: "call_var_kw", - CELL: "cell", - CIRCUMFLEX: "circumflex", - CJMP: "cjmp", - CONSTANT: "constant", - DUP2: "dup2", - DUP: "dup", - EQL: "eql", - EXCH: "exch", - FALSE: "false", - FREE: "free", - GE: "ge", - GLOBAL: "global", - GT: "gt", - GTGT: "gtgt", - IN: "in", - INDEX: "index", - INPLACE_ADD: "inplace_add", - ITERJMP: "iterjmp", - ITERPOP: "iterpop", - ITERPUSH: "iterpush", - JMP: "jmp", - LE: "le", - LOAD: "load", - LOCAL: "local", - LT: "lt", - LTLT: "ltlt", - MAKEDICT: "makedict", - MAKEFUNC: "makefunc", - MAKELIST: "makelist", - MAKETUPLE: "maketuple", - MANDATORY: "mandatory", - MINUS: "minus", - NEQ: "neq", - NONE: "none", - NOP: "nop", - NOT: "not", - PERCENT: "percent", - PIPE: "pipe", - PLUS: "plus", - POP: "pop", - PREDECLARED: "predeclared", - RETURN: "return", - SETCELL: "setcell", - SETDICT: "setdict", - SETDICTUNIQ: "setdictuniq", - SETFIELD: "setfield", - SETGLOBAL: "setglobal", - SETINDEX: "setindex", - SETLOCAL: "setlocal", - SLASH: "slash", - SLASHSLASH: "slashslash", - SLICE: "slice", - STAR: "star", - TILDE: "tilde", - TRUE: "true", - UMINUS: "uminus", - UNIVERSAL: "universal", - UNPACK: "unpack", - UPLUS: "uplus", + AMP: "amp", + APPEND: "append", + ATTR: "attr", + CALL: "call", + CALL_KW: "call_kw ", + CALL_VAR: "call_var", + CALL_VAR_KW: "call_var_kw", + CIRCUMFLEX: "circumflex", + CJMP: "cjmp", + CONSTANT: "constant", + DUP2: "dup2", + DUP: "dup", + EQL: "eql", + EXCH: "exch", + FALSE: "false", + FREE: "free", + FREECELL: "freecell", + GE: "ge", + GLOBAL: "global", + GT: "gt", + GTGT: "gtgt", + IN: "in", + INDEX: "index", + INPLACE_ADD: "inplace_add", + INPLACE_PIPE: "inplace_pipe", + ITERJMP: "iterjmp", + ITERPOP: "iterpop", + ITERPUSH: "iterpush", + JMP: "jmp", + LE: "le", + LOAD: "load", + LOCAL: "local", + LOCALCELL: "localcell", + LT: "lt", + LTLT: "ltlt", + MAKEDICT: "makedict", + MAKEFUNC: "makefunc", + MAKELIST: "makelist", + MAKETUPLE: "maketuple", + MANDATORY: "mandatory", + MINUS: "minus", + NEQ: "neq", + NONE: "none", + NOP: "nop", + NOT: "not", + PERCENT: "percent", + PIPE: "pipe", + PLUS: "plus", + POP: "pop", + PREDECLARED: "predeclared", + RETURN: "return", + SETDICT: "setdict", + SETDICTUNIQ: "setdictuniq", + SETFIELD: "setfield", + SETGLOBAL: "setglobal", + SETINDEX: "setindex", + SETLOCAL: "setlocal", + SETLOCALCELL: "setlocalcell", + SLASH: "slash", + SLASHSLASH: "slashslash", + SLICE: "slice", + STAR: "star", + TILDE: "tilde", + TRUE: "true", + UMINUS: "uminus", + UNIVERSAL: "universal", + UNPACK: "unpack", + UPLUS: "uplus", } const variableStackEffect = 0x7f @@ -224,70 +228,72 @@ const variableStackEffect = 0x7f // stackEffect records the effect on the size of the operand stack of // each kind of instruction. For some instructions this requires computation. var stackEffect = [...]int8{ - AMP: -1, - APPEND: -2, - ATTR: 0, - CALL: variableStackEffect, - CALL_KW: variableStackEffect, - CALL_VAR: variableStackEffect, - CALL_VAR_KW: variableStackEffect, - CELL: 0, - CIRCUMFLEX: -1, - CJMP: -1, - CONSTANT: +1, - DUP2: +2, - DUP: +1, - EQL: -1, - FALSE: +1, - FREE: +1, - GE: -1, - GLOBAL: +1, - GT: -1, - GTGT: -1, - IN: -1, - INDEX: -1, - INPLACE_ADD: -1, - ITERJMP: variableStackEffect, - ITERPOP: 0, - ITERPUSH: -1, - JMP: 0, - LE: -1, - LOAD: -1, - LOCAL: +1, - LT: -1, - LTLT: -1, - MAKEDICT: +1, - MAKEFUNC: 0, - MAKELIST: variableStackEffect, - MAKETUPLE: variableStackEffect, - MANDATORY: +1, - MINUS: -1, - NEQ: -1, - NONE: +1, - NOP: 0, - NOT: 0, - PERCENT: -1, - PIPE: -1, - PLUS: -1, - POP: -1, - PREDECLARED: +1, - RETURN: -1, - SETCELL: -2, - SETDICT: -3, - SETDICTUNIQ: -3, - SETFIELD: -2, - SETGLOBAL: -1, - SETINDEX: -3, - SETLOCAL: -1, - SLASH: -1, - SLASHSLASH: -1, - SLICE: -3, - STAR: -1, - TRUE: +1, - UMINUS: 0, - UNIVERSAL: +1, - UNPACK: variableStackEffect, - UPLUS: 0, + AMP: -1, + APPEND: -2, + ATTR: 0, + CALL: variableStackEffect, + CALL_KW: variableStackEffect, + CALL_VAR: variableStackEffect, + CALL_VAR_KW: variableStackEffect, + CIRCUMFLEX: -1, + CJMP: -1, + CONSTANT: +1, + DUP2: +2, + DUP: +1, + EQL: -1, + FALSE: +1, + FREE: +1, + FREECELL: +1, + GE: -1, + GLOBAL: +1, + GT: -1, + GTGT: -1, + IN: -1, + INDEX: -1, + INPLACE_ADD: -1, + INPLACE_PIPE: -1, + ITERJMP: variableStackEffect, + ITERPOP: 0, + ITERPUSH: -1, + JMP: 0, + LE: -1, + LOAD: -1, + LOCAL: +1, + LOCALCELL: +1, + LT: -1, + LTLT: -1, + MAKEDICT: +1, + MAKEFUNC: 0, + MAKELIST: variableStackEffect, + MAKETUPLE: variableStackEffect, + MANDATORY: +1, + MINUS: -1, + NEQ: -1, + NONE: +1, + NOP: 0, + NOT: 0, + PERCENT: -1, + PIPE: -1, + PLUS: -1, + POP: -1, + PREDECLARED: +1, + RETURN: -1, + SETLOCALCELL: -1, + SETDICT: -3, + SETDICTUNIQ: -3, + SETFIELD: -2, + SETGLOBAL: -1, + SETINDEX: -3, + SETLOCAL: -1, + SLASH: -1, + SLASHSLASH: -1, + SLICE: -3, + STAR: -1, + TRUE: +1, + UMINUS: 0, + UNIVERSAL: +1, + UNPACK: variableStackEffect, + UPLUS: 0, } func (op Opcode) String() string { @@ -306,12 +312,16 @@ func (op Opcode) String() string { type Program struct { Loads []Binding // name (really, string) and position of each load stmt Names []string // names of attributes and predeclared variables - Constants []interface{} // = string | int64 | float64 | *big.Int + Constants []interface{} // = string | int64 | float64 | *big.Int | Bytes Functions []*Funcode Globals []Binding // for error messages and tracing Toplevel *Funcode // module initialization function + Recursion bool // disable recursion check for functions in this file } +// The type of a bytes literal value, to distinguish from text string. +type Bytes string + // A Funcode is the code of a compiled Starlark function. // // Funcodes are serialized by the encoder.function method, @@ -325,7 +335,7 @@ type Funcode struct { pclinetab []uint16 // mapping from pc to linenum Locals []Binding // locals, parameters first Cells []int // indices of Locals that require cells - Freevars []Binding // for tracing + FreeVars []Binding // for tracing MaxStack int NumParams int NumKwonlyParams int @@ -476,17 +486,20 @@ func bindings(bindings []*resolve.Binding) []Binding { } // Expr compiles an expression to a program whose toplevel function evaluates it. -func Expr(expr syntax.Expr, name string, locals []*resolve.Binding) *Program { +// The options must be consistent with those used when parsing expr. +func Expr(opts *syntax.FileOptions, expr syntax.Expr, name string, locals []*resolve.Binding) *Program { pos := syntax.Start(expr) stmts := []syntax.Stmt{&syntax.ReturnStmt{Result: expr}} - return File(stmts, pos, name, locals, nil) + return File(opts, stmts, pos, name, locals, nil) } // File compiles the statements of a file into a program. -func File(stmts []syntax.Stmt, pos syntax.Position, name string, locals, globals []*resolve.Binding) *Program { +// The options must be consistent with those used when parsing stmts. +func File(opts *syntax.FileOptions, stmts []syntax.Stmt, pos syntax.Position, name string, locals, globals []*resolve.Binding) *Program { pcomp := &pcomp{ prog: &Program{ - Globals: bindings(globals), + Globals: bindings(globals), + Recursion: opts.Recursion, }, names: make(map[string]uint32), constants: make(map[interface{}]uint32), @@ -507,7 +520,7 @@ func (pcomp *pcomp) function(name string, pos syntax.Position, stmts []syntax.St Name: name, Doc: docStringFromBody(stmts), Locals: bindings(locals), - Freevars: bindings(freevars), + FreeVars: bindings(freevars), }, } @@ -860,6 +873,8 @@ func PrintOp(fn *Funcode, pc uint32, op Opcode, arg uint32) { switch x := fn.Prog.Constants[arg].(type) { case string: comment = strconv.Quote(x) + case Bytes: + comment = "b" + strconv.Quote(string(x)) default: comment = fmt.Sprint(x) } @@ -872,7 +887,7 @@ func PrintOp(fn *Funcode, pc uint32, op Opcode, arg uint32) { case ATTR, SETFIELD, PREDECLARED, UNIVERSAL: comment = fn.Prog.Names[arg] case FREE: - comment = fn.Freevars[arg].Name + comment = fn.FreeVars[arg].Name case CALL, CALL_VAR, CALL_KW, CALL_VAR_KW: comment = fmt.Sprintf("%d pos, %d named", arg>>8, arg&0xff) default: @@ -994,9 +1009,7 @@ func (fcomp *fcomp) set(id *syntax.Ident) { case resolve.Local: fcomp.emit1(SETLOCAL, uint32(bind.Index)) case resolve.Cell: - // TODO(adonovan): opt: make a single op for LOCAL, SETCELL. - fcomp.emit1(LOCAL, uint32(bind.Index)) - fcomp.emit(SETCELL) + fcomp.emit1(SETLOCALCELL, uint32(bind.Index)) case resolve.Global: fcomp.emit1(SETGLOBAL, uint32(bind.Index)) default: @@ -1014,13 +1027,9 @@ func (fcomp *fcomp) lookup(id *syntax.Ident) { case resolve.Local: fcomp.emit1(LOCAL, uint32(bind.Index)) case resolve.Free: - // TODO(adonovan): opt: make a single op for FREE, CELL. - fcomp.emit1(FREE, uint32(bind.Index)) - fcomp.emit(CELL) + fcomp.emit1(FREECELL, uint32(bind.Index)) case resolve.Cell: - // TODO(adonovan): opt: make a single op for LOCAL, CELL. - fcomp.emit1(LOCAL, uint32(bind.Index)) - fcomp.emit(CELL) + fcomp.emit1(LOCALCELL, uint32(bind.Index)) case resolve.Global: fcomp.emit1(GLOBAL, uint32(bind.Index)) case resolve.Predeclared: @@ -1142,11 +1151,16 @@ func (fcomp *fcomp) stmt(stmt syntax.Stmt) { fcomp.expr(stmt.RHS) - if stmt.Op == syntax.PLUS_EQ { - // Allow the runtime to optimize list += iterable. + // In-place x+=y and x|=y have special semantics: + // the resulting x aliases the original x. + switch stmt.Op { + case syntax.PLUS_EQ: fcomp.setPos(stmt.OpPos) fcomp.emit(INPLACE_ADD) - } else { + case syntax.PIPE_EQ: + fcomp.setPos(stmt.OpPos) + fcomp.emit(INPLACE_PIPE) + default: fcomp.binop(stmt.OpPos, stmt.Op-syntax.PLUS_EQ+syntax.PLUS) } set() @@ -1286,8 +1300,12 @@ func (fcomp *fcomp) expr(e syntax.Expr) { fcomp.lookup(e) case *syntax.Literal: - // e.Value is int64, float64, *bigInt, or string. - fcomp.emit1(CONSTANT, fcomp.pcomp.constantIndex(e.Value)) + // e.Value is int64, float64, *bigInt, string + v := e.Value + if e.Token == syntax.BYTES { + v = Bytes(v.(string)) + } + fcomp.emit1(CONSTANT, fcomp.pcomp.constantIndex(v)) case *syntax.ListExpr: for _, x := range e.List { @@ -1525,7 +1543,7 @@ func (fcomp *fcomp) plus(e *syntax.BinaryExpr) { } // addable reports whether e is a statically addable -// expression: a [s]tring, [l]ist, or [t]uple. +// expression: a [s]tring, [b]ytes, [l]ist, or [t]uple. func addable(e syntax.Expr) rune { switch e := e.(type) { case *syntax.Literal: @@ -1533,6 +1551,8 @@ func addable(e syntax.Expr) rune { switch e.Token { case syntax.STRING: return 's' + case syntax.BYTES: + return 'b' } case *syntax.ListExpr: return 'l' @@ -1547,12 +1567,16 @@ func addable(e syntax.Expr) rune { // The resulting syntax is degenerate, lacking position, etc. func add(code rune, args []summand) syntax.Expr { switch code { - case 's': - var buf bytes.Buffer + case 's', 'b': + var buf strings.Builder for _, arg := range args { buf.WriteString(arg.x.(*syntax.Literal).Value.(string)) } - return &syntax.Literal{Token: syntax.STRING, Value: buf.String()} + tok := syntax.STRING + if code == 'b' { + tok = syntax.BYTES + } + return &syntax.Literal{Token: tok, Value: buf.String()} case 'l': var elems []syntax.Expr for _, arg := range args { diff --git a/vendor/go.starlark.net/internal/compile/serial.go b/vendor/go.starlark.net/internal/compile/serial.go index 0107ef9cd..0dbae47c2 100644 --- a/vendor/go.starlark.net/internal/compile/serial.go +++ b/vendor/go.starlark.net/internal/compile/serial.go @@ -25,6 +25,7 @@ package compile // toplevel Funcode // numfuncs varint // funcs []Funcode +// recursion varint (0 or 1) // []byte # concatenation of all referenced strings // EOF // @@ -51,9 +52,10 @@ package compile // // Constant: # type data // type varint # 0=string string -// data ... # 1=int varint -// # 2=float varint (bits as uint64) -// # 3=bigint string (decimal ASCII text) +// data ... # 1=bytes string +// # 2=int varint +// # 3=float varint (bits as uint64) +// # 4=bigint string (decimal ASCII text) // // The encoding starts with a four-byte magic number. // The next four bytes are a little-endian uint32 @@ -109,14 +111,17 @@ func (prog *Program) Encode() []byte { case string: e.int(0) e.string(c) - case int64: + case Bytes: e.int(1) + e.string(string(c)) + case int64: + e.int(2) e.int64(c) case float64: - e.int(2) + e.int(3) e.uint64(math.Float64bits(c)) case *big.Int: - e.int(3) + e.int(4) e.string(c.Text(10)) } } @@ -126,6 +131,7 @@ func (prog *Program) Encode() []byte { for _, fn := range prog.Functions { e.function(fn) } + e.int(b2i(prog.Recursion)) // Patch in the offset of the string data section. binary.LittleEndian.PutUint32(e.p[4:8], uint32(len(e.p))) @@ -189,7 +195,7 @@ func (e *encoder) function(fn *Funcode) { for _, index := range fn.Cells { e.int(index) } - e.bindings(fn.Freevars) + e.bindings(fn.FreeVars) e.int(fn.MaxStack) e.int(fn.NumParams) e.int(fn.NumKwonlyParams) @@ -249,10 +255,12 @@ func DecodeProgram(data []byte) (_ *Program, err error) { case 0: c = d.string() case 1: - c = d.int64() + c = Bytes(d.string()) case 2: - c = math.Float64frombits(d.uint64()) + c = d.int64() case 3: + c = math.Float64frombits(d.uint64()) + case 4: c, _ = new(big.Int).SetString(d.string(), 10) } constants[i] = c @@ -264,6 +272,7 @@ func DecodeProgram(data []byte) (_ *Program, err error) { for i := range funcs { funcs[i] = d.function() } + recursion := d.int() != 0 prog := &Program{ Loads: loads, @@ -272,6 +281,7 @@ func DecodeProgram(data []byte) (_ *Program, err error) { Globals: globals, Functions: funcs, Toplevel: toplevel, + Recursion: recursion, } toplevel.Prog = prog for _, f := range funcs { @@ -379,7 +389,7 @@ func (d *decoder) function() *Funcode { pclinetab: pclinetab, Locals: locals, Cells: cells, - Freevars: freevars, + FreeVars: freevars, MaxStack: maxStack, NumParams: numParams, NumKwonlyParams: numKwonlyParams, diff --git a/vendor/go.starlark.net/resolve/binding.go b/vendor/go.starlark.net/resolve/binding.go index 6b99f4b97..8507e64c7 100644 --- a/vendor/go.starlark.net/resolve/binding.go +++ b/vendor/go.starlark.net/resolve/binding.go @@ -10,7 +10,7 @@ import "go.starlark.net/syntax" // We cannot guarantee API stability for these types // as they are closely tied to the implementation. -// A Binding contains resolver information about an identifer. +// A Binding contains resolver information about an identifier. // The resolver populates the Binding field of each syntax.Identifier. // The Binding ties together all identifiers that denote the same variable. type Binding struct { diff --git a/vendor/go.starlark.net/resolve/resolve.go b/vendor/go.starlark.net/resolve/resolve.go index 440bcf081..e00d2b36d 100644 --- a/vendor/go.starlark.net/resolve/resolve.go +++ b/vendor/go.starlark.net/resolve/resolve.go @@ -97,15 +97,20 @@ const doesnt = "this Starlark dialect does not " // global options // These features are either not standard Starlark (yet), or deprecated // features of the BUILD language, so we put them behind flags. +// +// Deprecated: use an explicit [syntax.FileOptions] argument instead, +// as it avoids all the usual problems of global variables. var ( - AllowNestedDef = false // allow def statements within function bodies - AllowLambda = false // allow lambda expressions - AllowFloat = false // allow floating point literals, the 'float' built-in, and x / y AllowSet = false // allow the 'set' built-in AllowGlobalReassign = false // allow reassignment to top-level names; also, allow if/for/while at top-level AllowRecursion = false // allow while statements and recursive functions - AllowBitwise = true // obsolete; bitwise operations (&, |, ^, ~, <<, and >>) are always enabled LoadBindsGlobally = false // load creates global not file-local bindings (deprecated) + + // obsolete flags for features that are now standard. No effect. + AllowNestedDef = true + AllowLambda = true + AllowFloat = true + AllowBitwise = true ) // File resolves the specified file and records information about the @@ -128,7 +133,7 @@ func File(file *syntax.File, isPredeclared, isUniversal func(name string) bool) // REPLChunk is a generalization of the File function that supports a // non-empty initial global block, as occurs in a REPL. func REPLChunk(file *syntax.File, isGlobal, isPredeclared, isUniversal func(name string) bool) error { - r := newResolver(isGlobal, isPredeclared, isUniversal) + r := newResolver(file.Options, isGlobal, isPredeclared, isUniversal) r.stmts(file.Stmts) r.env.resolveLocalUses() @@ -149,12 +154,20 @@ func REPLChunk(file *syntax.File, isGlobal, isPredeclared, isUniversal func(name return nil } -// Expr resolves the specified expression. -// It returns the local variables bound within the expression. +// Expr calls [ExprOptions] using [syntax.LegacyFileOptions]. // -// The isPredeclared and isUniversal predicates behave as for the File function. +// Deprecated: use [ExprOptions] with [syntax.FileOptions] instead, +// because this function relies on legacy global variables. func Expr(expr syntax.Expr, isPredeclared, isUniversal func(name string) bool) ([]*Binding, error) { - r := newResolver(nil, isPredeclared, isUniversal) + return ExprOptions(syntax.LegacyFileOptions(), expr, isPredeclared, isUniversal) +} + +// ExprOptions resolves the specified expression. +// It returns the local variables bound within the expression. +// +// The isPredeclared and isUniversal predicates behave as for the File function +func ExprOptions(opts *syntax.FileOptions, expr syntax.Expr, isPredeclared, isUniversal func(name string) bool) ([]*Binding, error) { + r := newResolver(opts, nil, isPredeclared, isUniversal) r.expr(expr) r.env.resolveLocalUses() r.resolveNonLocalUses(r.env) // globals & universals @@ -177,9 +190,10 @@ type Error struct { func (e Error) Error() string { return e.Pos.String() + ": " + e.Msg } -func newResolver(isGlobal, isPredeclared, isUniversal func(name string) bool) *resolver { +func newResolver(options *syntax.FileOptions, isGlobal, isPredeclared, isUniversal func(name string) bool) *resolver { file := new(block) return &resolver{ + options: options, file: file, env: file, isGlobal: isGlobal, @@ -191,6 +205,8 @@ func newResolver(isGlobal, isPredeclared, isUniversal func(name string) bool) *r } type resolver struct { + options *syntax.FileOptions + // env is the current local environment: // a linked list of blocks, innermost first. // The tail of the list is the file block. @@ -214,7 +230,8 @@ type resolver struct { // isGlobal may be nil. isGlobal, isPredeclared, isUniversal func(name string) bool - loops int // number of enclosing for loops + loops int // number of enclosing for/while loops in current function (or file, if top-level) + ifstmts int // number of enclosing if statements loops errors ErrorList } @@ -311,7 +328,7 @@ func (r *resolver) bind(id *syntax.Ident) bool { r.moduleGlobals = append(r.moduleGlobals, bind) } } - if ok && !AllowGlobalReassign { + if ok && !r.options.GlobalReassign { r.errorf(id.NamePos, "cannot reassign %s %s declared at %s", bind.Scope, id.Name, bind.First.NamePos) } @@ -379,7 +396,7 @@ func (r *resolver) use(id *syntax.Ident) { // We will piggyback support for the legacy semantics on the // AllowGlobalReassign flag, which is loosely related and also // required for Bazel. - if AllowGlobalReassign && r.env == r.file { + if r.options.GlobalReassign && r.env == r.file { r.useToplevel(use) return } @@ -417,10 +434,7 @@ func (r *resolver) useToplevel(use use) (bind *Binding) { r.predeclared[id.Name] = bind // save it } else if r.isUniversal(id.Name) { // use of universal name - if !AllowFloat && id.Name == "float" { - r.errorf(id.NamePos, doesnt+"support floating point") - } - if !AllowSet && id.Name == "set" { + if !r.options.Set && id.Name == "set" { r.errorf(id.NamePos, doesnt+"support sets") } bind = &Binding{Scope: Universal} @@ -493,12 +507,14 @@ func (r *resolver) stmt(stmt syntax.Stmt) { } case *syntax.IfStmt: - if !AllowGlobalReassign && r.container().function == nil { + if !r.options.TopLevelControl && r.container().function == nil { r.errorf(stmt.If, "if statement not within a function") } r.expr(stmt.Cond) + r.ifstmts++ r.stmts(stmt.True) r.stmts(stmt.False) + r.ifstmts-- case *syntax.AssignStmt: r.expr(stmt.RHS) @@ -506,9 +522,6 @@ func (r *resolver) stmt(stmt syntax.Stmt) { r.assign(stmt.LHS, isAugmented) case *syntax.DefStmt: - if !AllowNestedDef && r.container().function != nil { - r.errorf(stmt.Def, doesnt+"support nested def") - } r.bind(stmt.Name) fn := &Function{ Name: stmt.Name.Name, @@ -520,7 +533,7 @@ func (r *resolver) stmt(stmt syntax.Stmt) { r.function(fn, stmt.Def) case *syntax.ForStmt: - if !AllowGlobalReassign && r.container().function == nil { + if !r.options.TopLevelControl && r.container().function == nil { r.errorf(stmt.For, "for loop not within a function") } r.expr(stmt.X) @@ -531,10 +544,10 @@ func (r *resolver) stmt(stmt syntax.Stmt) { r.loops-- case *syntax.WhileStmt: - if !AllowRecursion { + if !r.options.While { r.errorf(stmt.While, doesnt+"support while loops") } - if !AllowGlobalReassign && r.container().function == nil { + if !r.options.TopLevelControl && r.container().function == nil { r.errorf(stmt.While, "while loop not within a function") } r.expr(stmt.Cond) @@ -551,8 +564,13 @@ func (r *resolver) stmt(stmt syntax.Stmt) { } case *syntax.LoadStmt: + // A load statement may not be nested in any other statement. if r.container().function != nil { r.errorf(stmt.Load, "load statement within a function") + } else if r.loops > 0 { + r.errorf(stmt.Load, "load statement within a loop") + } else if r.ifstmts > 0 { + r.errorf(stmt.Load, "load statement within a conditional") } for i, from := range stmt.From { @@ -565,9 +583,9 @@ func (r *resolver) stmt(stmt syntax.Stmt) { } id := stmt.To[i] - if LoadBindsGlobally { + if r.options.LoadBindsGlobally { r.bind(id) - } else if r.bindLocal(id) && !AllowGlobalReassign { + } else if r.bindLocal(id) && !r.options.GlobalReassign { // "Global" in AllowGlobalReassign is a misnomer for "toplevel". // Sadly we can't report the previous declaration // as id.Binding may not be set yet. @@ -597,9 +615,6 @@ func (r *resolver) assign(lhs syntax.Expr, isAugmented bool) { case *syntax.TupleExpr: // (x, y) = ... - if len(lhs.List) == 0 { - r.errorf(syntax.Start(lhs), "can't assign to ()") - } if isAugmented { r.errorf(syntax.Start(lhs), "can't use tuple expression in augmented assignment") } @@ -609,9 +624,6 @@ func (r *resolver) assign(lhs syntax.Expr, isAugmented bool) { case *syntax.ListExpr: // [x, y, z] = ... - if len(lhs.List) == 0 { - r.errorf(syntax.Start(lhs), "can't assign to []") - } if isAugmented { r.errorf(syntax.Start(lhs), "can't use list expression in augmented assignment") } @@ -634,9 +646,6 @@ func (r *resolver) expr(e syntax.Expr) { r.use(e) case *syntax.Literal: - if !AllowFloat && e.Token == syntax.FLOAT { - r.errorf(e.TokenPos, doesnt+"support floating point") - } case *syntax.ListExpr: for _, x := range e.List { @@ -711,9 +720,6 @@ func (r *resolver) expr(e syntax.Expr) { r.expr(e.X) case *syntax.BinaryExpr: - if !AllowFloat && e.Op == syntax.SLASH { - r.errorf(e.OpPos, doesnt+"support floating point (use //)") - } r.expr(e.X) r.expr(e.Y) @@ -748,11 +754,13 @@ func (r *resolver) expr(e syntax.Expr) { // k=v n++ if seenKwargs { - r.errorf(pos, "argument may not follow **kwargs") + r.errorf(pos, "keyword argument may not follow **kwargs") + } else if seenVarargs { + r.errorf(pos, "keyword argument may not follow *args") } x := binop.X.(*syntax.Ident) if seenName[x.Name] { - r.errorf(x.NamePos, "keyword argument %s repeated", x.Name) + r.errorf(x.NamePos, "keyword argument %q is repeated", x.Name) } else { if seenName == nil { seenName = make(map[string]bool) @@ -764,9 +772,9 @@ func (r *resolver) expr(e syntax.Expr) { // positional argument p++ if seenVarargs { - r.errorf(pos, "argument may not follow *args") + r.errorf(pos, "positional argument may not follow *args") } else if seenKwargs { - r.errorf(pos, "argument may not follow **kwargs") + r.errorf(pos, "positional argument may not follow **kwargs") } else if len(seenName) > 0 { r.errorf(pos, "positional argument may not follow named") } @@ -785,9 +793,6 @@ func (r *resolver) expr(e syntax.Expr) { } case *syntax.LambdaExpr: - if !AllowLambda { - r.errorf(e.Lambda, doesnt+"support lambda") - } fn := &Function{ Name: "lambda", Pos: e.Lambda, @@ -817,6 +822,10 @@ func (r *resolver) function(function *Function, pos syntax.Position) { b := &block{function: function} r.push(b) + // Save the current loop count and reset it for the new function + outerLoops := r.loops + r.loops = 0 + var seenOptional bool var star *syntax.UnaryExpr // * or *args param var starStar *syntax.Ident // **kwargs ident @@ -900,6 +909,9 @@ func (r *resolver) function(function *Function, pos syntax.Position) { // Leave function block. r.pop() + // Restore the outer loop count + r.loops = outerLoops + // References within the function body to globals are not // resolved until the end of the module. } diff --git a/vendor/go.starlark.net/starlark/debug.go b/vendor/go.starlark.net/starlark/debug.go index 22a21240f..bbb37b55f 100644 --- a/vendor/go.starlark.net/starlark/debug.go +++ b/vendor/go.starlark.net/starlark/debug.go @@ -1,41 +1,59 @@ package starlark -import "go.starlark.net/syntax" +import ( + "go.starlark.net/syntax" +) // This file defines an experimental API for the debugging tools. // Some of these declarations expose details of internal packages. // (The debugger makes liberal use of exported fields of unexported types.) // Breaking changes may occur without notice. -// Local returns the value of the i'th local variable. -// It may be nil if not yet assigned. +// A Binding is the name and position of a binding identifier. +type Binding struct { + Name string + Pos syntax.Position +} + +// NumLocals returns the number of local variables of this frame. +// It is zero unless fr.Callable() is a *Function. +func (fr *frame) NumLocals() int { return len(fr.locals) } + +// Local returns the binding (name and binding position) and value of +// the i'th local variable of the frame's function. +// Beware: the value may be nil if it has not yet been assigned! // -// Local may be called only for frames whose Callable is a *Function (a -// function defined by Starlark source code), and only while the frame -// is active; it will panic otherwise. +// The index i must be less than [NumLocals]. +// Local may be called only while the frame is active. // // This function is provided only for debugging tools. -// -// THIS API IS EXPERIMENTAL AND MAY CHANGE WITHOUT NOTICE. -func (fr *frame) Local(i int) Value { return fr.locals[i] } +func (fr *frame) Local(i int) (Binding, Value) { + return Binding(fr.callable.(*Function).funcode.Locals[i]), fr.locals[i] +} // DebugFrame is the debugger API for a frame of the interpreter's call stack. // // Most applications have no need for this API; use CallFrame instead. // +// It may be tempting to use this interface when implementing built-in +// functions. Beware that reflection over the call stack is easily +// abused, leading to built-in functions whose behavior is mysterious +// and unpredictable. +// // Clients must not retain a DebugFrame nor call any of its methods once // the current built-in call has returned or execution has resumed // after a breakpoint as this may have unpredictable effects, including // but not limited to retention of object that would otherwise be garbage. type DebugFrame interface { - Callable() Callable // returns the frame's function - Local(i int) Value // returns the value of the (Starlark) frame's ith local variable - Position() syntax.Position // returns the current position of execution in this frame + Callable() Callable // returns the frame's function + NumLocals() int // returns the number of local variables in this frame + Local(i int) (Binding, Value) // returns the binding and value of the (Starlark) frame's ith local variable + Position() syntax.Position // returns the current position of execution in this frame } // DebugFrame returns the debugger interface for // the specified frame of the interpreter's call stack. -// Frame numbering is as for Thread.CallFrame. +// Frame numbering is as for Thread.CallFrame: 0 <= depth < thread.CallStackDepth(). // // This function is intended for use in debugging tools. // Most applications should have no need for it; use CallFrame instead. diff --git a/vendor/go.starlark.net/starlark/eval.go b/vendor/go.starlark.net/starlark/eval.go index de492ca3c..04e1a02a6 100644 --- a/vendor/go.starlark.net/starlark/eval.go +++ b/vendor/go.starlark.net/starlark/eval.go @@ -7,15 +7,16 @@ package starlark import ( "fmt" "io" - "io/ioutil" "log" - "math" "math/big" + "math/bits" "sort" "strings" + "sync/atomic" "time" "unicode" "unicode/utf8" + "unsafe" "go.starlark.net/internal/compile" "go.starlark.net/internal/spell" @@ -46,6 +47,21 @@ type Thread struct { // See example_test.go for some example implementations of Load. Load func(thread *Thread, module string) (StringDict, error) + // OnMaxSteps is called when the thread reaches the limit set by SetMaxExecutionSteps. + // The default behavior is to call thread.Cancel("too many steps"). + OnMaxSteps func(thread *Thread) + + // Steps a count of abstract computation steps executed + // by this thread. It is incremented by the interpreter. It may be used + // as a measure of the approximate cost of Starlark execution, by + // computing the difference in its value before and after a computation. + // + // The precise meaning of "step" is not specified and may change. + Steps, maxSteps uint64 + + // cancelReason records the reason from the first call to Cancel. + cancelReason *string + // locals holds arbitrary "thread-local" Go values belonging to the client. // They are accessible to the client but not to any Starlark program. locals map[string]interface{} @@ -54,6 +70,42 @@ type Thread struct { proftime time.Duration } +// ExecutionSteps returns the current value of Steps. +func (thread *Thread) ExecutionSteps() uint64 { + return thread.Steps +} + +// SetMaxExecutionSteps sets a limit on the number of Starlark +// computation steps that may be executed by this thread. If the +// thread's step counter exceeds this limit, the interpreter calls +// the optional OnMaxSteps function or the default behavior +// of calling thread.Cancel("too many steps"). +func (thread *Thread) SetMaxExecutionSteps(max uint64) { + thread.maxSteps = max +} + +// Uncancel resets the cancellation state. +// +// Unlike most methods of Thread, it is safe to call Uncancel from any +// goroutine, even if the thread is actively executing. +func (thread *Thread) Uncancel() { + atomic.StorePointer((*unsafe.Pointer)(unsafe.Pointer(&thread.cancelReason)), nil) +} + +// Cancel causes execution of Starlark code in the specified thread to +// promptly fail with an EvalError that includes the specified reason. +// There may be a delay before the interpreter observes the cancellation +// if the thread is currently in a call to a built-in function. +// +// Call [Uncancel] to reset the cancellation state. +// +// Unlike most methods of Thread, it is safe to call Cancel from any +// goroutine, even if the thread is actively executing. +func (thread *Thread) Cancel(reason string) { + // Atomically set cancelReason, preserving earlier reason if any. + atomic.CompareAndSwapPointer((*unsafe.Pointer)(unsafe.Pointer(&thread.cancelReason)), nil, unsafe.Pointer(&reason)) +} + // SetLocal sets the thread-local value associated with the specified key. // It must not be called after execution begins. func (thread *Thread) SetLocal(key string, value interface{}) { @@ -178,7 +230,9 @@ func (stack *CallStack) Pop() CallFrame { // String returns a user-friendly description of the stack. func (stack CallStack) String() string { out := new(strings.Builder) - fmt.Fprintf(out, "Traceback (most recent call last):\n") + if len(stack) > 0 { + fmt.Fprintf(out, "Traceback (most recent call last):\n") + } for _, fr := range stack { fmt.Fprintf(out, " %s: in %s\n", fr.Pos, fr.Name) } @@ -220,7 +274,15 @@ func (e *EvalError) Error() string { return e.Msg } // Backtrace returns a user-friendly error message describing the stack // of calls that led to this error. func (e *EvalError) Backtrace() string { - return fmt.Sprintf("%sError: %s", e.CallStack, e.Msg) + // If the topmost stack frame is a built-in function, + // remove it from the stack and add print "Error in fn:". + stack := e.CallStack + suffix := "" + if last := len(stack) - 1; last >= 0 && stack[last].Pos.Filename() == builtinFilename { + suffix = " in " + stack[last].Name + stack = stack[:last] + } + return fmt.Sprintf("%sError%s: %s", stack, suffix, e.Msg) } func (e *EvalError) Unwrap() error { return e.cause } @@ -263,7 +325,15 @@ func (prog *Program) Write(out io.Writer) error { return err } -// ExecFile parses, resolves, and executes a Starlark file in the +// ExecFile calls [ExecFileOptions] using [syntax.LegacyFileOptions]. +// +// Deprecated: use [ExecFileOptions] with [syntax.FileOptions] instead, +// because this function relies on legacy global variables. +func ExecFile(thread *Thread, filename string, src interface{}, predeclared StringDict) (StringDict, error) { + return ExecFileOptions(syntax.LegacyFileOptions(), thread, filename, src, predeclared) +} + +// ExecFileOptions parses, resolves, and executes a Starlark file in the // specified global environment, which may be modified during execution. // // Thread is the state associated with the Starlark thread. @@ -278,11 +348,11 @@ func (prog *Program) Write(out io.Writer) error { // Execution does not modify this dictionary, though it may mutate // its values. // -// If ExecFile fails during evaluation, it returns an *EvalError +// If ExecFileOptions fails during evaluation, it returns an *EvalError // containing a backtrace. -func ExecFile(thread *Thread, filename string, src interface{}, predeclared StringDict) (StringDict, error) { +func ExecFileOptions(opts *syntax.FileOptions, thread *Thread, filename string, src interface{}, predeclared StringDict) (StringDict, error) { // Parse, resolve, and compile a Starlark source file. - _, mod, err := SourceProgram(filename, src, predeclared.Has) + _, mod, err := SourceProgramOptions(opts, filename, src, predeclared.Has) if err != nil { return nil, err } @@ -292,7 +362,15 @@ func ExecFile(thread *Thread, filename string, src interface{}, predeclared Stri return g, err } -// SourceProgram produces a new program by parsing, resolving, +// SourceProgram calls [SourceProgramOptions] using [syntax.LegacyFileOptions]. +// +// Deprecated: use [SourceProgramOptions] with [syntax.FileOptions] instead, +// because this function relies on legacy global variables. +func SourceProgram(filename string, src interface{}, isPredeclared func(string) bool) (*syntax.File, *Program, error) { + return SourceProgramOptions(syntax.LegacyFileOptions(), filename, src, isPredeclared) +} + +// SourceProgramOptions produces a new program by parsing, resolving, // and compiling a Starlark source file. // On success, it returns the parsed file and the compiled program. // The filename and src parameters are as for syntax.Parse. @@ -301,8 +379,8 @@ func ExecFile(thread *Thread, filename string, src interface{}, predeclared Stri // a pre-declared identifier of the current module. // Its typical value is predeclared.Has, // where predeclared is a StringDict of pre-declared values. -func SourceProgram(filename string, src interface{}, isPredeclared func(string) bool) (*syntax.File, *Program, error) { - f, err := syntax.Parse(filename, src, 0) +func SourceProgramOptions(opts *syntax.FileOptions, filename string, src interface{}, isPredeclared func(string) bool) (*syntax.File, *Program, error) { + f, err := opts.Parse(filename, src, 0) if err != nil { return nil, nil, err } @@ -334,7 +412,7 @@ func FileProgram(f *syntax.File, isPredeclared func(string) bool) (*Program, err } module := f.Module.(*resolve.Module) - compiled := compile.File(f.Stmts, pos, "", module.Locals, module.Globals) + compiled := compile.File(f.Options, f.Stmts, pos, "", module.Locals, module.Globals) return &Program{compiled}, nil } @@ -342,7 +420,7 @@ func FileProgram(f *syntax.File, isPredeclared func(string) bool) (*Program, err // CompiledProgram produces a new program from the representation // of a compiled program previously saved by Program.Write. func CompiledProgram(in io.Reader) (*Program, error) { - data, err := ioutil.ReadAll(in) + data, err := io.ReadAll(in) if err != nil { return nil, err } @@ -357,7 +435,7 @@ func CompiledProgram(in io.Reader) (*Program, error) { // executes the toplevel code of the specified program, // and returns a new, unfrozen dictionary of the globals. func (prog *Program) Init(thread *Thread, predeclared StringDict) (StringDict, error) { - toplevel := makeToplevelFunction(prog.compiled, predeclared) + toplevel := makeToplevelFunction(prog, predeclared) _, err := Call(thread, toplevel, nil, nil) @@ -391,12 +469,12 @@ func ExecREPLChunk(f *syntax.File, thread *Thread, globals StringDict) error { } module := f.Module.(*resolve.Module) - compiled := compile.File(f.Stmts, pos, "", module.Locals, module.Globals) + compiled := compile.File(f.Options, f.Stmts, pos, "", module.Locals, module.Globals) prog := &Program{compiled} // -- variant of Program.Init -- - toplevel := makeToplevelFunction(prog.compiled, predeclared) + toplevel := makeToplevelFunction(prog, predeclared) // Initialize module globals from parameter. for i, id := range prog.compiled.Globals { @@ -417,10 +495,10 @@ func ExecREPLChunk(f *syntax.File, thread *Thread, globals StringDict) error { return err } -func makeToplevelFunction(prog *compile.Program, predeclared StringDict) *Function { +func makeToplevelFunction(prog *Program, predeclared StringDict) *Function { // Create the Starlark value denoted by each program constant c. - constants := make([]Value, len(prog.Constants)) - for i, c := range prog.Constants { + constants := make([]Value, len(prog.compiled.Constants)) + for i, c := range prog.compiled.Constants { var v Value switch c := c.(type) { case int64: @@ -429,6 +507,8 @@ func makeToplevelFunction(prog *compile.Program, predeclared StringDict) *Functi v = MakeBigInt(c) case string: v = String(c) + case compile.Bytes: + v = Bytes(c) case float64: v = Float(c) default: @@ -438,17 +518,25 @@ func makeToplevelFunction(prog *compile.Program, predeclared StringDict) *Functi } return &Function{ - funcode: prog.Toplevel, - module: &module{ + funcode: prog.compiled.Toplevel, + module: &Module{ program: prog, predeclared: predeclared, - globals: make([]Value, len(prog.Globals)), + globals: make([]Value, len(prog.compiled.Globals)), constants: constants, }, } } -// Eval parses, resolves, and evaluates an expression within the +// Eval calls [EvalOptions] using [syntax.LegacyFileOptions]. +// +// Deprecated: use [EvalOptions] with [syntax.FileOptions] instead, +// because this function relies on legacy global variables. +func Eval(thread *Thread, filename string, src interface{}, env StringDict) (Value, error) { + return EvalOptions(syntax.LegacyFileOptions(), thread, filename, src, env) +} + +// EvalOptions parses, resolves, and evaluates an expression within the // specified (predeclared) environment. // // Evaluation cannot mutate the environment dictionary itself, @@ -456,58 +544,76 @@ func makeToplevelFunction(prog *compile.Program, predeclared StringDict) *Functi // // The filename and src parameters are as for syntax.Parse. // -// If Eval fails during evaluation, it returns an *EvalError +// If EvalOptions fails during evaluation, it returns an *EvalError // containing a backtrace. -func Eval(thread *Thread, filename string, src interface{}, env StringDict) (Value, error) { - expr, err := syntax.ParseExpr(filename, src, 0) +func EvalOptions(opts *syntax.FileOptions, thread *Thread, filename string, src interface{}, env StringDict) (Value, error) { + expr, err := opts.ParseExpr(filename, src, 0) if err != nil { return nil, err } - f, err := makeExprFunc(expr, env) + f, err := makeExprFunc(opts, expr, env) if err != nil { return nil, err } return Call(thread, f, nil, nil) } -// EvalExpr resolves and evaluates an expression within the +// EvalExpr calls [EvalExprOptions] using [syntax.LegacyFileOptions]. +// +// Deprecated: use [EvalExprOptions] with [syntax.FileOptions] instead, +// because this function relies on legacy global variables. +func EvalExpr(thread *Thread, expr syntax.Expr, env StringDict) (Value, error) { + return EvalExprOptions(syntax.LegacyFileOptions(), thread, expr, env) +} + +// EvalExprOptions resolves and evaluates an expression within the // specified (predeclared) environment. // Evaluating a comma-separated list of expressions yields a tuple value. // // Resolving an expression mutates it. -// Do not call EvalExpr more than once for the same expression. +// Do not call EvalExprOptions more than once for the same expression. // // Evaluation cannot mutate the environment dictionary itself, // though it may modify variables reachable from the dictionary. // -// If Eval fails during evaluation, it returns an *EvalError +// If EvalExprOptions fails during evaluation, it returns an *EvalError // containing a backtrace. -func EvalExpr(thread *Thread, expr syntax.Expr, env StringDict) (Value, error) { - fn, err := makeExprFunc(expr, env) +func EvalExprOptions(opts *syntax.FileOptions, thread *Thread, expr syntax.Expr, env StringDict) (Value, error) { + fn, err := makeExprFunc(opts, expr, env) if err != nil { return nil, err } return Call(thread, fn, nil, nil) } +// ExprFunc calls [ExprFuncOptions] using [syntax.LegacyFileOptions]. +// +// Deprecated: use [ExprFuncOptions] with [syntax.FileOptions] instead, +// because this function relies on legacy global variables. +func ExprFunc(filename string, src interface{}, env StringDict) (*Function, error) { + return ExprFuncOptions(syntax.LegacyFileOptions(), filename, src, env) +} + // ExprFunc returns a no-argument function // that evaluates the expression whose source is src. -func ExprFunc(filename string, src interface{}, env StringDict) (*Function, error) { - expr, err := syntax.ParseExpr(filename, src, 0) +func ExprFuncOptions(options *syntax.FileOptions, filename string, src interface{}, env StringDict) (*Function, error) { + expr, err := options.ParseExpr(filename, src, 0) if err != nil { return nil, err } - return makeExprFunc(expr, env) + return makeExprFunc(options, expr, env) } // makeExprFunc returns a no-argument function whose body is expr. -func makeExprFunc(expr syntax.Expr, env StringDict) (*Function, error) { - locals, err := resolve.Expr(expr, env.Has, Universe.Has) +// The options must be consistent with those used when parsing expr. +func makeExprFunc(opts *syntax.FileOptions, expr syntax.Expr, env StringDict) (*Function, error) { + locals, err := resolve.ExprOptions(opts, expr, env.Has, Universe.Has) if err != nil { return nil, err } - return makeToplevelFunction(compile.Expr(expr, "", locals), env), nil + prog := compile.Expr(opts, expr, "", locals) + return makeToplevelFunction(&Program{prog}, env), nil } // The following functions are primitive operations of the byte code interpreter. @@ -674,14 +780,22 @@ func Binary(op syntax.Token, x, y Value) (Value, error) { case Int: return x.Add(y), nil case Float: - return x.Float() + y, nil + xf, err := x.finiteFloat() + if err != nil { + return nil, err + } + return xf + y, nil } case Float: switch y := y.(type) { case Float: return x + y, nil case Int: - return x + y.Float(), nil + yf, err := y.finiteFloat() + if err != nil { + return nil, err + } + return x + yf, nil } case *List: if y, ok := y.(*List); ok { @@ -706,14 +820,28 @@ func Binary(op syntax.Token, x, y Value) (Value, error) { case Int: return x.Sub(y), nil case Float: - return x.Float() - y, nil + xf, err := x.finiteFloat() + if err != nil { + return nil, err + } + return xf - y, nil } case Float: switch y := y.(type) { case Float: return x - y, nil case Int: - return x - y.Float(), nil + yf, err := y.finiteFloat() + if err != nil { + return nil, err + } + return x - yf, nil + } + case *Set: // difference + if y, ok := y.(*Set); ok { + iter := y.Iterate() + defer iter.Done() + return x.Difference(iter) } } @@ -724,9 +852,15 @@ func Binary(op syntax.Token, x, y Value) (Value, error) { case Int: return x.Mul(y), nil case Float: - return x.Float() * y, nil + xf, err := x.finiteFloat() + if err != nil { + return nil, err + } + return xf * y, nil case String: return stringRepeat(y, x) + case Bytes: + return bytesRepeat(y, x) case *List: elems, err := tupleRepeat(Tuple(y.elems), x) if err != nil { @@ -741,12 +875,20 @@ func Binary(op syntax.Token, x, y Value) (Value, error) { case Float: return x * y, nil case Int: - return x * y.Float(), nil + yf, err := y.finiteFloat() + if err != nil { + return nil, err + } + return x * yf, nil } case String: if y, ok := y.(Int); ok { return stringRepeat(x, y) } + case Bytes: + if y, ok := y.(Int); ok { + return bytesRepeat(x, y) + } case *List: if y, ok := y.(Int); ok { elems, err := tupleRepeat(Tuple(x.elems), y) @@ -765,30 +907,40 @@ func Binary(op syntax.Token, x, y Value) (Value, error) { case syntax.SLASH: switch x := x.(type) { case Int: + xf, err := x.finiteFloat() + if err != nil { + return nil, err + } switch y := y.(type) { case Int: - yf := y.Float() + yf, err := y.finiteFloat() + if err != nil { + return nil, err + } if yf == 0.0 { - return nil, fmt.Errorf("real division by zero") + return nil, fmt.Errorf("floating-point division by zero") } - return x.Float() / yf, nil + return xf / yf, nil case Float: if y == 0.0 { - return nil, fmt.Errorf("real division by zero") + return nil, fmt.Errorf("floating-point division by zero") } - return x.Float() / y, nil + return xf / y, nil } case Float: switch y := y.(type) { case Float: if y == 0.0 { - return nil, fmt.Errorf("real division by zero") + return nil, fmt.Errorf("floating-point division by zero") } return x / y, nil case Int: - yf := y.Float() + yf, err := y.finiteFloat() + if err != nil { + return nil, err + } if yf == 0.0 { - return nil, fmt.Errorf("real division by zero") + return nil, fmt.Errorf("floating-point division by zero") } return x / yf, nil } @@ -804,10 +956,14 @@ func Binary(op syntax.Token, x, y Value) (Value, error) { } return x.Div(y), nil case Float: + xf, err := x.finiteFloat() + if err != nil { + return nil, err + } if y == 0.0 { return nil, fmt.Errorf("floored division by zero") } - return floor((x.Float() / y)), nil + return floor(xf / y), nil } case Float: switch y := y.(type) { @@ -817,7 +973,10 @@ func Binary(op syntax.Token, x, y Value) (Value, error) { } return floor(x / y), nil case Int: - yf := y.Float() + yf, err := y.finiteFloat() + if err != nil { + return nil, err + } if yf == 0.0 { return nil, fmt.Errorf("floored division by zero") } @@ -835,23 +994,31 @@ func Binary(op syntax.Token, x, y Value) (Value, error) { } return x.Mod(y), nil case Float: + xf, err := x.finiteFloat() + if err != nil { + return nil, err + } if y == 0 { - return nil, fmt.Errorf("float modulo by zero") + return nil, fmt.Errorf("floating-point modulo by zero") } - return x.Float().Mod(y), nil + return xf.Mod(y), nil } case Float: switch y := y.(type) { case Float: if y == 0.0 { - return nil, fmt.Errorf("float modulo by zero") + return nil, fmt.Errorf("floating-point modulo by zero") } - return Float(math.Mod(float64(x), float64(y))), nil + return x.Mod(y), nil case Int: if y.Sign() == 0 { - return nil, fmt.Errorf("float modulo by zero") + return nil, fmt.Errorf("floating-point modulo by zero") + } + yf, err := y.finiteFloat() + if err != nil { + return nil, err } - return x.Mod(y.Float()), nil + return x.Mod(yf), nil } case String: return interpolate(string(x), y) @@ -866,44 +1033,14 @@ func Binary(op syntax.Token, x, y Value) (Value, error) { case syntax.IN: switch y := y.(type) { - case *List: - for _, elem := range y.elems { - if eq, err := Equal(elem, x); err != nil { - return nil, err - } else if eq { - return True, nil - } - } - return False, nil - case Tuple: - for _, elem := range y { - if eq, err := Equal(elem, x); err != nil { - return nil, err - } else if eq { - return True, nil - } - } - return False, nil + case Container: // List, Tuple, Set, String, Bytes, rangeValue etc. + found, err := y.Has(x) + return Bool(found), err case Mapping: // e.g. dict // Ignore error from Get as we cannot distinguish true // errors (value cycle, type error) from "key not found". _, found, _ := y.Get(x) return Bool(found), nil - case *Set: - ok, err := y.Has(x) - return Bool(ok), err - case String: - needle, ok := x.(String) - if !ok { - return nil, fmt.Errorf("'in ' requires string as left operand, not %s", x.Type()) - } - return Bool(strings.Contains(string(y), string(needle))), nil - case rangeValue: - i, err := NumberToInt(x) - if err != nil { - return nil, fmt.Errorf("'in ' requires integer as left operand, not %s", x.Type()) - } - return Bool(y.contains(i)), nil } case syntax.PIPE: @@ -912,6 +1049,12 @@ func Binary(op syntax.Token, x, y Value) (Value, error) { if y, ok := y.(Int); ok { return x.Or(y), nil } + + case *Dict: // union + if y, ok := y.(*Dict); ok { + return x.Union(y), nil + } + case *Set: // union if y, ok := y.(*Set); ok { iter := Iterate(y) @@ -928,17 +1071,9 @@ func Binary(op syntax.Token, x, y Value) (Value, error) { } case *Set: // intersection if y, ok := y.(*Set); ok { - set := new(Set) - if x.Len() > y.Len() { - x, y = y, x // opt: range over smaller set - } - for _, xelem := range x.elems() { - // Has, Insert cannot fail here. - if found, _ := y.Has(xelem); found { - set.Insert(xelem) - } - } - return set, nil + iter := y.Iterate() + defer iter.Done() + return x.Intersection(iter) } } @@ -950,18 +1085,9 @@ func Binary(op syntax.Token, x, y Value) (Value, error) { } case *Set: // symmetric difference if y, ok := y.(*Set); ok { - set := new(Set) - for _, xelem := range x.elems() { - if found, _ := y.Has(xelem); !found { - set.Insert(xelem) - } - } - for _, yelem := range y.elems() { - if found, _ := x.Has(yelem); !found { - set.Insert(yelem) - } - } - return set, nil + iter := y.Iterate() + defer iter.Done() + return x.SymmetricDifference(iter) } } @@ -1025,9 +1151,10 @@ func tupleRepeat(elems Tuple, n Int) (Tuple, error) { return nil, nil } // Inv: i > 0, len > 0 - sz := len(elems) * i - if sz < 0 || sz >= maxAlloc { // sz < 0 => overflow - return nil, fmt.Errorf("excessive repeat (%d elements)", sz) + of, sz := bits.Mul(uint(len(elems)), uint(i)) + if of != 0 || sz >= maxAlloc { // of != 0 => overflow + // Don't print sz. + return nil, fmt.Errorf("excessive repeat (%d * %d elements)", len(elems), i) } res := make([]Value, sz) // copy elems into res, doubling each time @@ -1039,6 +1166,11 @@ func tupleRepeat(elems Tuple, n Int) (Tuple, error) { return res, nil } +func bytesRepeat(b Bytes, n Int) (Bytes, error) { + res, err := stringRepeat(String(b), n) + return Bytes(res), err +} + func stringRepeat(s String, n Int) (String, error) { if s == "" { return "", nil @@ -1051,9 +1183,10 @@ func stringRepeat(s String, n Int) (String, error) { return "", nil } // Inv: i > 0, len > 0 - sz := len(s) * i - if sz < 0 || sz >= maxAlloc { // sz < 0 => overflow - return "", fmt.Errorf("excessive repeat (%d elements)", sz) + of, sz := bits.Mul(uint(len(s)), uint(i)) + if of != 0 || sz >= maxAlloc { // of != 0 => overflow + // Don't print sz. + return "", fmt.Errorf("excessive repeat (%d * %d elements)", len(s), i) } return String(strings.Repeat(string(s), i)), nil } @@ -1075,13 +1208,35 @@ func Call(thread *Thread, fn Value, args Tuple, kwargs []Tuple) (Value, error) { if fr == nil { fr = new(frame) } + + if thread.stack == nil { + // one-time initialization of thread + if thread.maxSteps == 0 { + thread.maxSteps-- // (MaxUint64) + } + } + thread.stack = append(thread.stack, fr) // push fr.callable = c thread.beginProfSpan() + + // Use defer to ensure that panics from built-ins + // pass through the interpreter without leaving + // it in a bad state. + defer func() { + thread.endProfSpan() + + // clear out any references + // TODO(adonovan): opt: zero fr.Locals and + // reuse it if it is large enough. + *fr = frame{} + + thread.stack = thread.stack[:len(thread.stack)-1] // pop + }() + result, err := c.CallInternal(thread, args, kwargs) - thread.endProfSpan() // Sanity check: nil is not a valid Starlark value. if result == nil && err == nil { @@ -1095,9 +1250,6 @@ func Call(thread *Thread, fn Value, args Tuple, kwargs []Tuple) (Value, error) { } } - *fr = frame{} // clear out any references - thread.stack = thread.stack[:len(thread.stack)-1] // pop - return result, err } @@ -1113,7 +1265,7 @@ func slice(x, lo, hi, step_ Value) (Value, error) { var err error step, err = AsInt32(step_) if err != nil { - return nil, fmt.Errorf("got %s for slice step, want int", step_.Type()) + return nil, fmt.Errorf("invalid slice step: %s", err) } if step == 0 { return nil, fmt.Errorf("zero is not a valid slice step") @@ -1207,7 +1359,7 @@ func asIndex(v Value, len int, result *int) error { var err error *result, err = AsInt32(v) if err != nil { - return fmt.Errorf("got %s, want int", v.Type()) + return err } if *result < 0 { *result += len @@ -1448,20 +1600,7 @@ func interpolate(format string, x Value) (Value, error) { if !ok { return nil, fmt.Errorf("%%%c format requires float, not %s", c, arg.Type()) } - switch c { - case 'e': - fmt.Fprintf(buf, "%e", f) - case 'f': - fmt.Fprintf(buf, "%f", f) - case 'g': - fmt.Fprintf(buf, "%g", f) - case 'E': - fmt.Fprintf(buf, "%E", f) - case 'F': - fmt.Fprintf(buf, "%F", f) - case 'G': - fmt.Fprintf(buf, "%G", f) - } + Float(f).format(buf, c) case 'c': switch arg := arg.(type) { case Int: @@ -1489,9 +1628,14 @@ func interpolate(format string, x Value) (Value, error) { index++ } - if index < nargs { + if index < nargs && !is[Mapping](x) { return nil, fmt.Errorf("too many arguments for format string") } return String(buf.String()), nil } + +func is[T any](x any) bool { + _, ok := x.(T) + return ok +} diff --git a/vendor/go.starlark.net/starlark/hashtable.go b/vendor/go.starlark.net/starlark/hashtable.go index d4250194a..e1bbeaafc 100644 --- a/vendor/go.starlark.net/starlark/hashtable.go +++ b/vendor/go.starlark.net/starlark/hashtable.go @@ -6,12 +6,15 @@ package starlark import ( "fmt" - _ "unsafe" // for go:linkname hack + "hash/maphash" + "math/big" ) // hashtable is used to represent Starlark dict and set values. // It is a hash table whose key/value entries form a doubly-linked list // in the order the entries were inserted. +// +// Initialized instances of hashtable must not be copied. type hashtable struct { table []bucket // len is zero or a power of two bucket0 [1]bucket // inline allocation for small maps. @@ -20,8 +23,17 @@ type hashtable struct { head *entry // insertion order doubly-linked list; may be nil tailLink **entry // address of nil link at end of list (perhaps &head) frozen bool + + _ noCopy // triggers vet copylock check on this type. } +// noCopy is zero-sized type that triggers vet's copylock check. +// See https://github.com/golang/go/issues/8005#issuecomment-190753527. +type noCopy struct{} + +func (*noCopy) Lock() {} +func (*noCopy) Unlock() {} + const bucketSize = 8 type bucket struct { @@ -55,26 +67,16 @@ func (ht *hashtable) init(size int) { func (ht *hashtable) freeze() { if !ht.frozen { ht.frozen = true - for i := range ht.table { - for p := &ht.table[i]; p != nil; p = p.next { - for i := range p.entries { - e := &p.entries[i] - if e.hash != 0 { - e.key.Freeze() - e.value.Freeze() - } - } - } + for e := ht.head; e != nil; e = e.next { + e.key.Freeze() + e.value.Freeze() } } } func (ht *hashtable) insert(k, v Value) error { - if ht.frozen { - return fmt.Errorf("cannot insert into frozen hash table") - } - if ht.itercount > 0 { - return fmt.Errorf("cannot insert into hash table during iteration") + if err := ht.checkMutable("insert into"); err != nil { + return err } if ht.table == nil { ht.init(1) @@ -154,13 +156,12 @@ func overloaded(elems, buckets int) bool { func (ht *hashtable) grow() { // Double the number of buckets and rehash. - // TODO(adonovan): opt: - // - avoid reentrant calls to ht.insert, and specialize it. - // e.g. we know the calls to Equals will return false since - // there are no duplicates among the old keys. - // - saving the entire hash in the bucket would avoid the need to - // recompute the hash. - // - save the old buckets on a free list. + // + // Even though this makes reentrant calls to ht.insert, + // calls Equals unnecessarily (since there can't be duplicate keys), + // and recomputes the hash unnecessarily, the gains from + // avoiding these steps were found to be too small to justify + // the extra logic: -2% on hashtable benchmark. ht.table = make([]bucket, len(ht.table)<<1) oldhead := ht.head ht.head = nil @@ -200,6 +201,57 @@ func (ht *hashtable) lookup(k Value) (v Value, found bool, err error) { return None, false, nil // not found } +// count returns the number of distinct elements of iter that are elements of ht. +func (ht *hashtable) count(iter Iterator) (int, error) { + if ht.table == nil { + return 0, nil // empty + } + + var k Value + count := 0 + + // Use a bitset per table entry to record seen elements of ht. + // Elements are identified by their bucket number and index within the bucket. + // Each bitset gets one word initially, but may grow. + storage := make([]big.Word, len(ht.table)) + bitsets := make([]big.Int, len(ht.table)) + for i := range bitsets { + bitsets[i].SetBits(storage[i : i+1 : i+1]) + } + for iter.Next(&k) && count != int(ht.len) { + h, err := k.Hash() + if err != nil { + return 0, err // unhashable + } + if h == 0 { + h = 1 // zero is reserved + } + + // Inspect each bucket in the bucket list. + bucketId := h & (uint32(len(ht.table) - 1)) + i := 0 + for p := &ht.table[bucketId]; p != nil; p = p.next { + for j := range p.entries { + e := &p.entries[j] + if e.hash == h { + if eq, err := Equal(k, e.key); err != nil { + return 0, err + } else if eq { + bitIndex := i<<3 + j + if bitsets[bucketId].Bit(bitIndex) == 0 { + bitsets[bucketId].SetBit(&bitsets[bucketId], bitIndex, 1) + count++ + } + } + } + } + i++ + } + } + + return count, nil +} + // Items returns all the items in the map (as key/value pairs) in insertion order. func (ht *hashtable) items() []Tuple { items := make([]Tuple, 0, ht.len) @@ -230,11 +282,8 @@ func (ht *hashtable) keys() []Value { } func (ht *hashtable) delete(k Value) (v Value, found bool, err error) { - if ht.frozen { - return nil, false, fmt.Errorf("cannot delete from frozen hash table") - } - if ht.itercount > 0 { - return nil, false, fmt.Errorf("cannot delete from hash table during iteration") + if err := ht.checkMutable("delete from"); err != nil { + return nil, false, err } if ht.table == nil { return None, false, nil // empty @@ -277,12 +326,21 @@ func (ht *hashtable) delete(k Value) (v Value, found bool, err error) { return None, false, nil // not found } -func (ht *hashtable) clear() error { +// checkMutable reports an error if the hash table should not be mutated. +// verb+" dict" should describe the operation. +func (ht *hashtable) checkMutable(verb string) error { if ht.frozen { - return fmt.Errorf("cannot clear frozen hash table") + return fmt.Errorf("cannot %s frozen hash table", verb) } if ht.itercount > 0 { - return fmt.Errorf("cannot clear hash table during iteration") + return fmt.Errorf("cannot %s hash table during iteration", verb) + } + return nil +} + +func (ht *hashtable) clear() error { + if err := ht.checkMutable("clear"); err != nil { + return err } if ht.table != nil { for i := range ht.table { @@ -295,6 +353,15 @@ func (ht *hashtable) clear() error { return nil } +func (ht *hashtable) addAll(other *hashtable) error { + for e := other.head; e != nil; e = e.next { + if err := ht.insert(e.key, e.value); err != nil { + return err + } + } + return nil +} + // dump is provided as an aid to debugging. func (ht *hashtable) dump() { fmt.Printf("hashtable %p len=%d head=%p tailLink=%p", @@ -349,22 +416,32 @@ func (it *keyIterator) Done() { } } +// entries is a go1.23 iterator over the entries of the hash table. +func (ht *hashtable) entries(yield func(k, v Value) bool) { + if !ht.frozen { + ht.itercount++ + defer func() { ht.itercount-- }() + } + for e := ht.head; e != nil && yield(e.key, e.value); e = e.next { + } +} + +var seed = maphash.MakeSeed() + // hashString computes the hash of s. func hashString(s string) uint32 { if len(s) >= 12 { // Call the Go runtime's optimized hash implementation, - // which uses the AESENC instruction on amd64 machines. - return uint32(goStringHash(s, 0)) + // which uses the AES instructions on amd64 and arm64 machines. + h := maphash.String(seed, s) + return uint32(h>>32) | uint32(h) } return softHashString(s) } -//go:linkname goStringHash runtime.stringHash -func goStringHash(s string, seed uintptr) uintptr - -// softHashString computes the FNV hash of s in software. +// softHashString computes the 32-bit FNV-1a hash of s in software. func softHashString(s string) uint32 { - var h uint32 + var h uint32 = 2166136261 for i := 0; i < len(s); i++ { h ^= uint32(s[i]) h *= 16777619 diff --git a/vendor/go.starlark.net/starlark/int.go b/vendor/go.starlark.net/starlark/int.go index 35bd42b3f..8f2b279c7 100644 --- a/vendor/go.starlark.net/starlark/int.go +++ b/vendor/go.starlark.net/starlark/int.go @@ -8,33 +8,18 @@ import ( "fmt" "math" "math/big" + "reflect" "strconv" "go.starlark.net/syntax" ) // Int is the type of a Starlark int. -type Int struct { - // We use only the signed 32 bit range of small to ensure - // that small+small and small*small do not overflow. +// +// The zero value is not a legal value; use MakeInt(0). +type Int struct{ impl intImpl } - small int64 // minint32 <= small <= maxint32 - big *big.Int // big != nil <=> value is not representable as int32 -} - -// newBig allocates a new big.Int. -func newBig(x int64) *big.Int { - if 0 <= x && int64(big.Word(x)) == x { - // x is guaranteed to fit into a single big.Word. - // Most starlark ints are small, - // but math/big assumes that since you've chosen to use math/big, - // your big.Ints will probably grow, so it over-allocates. - // Avoid that over-allocation by manually constructing a single-word slice. - // See https://golang.org/cl/150999, which will hopefully land in Go 1.13. - return new(big.Int).SetBits([]big.Word{big.Word(x)}) - } - return big.NewInt(x) -} +// --- high-level accessors --- // MakeInt returns a Starlark int for the specified signed integer. func MakeInt(x int) Int { return MakeInt64(int64(x)) } @@ -42,9 +27,9 @@ func MakeInt(x int) Int { return MakeInt64(int64(x)) } // MakeInt64 returns a Starlark int for the specified int64. func MakeInt64(x int64) Int { if math.MinInt32 <= x && x <= math.MaxInt32 { - return Int{small: x} + return makeSmallInt(x) } - return Int{big: newBig(x)} + return makeBigInt(big.NewInt(x)) } // MakeUint returns a Starlark int for the specified unsigned integer. @@ -53,27 +38,29 @@ func MakeUint(x uint) Int { return MakeUint64(uint64(x)) } // MakeUint64 returns a Starlark int for the specified uint64. func MakeUint64(x uint64) Int { if x <= math.MaxInt32 { - return Int{small: int64(x)} + return makeSmallInt(int64(x)) } - if uint64(big.Word(x)) == x { - // See comment in newBig for an explanation of this optimization. - return Int{big: new(big.Int).SetBits([]big.Word{big.Word(x)})} - } - return Int{big: new(big.Int).SetUint64(x)} + return makeBigInt(new(big.Int).SetUint64(x)) } // MakeBigInt returns a Starlark int for the specified big.Int. -// The caller must not subsequently modify x. +// The new Int value will contain a copy of x. The caller is safe to modify x. func MakeBigInt(x *big.Int) Int { - if n := x.BitLen(); n < 32 || n == 32 && x.Int64() == math.MinInt32 { - return Int{small: x.Int64()} + if isSmall(x) { + return makeSmallInt(x.Int64()) } - return Int{big: x} + z := new(big.Int).Set(x) + return makeBigInt(z) +} + +func isSmall(x *big.Int) bool { + n := x.BitLen() + return n < 32 || n == 32 && x.Int64() == math.MinInt32 } var ( - zero, one = Int{small: 0}, Int{small: 1} - oneBig = newBig(1) + zero, one = makeSmallInt(0), makeSmallInt(1) + oneBig = big.NewInt(1) _ HasUnary = Int{} ) @@ -94,39 +81,52 @@ func (i Int) Unary(op syntax.Token) (Value, error) { // Int64 returns the value as an int64. // If it is not exactly representable the result is undefined and ok is false. func (i Int) Int64() (_ int64, ok bool) { - if i.big != nil { - x, acc := bigintToInt64(i.big) + iSmall, iBig := i.get() + if iBig != nil { + x, acc := bigintToInt64(iBig) if acc != big.Exact { return // inexact } return x, true } - return i.small, true + return iSmall, true } -// BigInt returns the value as a big.Int. -// The returned variable must not be modified by the client. +// BigInt returns a new big.Int with the same value as the Int. func (i Int) BigInt() *big.Int { - if i.big != nil { - return i.big + iSmall, iBig := i.get() + if iBig != nil { + return new(big.Int).Set(iBig) } - return newBig(i.small) + return big.NewInt(iSmall) +} + +// bigInt returns the value as a big.Int. +// It differs from BigInt in that this method returns the actual +// reference and any modification will change the state of i. +func (i Int) bigInt() *big.Int { + iSmall, iBig := i.get() + if iBig != nil { + return iBig + } + return big.NewInt(iSmall) } // Uint64 returns the value as a uint64. // If it is not exactly representable the result is undefined and ok is false. func (i Int) Uint64() (_ uint64, ok bool) { - if i.big != nil { - x, acc := bigintToUint64(i.big) + iSmall, iBig := i.get() + if iBig != nil { + x, acc := bigintToUint64(iBig) if acc != big.Exact { return // inexact } return x, true } - if i.small < 0 { + if iSmall < 0 { return // inexact } - return uint64(i.small), true + return uint64(iSmall), true } // The math/big API should provide this function. @@ -163,104 +163,152 @@ var ( ) func (i Int) Format(s fmt.State, ch rune) { - if i.big != nil { - i.big.Format(s, ch) + iSmall, iBig := i.get() + if iBig != nil { + iBig.Format(s, ch) return } - newBig(i.small).Format(s, ch) + big.NewInt(iSmall).Format(s, ch) } func (i Int) String() string { - if i.big != nil { - return i.big.Text(10) + iSmall, iBig := i.get() + if iBig != nil { + return iBig.Text(10) } - return strconv.FormatInt(i.small, 10) + return strconv.FormatInt(iSmall, 10) } func (i Int) Type() string { return "int" } func (i Int) Freeze() {} // immutable func (i Int) Truth() Bool { return i.Sign() != 0 } func (i Int) Hash() (uint32, error) { + iSmall, iBig := i.get() var lo big.Word - if i.big != nil { - lo = i.big.Bits()[0] + if iBig != nil { + lo = iBig.Bits()[0] } else { - lo = big.Word(i.small) + lo = big.Word(iSmall) } return 12582917 * uint32(lo+3), nil } -func (x Int) CompareSameType(op syntax.Token, v Value, depth int) (bool, error) { - y := v.(Int) - if x.big != nil || y.big != nil { - return threeway(op, x.BigInt().Cmp(y.BigInt())), nil + +// Cmp implements comparison of two Int values. +// Required by the TotallyOrdered interface. +func (i Int) Cmp(v Value, depth int) (int, error) { + j := v.(Int) + iSmall, iBig := i.get() + jSmall, jBig := j.get() + if iBig != nil || jBig != nil { + return i.bigInt().Cmp(j.bigInt()), nil } - return threeway(op, signum64(x.small-y.small)), nil + return signum64(iSmall - jSmall), nil // safe: int32 operands } // Float returns the float value nearest i. func (i Int) Float() Float { - if i.big != nil { - f, _ := new(big.Float).SetInt(i.big).Float64() + iSmall, iBig := i.get() + if iBig != nil { + // Fast path for hardware int-to-float conversions. + if iBig.IsUint64() { + return Float(iBig.Uint64()) + } else if iBig.IsInt64() { + return Float(iBig.Int64()) + } else { + // Fast path for very big ints. + const maxFiniteLen = 1023 + 1 // max exponent value + implicit mantissa bit + if iBig.BitLen() > maxFiniteLen { + return Float(math.Inf(iBig.Sign())) + } + } + + f, _ := new(big.Float).SetInt(iBig).Float64() return Float(f) } - return Float(i.small) + return Float(iSmall) +} + +// finiteFloat returns the finite float value nearest i, +// or an error if the magnitude is too large. +func (i Int) finiteFloat() (Float, error) { + f := i.Float() + if math.IsInf(float64(f), 0) { + return 0, fmt.Errorf("int too large to convert to float") + } + return f, nil } func (x Int) Sign() int { - if x.big != nil { - return x.big.Sign() + xSmall, xBig := x.get() + if xBig != nil { + return xBig.Sign() } - return signum64(x.small) + return signum64(xSmall) } func (x Int) Add(y Int) Int { - if x.big != nil || y.big != nil { - return MakeBigInt(new(big.Int).Add(x.BigInt(), y.BigInt())) + xSmall, xBig := x.get() + ySmall, yBig := y.get() + if xBig != nil || yBig != nil { + return MakeBigInt(new(big.Int).Add(x.bigInt(), y.bigInt())) } - return MakeInt64(x.small + y.small) + return MakeInt64(xSmall + ySmall) } func (x Int) Sub(y Int) Int { - if x.big != nil || y.big != nil { - return MakeBigInt(new(big.Int).Sub(x.BigInt(), y.BigInt())) + xSmall, xBig := x.get() + ySmall, yBig := y.get() + if xBig != nil || yBig != nil { + return MakeBigInt(new(big.Int).Sub(x.bigInt(), y.bigInt())) } - return MakeInt64(x.small - y.small) + return MakeInt64(xSmall - ySmall) } func (x Int) Mul(y Int) Int { - if x.big != nil || y.big != nil { - return MakeBigInt(new(big.Int).Mul(x.BigInt(), y.BigInt())) + xSmall, xBig := x.get() + ySmall, yBig := y.get() + if xBig != nil || yBig != nil { + return MakeBigInt(new(big.Int).Mul(x.bigInt(), y.bigInt())) } - return MakeInt64(x.small * y.small) + return MakeInt64(xSmall * ySmall) } func (x Int) Or(y Int) Int { - if x.big != nil || y.big != nil { - return Int{big: new(big.Int).Or(x.BigInt(), y.BigInt())} + xSmall, xBig := x.get() + ySmall, yBig := y.get() + if xBig != nil || yBig != nil { + return MakeBigInt(new(big.Int).Or(x.bigInt(), y.bigInt())) } - return Int{small: x.small | y.small} + return makeSmallInt(xSmall | ySmall) } func (x Int) And(y Int) Int { - if x.big != nil || y.big != nil { - return MakeBigInt(new(big.Int).And(x.BigInt(), y.BigInt())) + xSmall, xBig := x.get() + ySmall, yBig := y.get() + if xBig != nil || yBig != nil { + return MakeBigInt(new(big.Int).And(x.bigInt(), y.bigInt())) } - return Int{small: x.small & y.small} + return makeSmallInt(xSmall & ySmall) } func (x Int) Xor(y Int) Int { - if x.big != nil || y.big != nil { - return MakeBigInt(new(big.Int).Xor(x.BigInt(), y.BigInt())) + xSmall, xBig := x.get() + ySmall, yBig := y.get() + if xBig != nil || yBig != nil { + return MakeBigInt(new(big.Int).Xor(x.bigInt(), y.bigInt())) } - return Int{small: x.small ^ y.small} + return makeSmallInt(xSmall ^ ySmall) } func (x Int) Not() Int { - if x.big != nil { - return MakeBigInt(new(big.Int).Not(x.big)) + xSmall, xBig := x.get() + if xBig != nil { + return MakeBigInt(new(big.Int).Not(xBig)) } - return Int{small: ^x.small} + return makeSmallInt(^xSmall) } -func (x Int) Lsh(y uint) Int { return MakeBigInt(new(big.Int).Lsh(x.BigInt(), y)) } -func (x Int) Rsh(y uint) Int { return MakeBigInt(new(big.Int).Rsh(x.BigInt(), y)) } +func (x Int) Lsh(y uint) Int { return MakeBigInt(new(big.Int).Lsh(x.bigInt(), y)) } +func (x Int) Rsh(y uint) Int { return MakeBigInt(new(big.Int).Rsh(x.bigInt(), y)) } // Precondition: y is nonzero. func (x Int) Div(y Int) Int { + xSmall, xBig := x.get() + ySmall, yBig := y.get() // http://python-history.blogspot.com/2010/08/why-pythons-integer-division-floors.html - if x.big != nil || y.big != nil { - xb, yb := x.BigInt(), y.BigInt() + if xBig != nil || yBig != nil { + xb, yb := x.bigInt(), y.bigInt() var quo, rem big.Int quo.QuoRem(xb, yb, &rem) @@ -269,9 +317,9 @@ func (x Int) Div(y Int) Int { } return MakeBigInt(&quo) } - quo := x.small / y.small - rem := x.small % y.small - if (x.small < 0) != (y.small < 0) && rem != 0 { + quo := xSmall / ySmall + rem := xSmall % ySmall + if (xSmall < 0) != (ySmall < 0) && rem != 0 { quo -= 1 } return MakeInt64(quo) @@ -279,8 +327,10 @@ func (x Int) Div(y Int) Int { // Precondition: y is nonzero. func (x Int) Mod(y Int) Int { - if x.big != nil || y.big != nil { - xb, yb := x.BigInt(), y.BigInt() + xSmall, xBig := x.get() + ySmall, yBig := y.get() + if xBig != nil || yBig != nil { + xb, yb := x.bigInt(), y.bigInt() var quo, rem big.Int quo.QuoRem(xb, yb, &rem) @@ -289,18 +339,19 @@ func (x Int) Mod(y Int) Int { } return MakeBigInt(&rem) } - rem := x.small % y.small - if (x.small < 0) != (y.small < 0) && rem != 0 { - rem += y.small + rem := xSmall % ySmall + if (xSmall < 0) != (ySmall < 0) && rem != 0 { + rem += ySmall } - return Int{small: rem} + return makeSmallInt(rem) } func (i Int) rational() *big.Rat { - if i.big != nil { - return new(big.Rat).SetInt(i.big) + iSmall, iBig := i.get() + if iBig != nil { + return new(big.Rat).SetInt(iBig) } - return new(big.Rat).SetInt64(i.small) + return new(big.Rat).SetInt64(iSmall) } // AsInt32 returns the value of x if is representable as an int32. @@ -309,10 +360,66 @@ func AsInt32(x Value) (int, error) { if !ok { return 0, fmt.Errorf("got %s, want int", x.Type()) } - if i.big != nil { + iSmall, iBig := i.get() + if iBig != nil { return 0, fmt.Errorf("%s out of range", i) } - return int(i.small), nil + return int(iSmall), nil +} + +// AsInt sets *ptr to the value of Starlark int x, if it is exactly representable, +// otherwise it returns an error. +// The type of ptr must be one of the pointer types *int, *int8, *int16, *int32, or *int64, +// or one of their unsigned counterparts including *uintptr. +func AsInt(x Value, ptr interface{}) error { + xint, ok := x.(Int) + if !ok { + return fmt.Errorf("got %s, want int", x.Type()) + } + + bits := reflect.TypeOf(ptr).Elem().Size() * 8 + switch ptr.(type) { + case *int, *int8, *int16, *int32, *int64: + i, ok := xint.Int64() + if !ok || bits < 64 && !(-1<<(bits-1) <= i && i < 1<<(bits-1)) { + return fmt.Errorf("%s out of range (want value in signed %d-bit range)", xint, bits) + } + switch ptr := ptr.(type) { + case *int: + *ptr = int(i) + case *int8: + *ptr = int8(i) + case *int16: + *ptr = int16(i) + case *int32: + *ptr = int32(i) + case *int64: + *ptr = int64(i) + } + + case *uint, *uint8, *uint16, *uint32, *uint64, *uintptr: + i, ok := xint.Uint64() + if !ok || bits < 64 && i >= 1< value is not representable as int32 +} + +// --- low-level accessors --- + +// get returns the small and big components of the Int. +// small is defined only if big is nil. +// small is sign-extended to 64 bits for ease of subsequent arithmetic. +func (i Int) get() (small int64, big *big.Int) { + return i.impl.small_, i.impl.big_ +} + +// Precondition: math.MinInt32 <= x && x <= math.MaxInt32 +func makeSmallInt(x int64) Int { + return Int{intImpl{small_: x}} +} + +// Precondition: x cannot be represented as int32. +func makeBigInt(x *big.Int) Int { + return Int{intImpl{big_: x}} +} diff --git a/vendor/go.starlark.net/starlark/int_posix64.go b/vendor/go.starlark.net/starlark/int_posix64.go new file mode 100644 index 000000000..70c949efc --- /dev/null +++ b/vendor/go.starlark.net/starlark/int_posix64.go @@ -0,0 +1,89 @@ +//go:build (linux || darwin || dragonfly || freebsd || netbsd || solaris) && (amd64 || arm64 || mips64x || ppc64 || ppc64le || loong64 || s390x) + +package starlark + +// This file defines an optimized Int implementation for 64-bit machines +// running POSIX. It reserves a 4GB portion of the address space using +// mmap and represents int32 values as addresses within that range. This +// disambiguates int32 values from *big.Int pointers, letting all Int +// values be represented as an unsafe.Pointer, so that Int-to-Value +// interface conversion need not allocate. + +// Although iOS (which, like macOS, appears as darwin/arm64) is +// POSIX-compliant, it limits each process to about 700MB of virtual +// address space, which defeats the optimization. Similarly, +// OpenBSD's default ulimit for virtual memory is a measly GB or so. +// On both those platforms the attempted optimization will fail and +// fall back to the slow implementation. + +// An alternative approach to this optimization would be to embed the +// int32 values in pointers using odd values, which can be distinguished +// from (even) *big.Int pointers. However, the Go runtime does not allow +// user programs to manufacture pointers to arbitrary locations such as +// within the zero page, or non-span, non-mmap, non-stack locations, +// and it may panic if it encounters them; see Issue #382. + +import ( + "log" + "math" + "math/big" + "unsafe" + + "golang.org/x/sys/unix" +) + +// intImpl represents a union of (int32, *big.Int) in a single pointer, +// so that Int-to-Value conversions need not allocate. +// +// The pointer is either a *big.Int, if the value is big, or a pointer into a +// reserved portion of the address space (smallints), if the value is small +// and the address space allocation succeeded. +// +// See int_generic.go for the basic representation concepts. +type intImpl unsafe.Pointer + +// get returns the (small, big) arms of the union. +func (i Int) get() (int64, *big.Int) { + if smallints == 0 { + // optimization disabled + if x := (*big.Int)(i.impl); isSmall(x) { + return x.Int64(), nil + } else { + return 0, x + } + } + + if ptr := uintptr(i.impl); ptr >= smallints && ptr < smallints+1<<32 { + return math.MinInt32 + int64(ptr-smallints), nil + } + return 0, (*big.Int)(i.impl) +} + +// Precondition: math.MinInt32 <= x && x <= math.MaxInt32 +func makeSmallInt(x int64) Int { + if smallints == 0 { + // optimization disabled + return Int{intImpl(big.NewInt(x))} + } + + return Int{intImpl(uintptr(x-math.MinInt32) + smallints)} +} + +// Precondition: x cannot be represented as int32. +func makeBigInt(x *big.Int) Int { return Int{intImpl(x)} } + +// smallints is the base address of a 2^32 byte memory region. +// Pointers to addresses in this region represent int32 values. +// We assume smallints is not at the very top of the address space. +// +// Zero means the optimization is disabled and all Ints allocate a big.Int. +var smallints = reserveAddresses(1 << 32) + +func reserveAddresses(len int) uintptr { + b, err := unix.Mmap(-1, 0, len, unix.PROT_READ, unix.MAP_PRIVATE|unix.MAP_ANON) + if err != nil { + log.Printf("Starlark failed to allocate 4GB address space: %v. Integer performance may suffer.", err) + return 0 // optimization disabled + } + return uintptr(unsafe.Pointer(&b[0])) +} diff --git a/vendor/go.starlark.net/starlark/interp.go b/vendor/go.starlark.net/starlark/interp.go index 529073008..261077fb6 100644 --- a/vendor/go.starlark.net/starlark/interp.go +++ b/vendor/go.starlark.net/starlark/interp.go @@ -5,10 +5,11 @@ package starlark import ( "fmt" "os" + "sync/atomic" + "unsafe" "go.starlark.net/internal/compile" "go.starlark.net/internal/spell" - "go.starlark.net/resolve" "go.starlark.net/syntax" ) @@ -19,19 +20,22 @@ const vmdebug = false // TODO(adonovan): use a bitfield of specific kinds of err // - opt: record MaxIterStack during compilation and preallocate the stack. func (fn *Function) CallInternal(thread *Thread, args Tuple, kwargs []Tuple) (Value, error) { - if !resolve.AllowRecursion { + // Postcondition: args is not mutated. This is stricter than required by Callable, + // but allows CALL to avoid a copy. + + f := fn.funcode + if !f.Prog.Recursion { // detect recursion for _, fr := range thread.stack[:len(thread.stack)-1] { // We look for the same function code, // not function value, otherwise the user could // defeat the check by writing the Y combinator. - if frfn, ok := fr.Callable().(*Function); ok && frfn.funcode == fn.funcode { + if frfn, ok := fr.Callable().(*Function); ok && frfn.funcode == f { return nil, fmt.Errorf("function %s called recursively", fn.Name()) } } } - f := fn.funcode fr := thread.frameAt(0) // Allocate space for stack and locals. @@ -76,12 +80,36 @@ func (fn *Function) CallInternal(thread *Thread, args Tuple, kwargs []Tuple) (Va var iterstack []Iterator // stack of active iterators + // Use defer so that application panics can pass through + // interpreter without leaving thread in a bad state. + defer func() { + // ITERPOP the rest of the iterator stack. + for _, iter := range iterstack { + iter.Done() + } + + fr.locals = nil + }() + sp := 0 var pc uint32 var result Value code := f.Code loop: for { + thread.Steps++ + if thread.Steps >= thread.maxSteps { + if thread.OnMaxSteps != nil { + thread.OnMaxSteps(thread) + } else { + thread.Cancel("too many steps") + } + } + if reason := atomic.LoadPointer((*unsafe.Pointer)(unsafe.Pointer(&thread.cancelReason))); reason != nil { + err = fmt.Errorf("Starlark computation cancelled: %s", *(*string)(reason)) + break loop + } + fr.pc = pc op := compile.Opcode(code[pc]) @@ -206,6 +234,34 @@ loop: stack[sp] = z sp++ + case compile.INPLACE_PIPE: + y := stack[sp-1] + x := stack[sp-2] + sp -= 2 + + // It's possible that y is not Dict but + // nonetheless defines x|y, in which case we + // should fall back to the general case. + var z Value + if xdict, ok := x.(*Dict); ok { + if ydict, ok := y.(*Dict); ok { + if err = xdict.ht.checkMutable("apply |= to"); err != nil { + break loop + } + xdict.ht.addAll(&ydict.ht) // can't fail + z = xdict + } + } + if z == nil { + z, err = Binary(syntax.PIPE, x, y) + if err != nil { + break loop + } + } + + stack[sp] = z + sp++ + case compile.NONE: stack[sp] = None sp++ @@ -276,9 +332,15 @@ loop: // positional args var positional Tuple if npos := int(arg >> 8); npos > 0 { - positional = make(Tuple, npos) + positional = stack[sp-npos : sp] sp -= npos - copy(positional, stack[sp:]) + + // Copy positional arguments into a new array, + // unless the callee is another Starlark function, + // in which case it can be trusted not to mutate them. + if _, ok := stack[sp-1].(*Function); !ok || args != nil { + positional = append(Tuple(nil), positional...) + } } if args != nil { // Add elements from *args sequence. @@ -479,7 +541,7 @@ loop: case compile.MAKEFUNC: funcode := f.Prog.Functions[arg] tuple := stack[sp-1].(Tuple) - n := len(tuple) - len(funcode.Freevars) + n := len(tuple) - len(funcode.FreeVars) defaults := tuple[:n:n] freevars := tuple[n:] stack[sp-1] = &Function{ @@ -527,11 +589,9 @@ loop: locals[arg] = stack[sp-1] sp-- - case compile.SETCELL: - x := stack[sp-2] - y := stack[sp-1] - sp -= 2 - y.(*cell).v = x + case compile.SETLOCALCELL: + locals[arg].(*cell).v = stack[sp-1] + sp-- case compile.SETGLOBAL: fn.module.globals[arg] = stack[sp-1] @@ -550,9 +610,23 @@ loop: stack[sp] = fn.freevars[arg] sp++ - case compile.CELL: - x := stack[sp-1] - stack[sp-1] = x.(*cell).v + case compile.LOCALCELL: + v := locals[arg].(*cell).v + if v == nil { + err = fmt.Errorf("local variable %s referenced before assignment", f.Locals[arg].Name) + break loop + } + stack[sp] = v + sp++ + + case compile.FREECELL: + v := fn.freevars[arg].(*cell).v + if v == nil { + err = fmt.Errorf("local variable %s referenced before assignment", f.FreeVars[arg].Name) + break loop + } + stack[sp] = v + sp++ case compile.GLOBAL: x := fn.module.globals[arg] @@ -582,14 +656,7 @@ loop: break loop } } - - // ITERPOP the rest of the iterator stack. - for _, iter := range iterstack { - iter.Done() - } - - fr.locals = nil - + // (deferred cleanup runs here) return result, err } @@ -621,7 +688,7 @@ func (mandatory) Hash() (uint32, error) { return 0, nil } // A cell is a box containing a Value. // Local variables marked as cells hold their value indirectly // so that they may be shared by outer and inner nested functions. -// Cells are always accessed using indirect CELL/SETCELL instructions. +// Cells are always accessed using indirect {FREE,LOCAL,SETLOCAL}CELL instructions. // The FreeVars tuple contains only cells. // The FREE instruction always yields a cell. type cell struct{ v Value } diff --git a/vendor/go.starlark.net/starlark/iter.go b/vendor/go.starlark.net/starlark/iter.go new file mode 100644 index 000000000..5436d9fef --- /dev/null +++ b/vendor/go.starlark.net/starlark/iter.go @@ -0,0 +1,118 @@ +// Copyright 2024 The Bazel Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build go1.23 + +package starlark + +import ( + "fmt" + "iter" +) + +func (d *Dict) Entries() iter.Seq2[Value, Value] { return d.ht.entries } + +// Elements returns a go1.23 iterator over the elements of the list. +// +// Example: +// +// for elem := range list.Elements() { ... } +func (l *List) Elements() iter.Seq[Value] { + return func(yield func(Value) bool) { + if !l.frozen { + l.itercount++ + defer func() { l.itercount-- }() + } + for _, x := range l.elems { + if !yield(x) { + break + } + } + } +} + +// Elements returns a go1.23 iterator over the elements of the tuple. +// +// (A Tuple is a slice, so it is of course directly iterable. This +// method exists to provide a fast path for the [Elements] standalone +// function.) +func (t Tuple) Elements() iter.Seq[Value] { + return func(yield func(Value) bool) { + for _, x := range t { + if !yield(x) { + break + } + } + } +} + +func (s *Set) Elements() iter.Seq[Value] { + return func(yield func(k Value) bool) { + s.ht.entries(func(k, _ Value) bool { return yield(k) }) + } +} + +// Elements returns an iterator for the elements of the iterable value. +// +// Example of go1.23 iteration: +// +// for elem := range Elements(iterable) { ... } +// +// Push iterators are provided as a convenience for Go client code. The +// core iteration behavior of Starlark for-loops is defined by the +// [Iterable] interface. +func Elements(iterable Iterable) iter.Seq[Value] { + // Use specialized push iterator if available (*List, Tuple, *Set). + type hasElements interface { + Elements() iter.Seq[Value] + } + if iterable, ok := iterable.(hasElements); ok { + return iterable.Elements() + } + + iter := iterable.Iterate() + return func(yield func(Value) bool) { + defer iter.Done() + var x Value + for iter.Next(&x) && yield(x) { + } + } +} + +// Entries returns an iterator over the entries (key/value pairs) of +// the iterable mapping. +// +// Example of go1.23 iteration: +// +// for k, v := range Entries(mapping) { ... } +// +// Push iterators are provided as a convenience for Go client code. The +// core iteration behavior of Starlark for-loops is defined by the +// [Iterable] interface. +func Entries(mapping IterableMapping) iter.Seq2[Value, Value] { + // If available (e.g. *Dict), use specialized push iterator, + // as it gets k and v in one shot. + type hasEntries interface { + Entries() iter.Seq2[Value, Value] + } + if mapping, ok := mapping.(hasEntries); ok { + return mapping.Entries() + } + + iter := mapping.Iterate() + return func(yield func(k, v Value) bool) { + defer iter.Done() + var k Value + for iter.Next(&k) { + v, found, err := mapping.Get(k) + if err != nil || !found { + panic(fmt.Sprintf("Iterate and Get are inconsistent (mapping=%v, key=%v)", + mapping.Type(), k.Type())) + } + if !yield(k, v) { + break + } + } + } +} diff --git a/vendor/go.starlark.net/starlark/library.go b/vendor/go.starlark.net/starlark/library.go index 7a9440ed1..78de54279 100644 --- a/vendor/go.starlark.net/starlark/library.go +++ b/vendor/go.starlark.net/starlark/library.go @@ -12,6 +12,7 @@ package starlark import ( "errors" "fmt" + "math" "math/big" "os" "sort" @@ -38,15 +39,17 @@ func init() { "None": None, "True": True, "False": False, - "any": NewBuiltin("any", any), + "abs": NewBuiltin("abs", abs), + "any": NewBuiltin("any", any_), "all": NewBuiltin("all", all), "bool": NewBuiltin("bool", bool_), + "bytes": NewBuiltin("bytes", bytes_), "chr": NewBuiltin("chr", chr), "dict": NewBuiltin("dict", dict), "dir": NewBuiltin("dir", dir), "enumerate": NewBuiltin("enumerate", enumerate), "fail": NewBuiltin("fail", fail), - "float": NewBuiltin("float", float), // requires resolve.AllowFloat + "float": NewBuiltin("float", float), "getattr": NewBuiltin("getattr", getattr), "hasattr": NewBuiltin("hasattr", hasattr), "hash": NewBuiltin("hash", hash), @@ -72,6 +75,10 @@ func init() { // methods of built-in types // https://github.com/google/starlark-go/blob/master/doc/spec.md#built-in-methods var ( + bytesMethods = map[string]*Builtin{ + "elems": NewBuiltin("elems", bytes_elems), + } + dictMethods = map[string]*Builtin{ "clear": NewBuiltin("clear", dict_clear), "get": NewBuiltin("get", dict_get), @@ -116,6 +123,8 @@ var ( "lower": NewBuiltin("lower", string_lower), "lstrip": NewBuiltin("lstrip", string_strip), // sic "partition": NewBuiltin("partition", string_partition), + "removeprefix": NewBuiltin("removeprefix", string_removefix), + "removesuffix": NewBuiltin("removesuffix", string_removefix), "replace": NewBuiltin("replace", string_replace), "rfind": NewBuiltin("rfind", string_rfind), "rindex": NewBuiltin("rindex", string_rindex), @@ -131,7 +140,18 @@ var ( } setMethods = map[string]*Builtin{ - "union": NewBuiltin("union", set_union), + "add": NewBuiltin("add", set_add), + "clear": NewBuiltin("clear", set_clear), + "difference": NewBuiltin("difference", set_difference), + "discard": NewBuiltin("discard", set_discard), + "intersection": NewBuiltin("intersection", set_intersection), + "issubset": NewBuiltin("issubset", set_issubset), + "issuperset": NewBuiltin("issuperset", set_issuperset), + "pop": NewBuiltin("pop", set_pop), + "remove": NewBuiltin("remove", set_remove), + "symmetric_difference": NewBuiltin("symmetric_difference", set_symmetric_difference), + "union": NewBuiltin("union", set_union), + "update": NewBuiltin("update", set_update), } ) @@ -154,6 +174,25 @@ func builtinAttrNames(methods map[string]*Builtin) []string { // ---- built-in functions ---- +// https://github.com/google/starlark-go/blob/master/doc/spec.md#abs +func abs(thread *Thread, _ *Builtin, args Tuple, kwargs []Tuple) (Value, error) { + var x Value + if err := UnpackPositionalArgs("abs", args, kwargs, 1, &x); err != nil { + return nil, err + } + switch x := x.(type) { + case Float: + return Float(math.Abs(float64(x))), nil + case Int: + if x.Sign() >= 0 { + return x, nil + } + return zero.Sub(x), nil + default: + return nil, fmt.Errorf("got %s, want int or float", x.Type()) + } +} + // https://github.com/google/starlark-go/blob/master/doc/spec.md#all func all(thread *Thread, _ *Builtin, args Tuple, kwargs []Tuple) (Value, error) { var iterable Iterable @@ -172,7 +211,7 @@ func all(thread *Thread, _ *Builtin, args Tuple, kwargs []Tuple) (Value, error) } // https://github.com/google/starlark-go/blob/master/doc/spec.md#any -func any(thread *Thread, _ *Builtin, args Tuple, kwargs []Tuple) (Value, error) { +func any_(thread *Thread, _ *Builtin, args Tuple, kwargs []Tuple) (Value, error) { var iterable Iterable if err := UnpackPositionalArgs("any", args, kwargs, 1, &iterable); err != nil { return nil, err @@ -197,6 +236,45 @@ func bool_(thread *Thread, _ *Builtin, args Tuple, kwargs []Tuple) (Value, error return x.Truth(), nil } +// https://github.com/google/starlark-go/blob/master/doc/spec.md#bytes +func bytes_(thread *Thread, _ *Builtin, args Tuple, kwargs []Tuple) (Value, error) { + if len(kwargs) > 0 { + return nil, fmt.Errorf("bytes does not accept keyword arguments") + } + if len(args) != 1 { + return nil, fmt.Errorf("bytes: got %d arguments, want exactly 1", len(args)) + } + switch x := args[0].(type) { + case Bytes: + return x, nil + case String: + // Invalid encodings are replaced by that of U+FFFD. + return Bytes(utf8Transcode(string(x))), nil + case Iterable: + // iterable of numeric byte values + var buf strings.Builder + if n := Len(x); n >= 0 { + // common case: known length + buf.Grow(n) + } + iter := x.Iterate() + defer iter.Done() + var elem Value + var b byte + for i := 0; iter.Next(&elem); i++ { + if err := AsInt(elem, &b); err != nil { + return nil, fmt.Errorf("bytes: at index %d, %s", i, err) + } + buf.WriteByte(b) + } + return Bytes(buf.String()), nil + + default: + // Unlike string(foo), which stringifies it, bytes(foo) is an error. + return nil, fmt.Errorf("bytes: got %s, want string, bytes, or iterable of ints", x.Type()) + } +} + // https://github.com/google/starlark-go/blob/master/doc/spec.md#chr func chr(thread *Thread, _ *Builtin, args Tuple, kwargs []Tuple) (Value, error) { if len(kwargs) > 0 { @@ -207,7 +285,7 @@ func chr(thread *Thread, _ *Builtin, args Tuple, kwargs []Tuple) (Value, error) } i, err := AsInt32(args[0]) if err != nil { - return nil, fmt.Errorf("chr: got %s, want int", args[0].Type()) + return nil, fmt.Errorf("chr: %s", err) } if i < 0 { return nil, fmt.Errorf("chr: Unicode code point %d out of range (<0)", i) @@ -215,7 +293,7 @@ func chr(thread *Thread, _ *Builtin, args Tuple, kwargs []Tuple) (Value, error) if i > unicode.MaxRune { return nil, fmt.Errorf("chr: Unicode code point U+%X out of range (>0x10FFFF)", i) } - return String(string(i)), nil + return String(string(rune(i))), nil } // https://github.com/google/starlark-go/blob/master/doc/spec.md#dict @@ -260,9 +338,6 @@ func enumerate(thread *Thread, _ *Builtin, args Tuple, kwargs []Tuple) (Value, e } iter := iterable.Iterate() - if iter == nil { - return nil, fmt.Errorf("enumerate: got %s, want iterable", iterable.Type()) - } defer iter.Done() var pairs []Value @@ -330,13 +405,39 @@ func float(thread *Thread, b *Builtin, args Tuple, kwargs []Tuple) (Value, error return Float(0.0), nil } case Int: - return x.Float(), nil + return x.finiteFloat() case Float: return x, nil case String: - f, err := strconv.ParseFloat(string(x), 64) + if x == "" { + return nil, fmt.Errorf("float: empty string") + } + // +/- NaN or Inf or Infinity (case insensitive)? + s := string(x) + switch x[len(x)-1] { + case 'y', 'Y': + if strings.EqualFold(s, "infinity") || strings.EqualFold(s, "+infinity") { + return inf, nil + } else if strings.EqualFold(s, "-infinity") { + return neginf, nil + } + case 'f', 'F': + if strings.EqualFold(s, "inf") || strings.EqualFold(s, "+inf") { + return inf, nil + } else if strings.EqualFold(s, "-inf") { + return neginf, nil + } + case 'n', 'N': + if strings.EqualFold(s, "nan") || strings.EqualFold(s, "+nan") || strings.EqualFold(s, "-nan") { + return nan, nil + } + } + f, err := strconv.ParseFloat(s, 64) + if math.IsInf(f, 0) { + return nil, fmt.Errorf("floating-point number too large") + } if err != nil { - return nil, nameErr(b, err) + return nil, fmt.Errorf("invalid float literal: %s", s) } return Float(f), nil default: @@ -344,6 +445,12 @@ func float(thread *Thread, b *Builtin, args Tuple, kwargs []Tuple) (Value, error } } +var ( + inf = Float(math.Inf(+1)) + neginf = Float(math.Inf(-1)) + nan = Float(math.NaN()) +) + // https://github.com/google/starlark-go/blob/master/doc/spec.md#getattr func getattr(thread *Thread, b *Builtin, args Tuple, kwargs []Tuple) (Value, error) { var object, dflt Value @@ -400,19 +507,27 @@ func hasattr(thread *Thread, _ *Builtin, args Tuple, kwargs []Tuple) (Value, err // https://github.com/google/starlark-go/blob/master/doc/spec.md#hash func hash(thread *Thread, _ *Builtin, args Tuple, kwargs []Tuple) (Value, error) { - var s string - if err := UnpackPositionalArgs("hash", args, kwargs, 1, &s); err != nil { + var x Value + if err := UnpackPositionalArgs("hash", args, kwargs, 1, &x); err != nil { return nil, err } - // The Starlark spec requires that the hash function be - // deterministic across all runs, motivated by the need - // for reproducibility of builds. Thus we cannot call - // String.Hash, which uses the fastest implementation - // available, because as varies across process restarts, - // and may evolve with the implementation. - - return MakeInt(int(javaStringHash(s))), nil + var h int64 + switch x := x.(type) { + case String: + // The Starlark spec requires that the hash function be + // deterministic across all runs, motivated by the need + // for reproducibility of builds. Thus we cannot call + // String.Hash, which uses the fastest implementation + // available, because as varies across process restarts, + // and may evolve with the implementation. + h = int64(javaStringHash(string(x))) + case Bytes: + h = int64(softHashString(string(x))) // FNV32 + default: + return nil, fmt.Errorf("hash: got %s, want string or bytes", x.Type()) + } + return MakeInt64(h), nil } // javaStringHash returns the same hash as would be produced by @@ -440,114 +555,112 @@ func int_(thread *Thread, _ *Builtin, args Tuple, kwargs []Tuple) (Value, error) return nil, err } - // "If x is not a number or base is given, x must be a string." if s, ok := AsString(x); ok { b := 10 if base != nil { var err error b, err = AsInt32(base) - if err != nil || b != 0 && (b < 2 || b > 36) { + if err != nil { + return nil, fmt.Errorf("int: for base, got %s, want int", base.Type()) + } + if b != 0 && (b < 2 || b > 36) { return nil, fmt.Errorf("int: base must be an integer >= 2 && <= 36") } } + res := parseInt(s, b) + if res == nil { + return nil, fmt.Errorf("int: invalid literal with base %d: %s", b, s) + } + return res, nil + } - orig := s // save original for error message + if base != nil { + return nil, fmt.Errorf("int: can't convert non-string with explicit base") + } - // remove sign - var neg bool - if s != "" { - if s[0] == '+' { - s = s[1:] - } else if s[0] == '-' { - neg = true - s = s[1:] - } + if b, ok := x.(Bool); ok { + if b { + return one, nil + } else { + return zero, nil } + } - // remove base prefix - baseprefix := 0 - if len(s) > 1 && s[0] == '0' { - if len(s) > 2 { - switch s[1] { - case 'o', 'O': - s = s[2:] - baseprefix = 8 - case 'x', 'X': - s = s[2:] - baseprefix = 16 - case 'b', 'B': - s = s[2:] - baseprefix = 2 - } - } + i, err := NumberToInt(x) + if err != nil { + return nil, fmt.Errorf("int: %s", err) + } + return i, nil +} +// parseInt defines the behavior of int(string, base=int). It returns nil on error. +func parseInt(s string, base int) Value { + // remove sign + var neg bool + if s != "" { + if s[0] == '+' { + s = s[1:] + } else if s[0] == '-' { + neg = true + s = s[1:] + } + } + + // remove optional base prefix + baseprefix := 0 + if len(s) > 1 && s[0] == '0' { + if len(s) > 2 { + switch s[1] { + case 'o', 'O': + baseprefix = 8 + case 'x', 'X': + baseprefix = 16 + case 'b', 'B': + baseprefix = 2 + } + } + if baseprefix != 0 { + // Remove the base prefix if it matches + // the explicit base, or if base=0. + if base == 0 || baseprefix == base { + base = baseprefix + s = s[2:] + } + } else { // For automatic base detection, // a string starting with zero // must be all zeros. // Thus we reject int("0755", 0). - if baseprefix == 0 && b == 0 { + if base == 0 { for i := 1; i < len(s); i++ { if s[i] != '0' { - goto invalid + return nil } } - return zero, nil - } - - if b != 0 && baseprefix != 0 && baseprefix != b { - // Explicit base doesn't match prefix, - // e.g. int("0o755", 16). - goto invalid - } - } - - // select base - if b == 0 { - if baseprefix != 0 { - b = baseprefix - } else { - b = 10 + return zero } } - - // we explicitly handled sign above. - // if a sign remains, it is invalid. - if s != "" && (s[0] == '-' || s[0] == '+') { - goto invalid - } - - // s has no sign or base prefix. - // - // int(x) permits arbitrary precision, unlike the scanner. - if i, ok := new(big.Int).SetString(s, b); ok { - res := MakeBigInt(i) - if neg { - res = zero.Sub(res) - } - return res, nil - } - - invalid: - return nil, fmt.Errorf("int: invalid literal with base %d: %s", b, orig) + } + if base == 0 { + base = 10 } - if base != nil { - return nil, fmt.Errorf("int: can't convert non-string with explicit base") + // we explicitly handled sign above. + // if a sign remains, it is invalid. + if s != "" && (s[0] == '-' || s[0] == '+') { + return nil } - if b, ok := x.(Bool); ok { - if b { - return one, nil - } else { - return zero, nil + // s has no sign or base prefix. + if i, ok := new(big.Int).SetString(s, base); ok { + res := MakeBigInt(i) + if neg { + res = zero.Sub(res) } + return res } - i, err := NumberToInt(x) - if err != nil { - return nil, fmt.Errorf("int: %s", err) - } - return i, nil + return nil } // https://github.com/google/starlark-go/blob/master/doc/spec.md#len @@ -660,16 +773,26 @@ func ord(thread *Thread, _ *Builtin, args Tuple, kwargs []Tuple) (Value, error) if len(args) != 1 { return nil, fmt.Errorf("ord: got %d arguments, want 1", len(args)) } - s, ok := AsString(args[0]) - if !ok { - return nil, fmt.Errorf("ord: got %s, want string", args[0].Type()) - } - r, sz := utf8.DecodeRuneInString(s) - if sz == 0 || sz != len(s) { - n := utf8.RuneCountInString(s) - return nil, fmt.Errorf("ord: string encodes %d Unicode code points, want 1", n) + switch x := args[0].(type) { + case String: + // ord(string) returns int value of sole rune. + s := string(x) + r, sz := utf8.DecodeRuneInString(s) + if sz == 0 || sz != len(s) { + n := utf8.RuneCountInString(s) + return nil, fmt.Errorf("ord: string encodes %d Unicode code points, want 1", n) + } + return MakeInt(int(r)), nil + + case Bytes: + // ord(bytes) returns int value of sole byte. + if len(x) != 1 { + return nil, fmt.Errorf("ord: bytes has length %d, want 1", len(x)) + } + return MakeInt(int(x[0])), nil + default: + return nil, fmt.Errorf("ord: got %s, want string or bytes", x.Type()) } - return MakeInt(int(r)), nil } // https://github.com/google/starlark-go/blob/master/doc/spec.md#print @@ -685,6 +808,8 @@ func print(thread *Thread, b *Builtin, args Tuple, kwargs []Tuple) (Value, error } if s, ok := AsString(v); ok { buf.WriteString(s) + } else if b, ok := v.(Bytes); ok { + buf.WriteString(string(b)) } else { writeValue(buf, v, nil) } @@ -707,7 +832,6 @@ func range_(thread *Thread, b *Builtin, args Tuple, kwargs []Tuple) (Value, erro return nil, err } - // TODO(adonovan): analyze overflow/underflows cases for 32-bit implementations. if len(args) == 1 { // range(stop) start, stop = 0, start @@ -730,6 +854,7 @@ var ( _ Sequence = rangeValue{} _ Comparable = rangeValue{} _ Sliceable = rangeValue{} + _ Container = rangeValue{} ) func (r rangeValue) Len() int { return r.len } @@ -792,6 +917,14 @@ func (x rangeValue) CompareSameType(op syntax.Token, y_ Value, depth int) (bool, } } +func (r rangeValue) Has(y Value) (bool, error) { + i, err := NumberToInt(y) + if err != nil { + return false, fmt.Errorf("'in ' requires integer as left operand, not %s", y.Type()) + } + return r.contains(i), nil +} + func rangeEqual(x, y rangeValue) bool { // Two ranges compare equal if they denote the same sequence. if x.len != y.len { @@ -963,11 +1096,29 @@ func str(thread *Thread, _ *Builtin, args Tuple, kwargs []Tuple) (Value, error) if len(args) != 1 { return nil, fmt.Errorf("str: got %d arguments, want exactly 1", len(args)) } - x := args[0] - if _, ok := AsString(x); !ok { - x = String(x.String()) + switch x := args[0].(type) { + case String: + return x, nil + case Bytes: + // Invalid encodings are replaced by that of U+FFFD. + return String(utf8Transcode(string(x))), nil + default: + return String(x.String()), nil } - return x, nil +} + +// utf8Transcode returns the UTF-8-to-UTF-8 transcoding of s. +// The effect is that each code unit that is part of an +// invalid sequence is replaced by U+FFFD. +func utf8Transcode(s string) string { + if utf8.ValidString(s) { + return s + } + var out strings.Builder + for _, r := range s { + out.WriteRune(r) + } + return out.String() } // https://github.com/google/starlark-go/blob/master/doc/spec.md#tuple @@ -1344,13 +1495,51 @@ func string_iterable(_ *Thread, b *Builtin, args Tuple, kwargs []Tuple) (Value, if err := UnpackPositionalArgs(b.Name(), args, kwargs, 0); err != nil { return nil, err } - return stringIterable{ - s: b.Receiver().(String), - ords: b.Name()[len(b.Name())-2] == 'd', - codepoints: b.Name()[0] == 'c', - }, nil + s := b.Receiver().(String) + ords := b.Name()[len(b.Name())-2] == 'd' + codepoints := b.Name()[0] == 'c' + if codepoints { + return stringCodepoints{s, ords}, nil + } else { + return stringElems{s, ords}, nil + } +} + +// bytes_elems returns an unspecified iterable value whose +// iterator yields the int values of successive elements. +func bytes_elems(_ *Thread, b *Builtin, args Tuple, kwargs []Tuple) (Value, error) { + if err := UnpackPositionalArgs(b.Name(), args, kwargs, 0); err != nil { + return nil, err + } + return bytesIterable{b.Receiver().(Bytes)}, nil +} + +// A bytesIterable is an iterable returned by bytes.elems(), +// whose iterator yields a sequence of numeric bytes values. +type bytesIterable struct{ bytes Bytes } + +var _ Iterable = (*bytesIterable)(nil) + +func (bi bytesIterable) String() string { return bi.bytes.String() + ".elems()" } +func (bi bytesIterable) Type() string { return "bytes.elems" } +func (bi bytesIterable) Freeze() {} // immutable +func (bi bytesIterable) Truth() Bool { return True } +func (bi bytesIterable) Hash() (uint32, error) { return 0, fmt.Errorf("unhashable: %s", bi.Type()) } +func (bi bytesIterable) Iterate() Iterator { return &bytesIterator{bi.bytes} } + +type bytesIterator struct{ bytes Bytes } + +func (it *bytesIterator) Next(p *Value) bool { + if it.bytes == "" { + return false + } + *p = MakeInt(int(it.bytes[0])) + it.bytes = it.bytes[1:] + return true } +func (*bytesIterator) Done() {} + // https://github.com/google/starlark-go/blob/master/doc/spec.md#string·count func string_count(_ *Thread, b *Builtin, args Tuple, kwargs []Tuple) (Value, error) { var sub string @@ -1722,6 +1911,22 @@ func string_partition(_ *Thread, b *Builtin, args Tuple, kwargs []Tuple) (Value, return tuple, nil } +// https://github.com/google/starlark-go/blob/master/doc/spec.md#string·removeprefix +// https://github.com/google/starlark-go/blob/master/doc/spec.md#string·removesuffix +func string_removefix(_ *Thread, b *Builtin, args Tuple, kwargs []Tuple) (Value, error) { + recv := string(b.Receiver().(String)) + var fix string + if err := UnpackPositionalArgs(b.Name(), args, kwargs, 1, &fix); err != nil { + return nil, err + } + if b.name[len("remove")] == 'p' { + recv = strings.TrimPrefix(recv, fix) + } else { + recv = strings.TrimSuffix(recv, fix) + } + return String(recv), nil +} + // https://github.com/google/starlark-go/blob/master/doc/spec.md#string·replace func string_replace(_ *Thread, b *Builtin, args Tuple, kwargs []Tuple) (Value, error) { recv := string(b.Receiver().(String)) @@ -1983,19 +2188,189 @@ func string_splitlines(_ *Thread, b *Builtin, args Tuple, kwargs []Tuple) (Value return NewList(list), nil } -// https://github.com/google/starlark-go/blob/master/doc/spec.md#set·union. -func set_union(_ *Thread, b *Builtin, args Tuple, kwargs []Tuple) (Value, error) { - var iterable Iterable - if err := UnpackPositionalArgs(b.Name(), args, kwargs, 0, &iterable); err != nil { +// https://github.com/google/starlark-go/blob/master/doc/spec.md#set·add. +func set_add(_ *Thread, b *Builtin, args Tuple, kwargs []Tuple) (Value, error) { + var elem Value + if err := UnpackPositionalArgs(b.Name(), args, kwargs, 1, &elem); err != nil { return nil, err } - iter := iterable.Iterate() + recv := b.Receiver().(*Set) + // It is always an error to attempt to mutate a set that cannot be mutated + if err := recv.ht.checkMutable("insert into"); err != nil { + return nil, nameErr(b, err) + } + // TODO(adonovan): opt: combine Has+Insert. (e.g. use Insert and re-check Len) + if found, err := recv.Has(elem); err != nil { + return nil, nameErr(b, err) + } else if found { + return None, nil + } + err := recv.Insert(elem) + if err != nil { + return nil, nameErr(b, err) + } + return None, nil +} + +// https://github.com/google/starlark-go/blob/master/doc/spec.md#set·clear. +func set_clear(_ *Thread, b *Builtin, args Tuple, kwargs []Tuple) (Value, error) { + if err := UnpackPositionalArgs(b.Name(), args, kwargs, 0); err != nil { + return nil, err + } + if b.Receiver().(*Set).Len() > 0 { + if err := b.Receiver().(*Set).Clear(); err != nil { + return nil, nameErr(b, err) + } + } + return None, nil +} + +// https://github.com/google/starlark-go/blob/master/doc/spec.md#set·difference. +func set_difference(_ *Thread, b *Builtin, args Tuple, kwargs []Tuple) (Value, error) { + // TODO: support multiple others: s.difference(*others) + var other Iterable + if err := UnpackPositionalArgs(b.Name(), args, kwargs, 0, &other); err != nil { + return nil, err + } + iter := other.Iterate() defer iter.Done() - union, err := b.Receiver().(*Set).Union(iter) + diff, err := b.Receiver().(*Set).Difference(iter) if err != nil { return nil, nameErr(b, err) } - return union, nil + return diff, nil +} + +// https://github.com/google/starlark-go/blob/master/doc/spec.md#set_intersection. +func set_intersection(_ *Thread, b *Builtin, args Tuple, kwargs []Tuple) (Value, error) { + // TODO: support multiple others: s.difference(*others) + var other Iterable + if err := UnpackPositionalArgs(b.Name(), args, kwargs, 0, &other); err != nil { + return nil, err + } + iter := other.Iterate() + defer iter.Done() + diff, err := b.Receiver().(*Set).Intersection(iter) + if err != nil { + return nil, nameErr(b, err) + } + return diff, nil +} + +// https://github.com/google/starlark-go/blob/master/doc/spec.md#set_issubset. +func set_issubset(_ *Thread, b *Builtin, args Tuple, kwargs []Tuple) (Value, error) { + var other Iterable + if err := UnpackPositionalArgs(b.Name(), args, kwargs, 0, &other); err != nil { + return nil, err + } + iter := other.Iterate() + defer iter.Done() + diff, err := b.Receiver().(*Set).IsSubset(iter) + if err != nil { + return nil, nameErr(b, err) + } + return Bool(diff), nil +} + +// https://github.com/google/starlark-go/blob/master/doc/spec.md#set_issuperset. +func set_issuperset(_ *Thread, b *Builtin, args Tuple, kwargs []Tuple) (Value, error) { + var other Iterable + if err := UnpackPositionalArgs(b.Name(), args, kwargs, 0, &other); err != nil { + return nil, err + } + iter := other.Iterate() + defer iter.Done() + diff, err := b.Receiver().(*Set).IsSuperset(iter) + if err != nil { + return nil, nameErr(b, err) + } + return Bool(diff), nil +} + +// https://github.com/google/starlark-go/blob/master/doc/spec.md#set·discard. +func set_discard(_ *Thread, b *Builtin, args Tuple, kwargs []Tuple) (Value, error) { + var k Value + if err := UnpackPositionalArgs(b.Name(), args, kwargs, 1, &k); err != nil { + return nil, err + } + recv := b.Receiver().(*Set) + // It is always an error to attempt to mutate a set that cannot be mutated + if err := recv.ht.checkMutable("delete from"); err != nil { + return nil, nameErr(b, err) + } + // TODO(adonovan): opt: combine Has+Delete (e.g. use Delete and re-check Len) + if found, err := recv.Has(k); err != nil { + return nil, nameErr(b, err) + } else if !found { + return None, nil + } + if _, err := recv.Delete(k); err != nil { + return nil, nameErr(b, err) // set is frozen + } + return None, nil +} + +// https://github.com/google/starlark-go/blob/master/doc/spec.md#set·pop. +func set_pop(_ *Thread, b *Builtin, args Tuple, kwargs []Tuple) (Value, error) { + if err := UnpackPositionalArgs(b.Name(), args, kwargs, 0); err != nil { + return nil, err + } + recv := b.Receiver().(*Set) + k, ok := recv.ht.first() + if !ok { + return nil, nameErr(b, "empty set") + } + _, err := recv.Delete(k) + if err != nil { + return nil, nameErr(b, err) // set is frozen + } + return k, nil +} + +// https://github.com/google/starlark-go/blob/master/doc/spec.md#set·remove. +func set_remove(_ *Thread, b *Builtin, args Tuple, kwargs []Tuple) (Value, error) { + var k Value + if err := UnpackPositionalArgs(b.Name(), args, kwargs, 1, &k); err != nil { + return nil, err + } + if found, err := b.Receiver().(*Set).Delete(k); err != nil { + return nil, nameErr(b, err) // dict is frozen or key is unhashable + } else if found { + return None, nil + } + return nil, nameErr(b, "missing key") +} + +// https://github.com/google/starlark-go/blob/master/doc/spec.md#set·symmetric_difference. +func set_symmetric_difference(_ *Thread, b *Builtin, args Tuple, kwargs []Tuple) (Value, error) { + var other Iterable + if err := UnpackPositionalArgs(b.Name(), args, kwargs, 0, &other); err != nil { + return nil, err + } + iter := other.Iterate() + defer iter.Done() + diff, err := b.Receiver().(*Set).SymmetricDifference(iter) + if err != nil { + return nil, nameErr(b, err) + } + return diff, nil +} + +// https://github.com/google/starlark-go/blob/master/doc/spec.md#set·union. +func set_union(_ *Thread, b *Builtin, args Tuple, kwargs []Tuple) (Value, error) { + receiverSet := b.Receiver().(*Set).clone() + if err := setUpdate(receiverSet, args, kwargs); err != nil { + return nil, nameErr(b, err) + } + return receiverSet, nil +} + +// https://github.com/google/starlark-go/blob/master/doc/spec.md#set·update. +func set_update(_ *Thread, b *Builtin, args Tuple, kwargs []Tuple) (Value, error) { + if err := setUpdate(b.Receiver().(*Set), args, kwargs); err != nil { + return nil, nameErr(b, err) + } + return None, nil } // Common implementation of string_{r}{find,index}. @@ -2097,6 +2472,28 @@ func updateDict(dict *Dict, updates Tuple, kwargs []Tuple) error { return nil } +func setUpdate(s *Set, args Tuple, kwargs []Tuple) error { + if len(kwargs) > 0 { + return errors.New("does not accept keyword arguments") + } + + for i, arg := range args { + iterable, ok := arg.(Iterable) + if !ok { + return fmt.Errorf("argument #%d is not iterable: %s", i+1, arg.Type()) + } + if err := func() error { + iter := iterable.Iterate() + defer iter.Done() + return s.InsertAll(iter) + }(); err != nil { + return err + } + } + + return nil +} + // nameErr returns an error message of the form "name: msg" // where name is b.Name() and msg is a string or error. func nameErr(b *Builtin, msg interface{}) error { diff --git a/vendor/go.starlark.net/starlark/profile.go b/vendor/go.starlark.net/starlark/profile.go index 38da2b2e9..3af6a9db9 100644 --- a/vendor/go.starlark.net/starlark/profile.go +++ b/vendor/go.starlark.net/starlark/profile.go @@ -22,7 +22,7 @@ package starlark // stack and sends it to the profiler goroutine, along with the number // of quanta, which are subtracted. For example, if the accumulator // holds 3ms and then a completed span adds 25ms to it, its value is 28ms, -// which exceeeds 10ms. The profiler records a stack with the value 20ms +// which exceeds 10ms. The profiler records a stack with the value 20ms // (2 quanta), and the accumulator is left with 8ms. // // The profiler goroutine converts the stacks into the pprof format and @@ -101,11 +101,11 @@ func StartProfile(w io.Writer) error { return nil } -// StopProfiler stops the profiler started by a prior call to +// StopProfile stops the profiler started by a prior call to // StartProfile and finalizes the profile. It returns an error if the // profile could not be completed. // -// StopProfiler must not be called concurrently with Starlark execution. +// StopProfile must not be called concurrently with Starlark execution. func StopProfile() error { // Terminate the profiler goroutine and get its result. close(profiler.events) @@ -357,28 +357,37 @@ func profile(w io.Writer) { profiler.done <- err } -// nanotime returns the time in nanoseconds since epoch. -// It is implemented by runtime.nanotime using the linkname hack; -// runtime.nanotime is defined for all OSs/ARCHS and uses the -// monotonic system clock, which there is no portable way to access. -// Should that function ever go away, these alternatives exist: +// nanotime returns the time in nanoseconds since process start. // -// // POSIX only. REALTIME not MONOTONIC. 17ns. -// var tv syscall.Timeval -// syscall.Gettimeofday(&tv) // can't fail -// return tv.Nano() +// This approach, described at +// https://github.com/golang/go/issues/61765#issuecomment-1672090302, +// is fast, monotonic, and portable, and avoids the previous +// dependence on runtime.nanotime using the (unsafe) linkname hack. +// In particular, time.Since does less work than time.Now. // -// // Portable. REALTIME not MONOTONIC. 46ns. -// return time.Now().Nanoseconds() +// Rejected approaches: // -// // POSIX only. Adds a dependency. +// Using the linkname hack to unsafely access runtime.nanotime. +// See #546 and golang/go#67401. +// +// // POSIX only. REALTIME not MONOTONIC. 17ns. +// var tv syscall.Timeval +// syscall.Gettimeofday(&tv) // can't fail +// return tv.Nano() +// +// // Portable. REALTIME not MONOTONIC. 46ns. +// return time.Now().Nanoseconds() +// +// // POSIX only. Adds a dependency. // import "golang.org/x/sys/unix" // var ts unix.Timespec -// unix.ClockGettime(CLOCK_MONOTONIC, &ts) // can't fail +// unix.ClockGettime(CLOCK_MONOTONIC, &ts) // can't fail // return unix.TimespecToNsec(ts) -// -//go:linkname nanotime runtime.nanotime -func nanotime() int64 +func nanotime() int64 { + return time.Since(processStart).Nanoseconds() +} + +var processStart = time.Now() // profFuncAddr returns the canonical "address" // of a Callable for use by the profiler. @@ -391,7 +400,7 @@ func profFuncAddr(fn Callable) uintptr { } // User-defined callable types are typically of - // of kind pointer-to-struct. Handle them specially. + // kind pointer-to-struct. Handle them specially. if v := reflect.ValueOf(fn); v.Type().Kind() == reflect.Ptr { return v.Pointer() } diff --git a/vendor/go.starlark.net/starlark/unpack.go b/vendor/go.starlark.net/starlark/unpack.go index 6c870f951..e47134e2d 100644 --- a/vendor/go.starlark.net/starlark/unpack.go +++ b/vendor/go.starlark.net/starlark/unpack.go @@ -8,37 +8,102 @@ import ( "log" "reflect" "strings" + + "go.starlark.net/internal/spell" ) +// An Unpacker defines custom argument unpacking behavior. +// See UnpackArgs. +type Unpacker interface { + Unpack(v Value) error +} + // UnpackArgs unpacks the positional and keyword arguments into the // supplied parameter variables. pairs is an alternating list of names // and pointers to variables. // -// If the variable is a bool, int, string, *List, *Dict, Callable, +// If the variable is a bool, integer, string, *List, *Dict, Callable, // Iterable, or user-defined implementation of Value, // UnpackArgs performs the appropriate type check. -// An int uses the AsInt32 check. -// If the parameter name ends with "?", -// it and all following parameters are optional. +// Predeclared Go integer types use the AsInt check. +// +// If the parameter name ends with "?", it is optional. +// +// If the parameter name ends with "??", it is optional and treats the None value +// as if the argument was absent. +// +// If a parameter is marked optional, then all following parameters are +// implicitly optional whether or not they are marked. +// +// If the variable implements Unpacker, its Unpack argument +// is called with the argument value, allowing an application +// to define its own argument validation and conversion. // // If the variable implements Value, UnpackArgs may call // its Type() method while constructing the error message. // -// Beware: an optional *List, *Dict, Callable, Iterable, or Value variable that is -// not assigned is not a valid Starlark Value, so the caller must -// explicitly handle such cases by interpreting nil as None or some -// computed default. -func UnpackArgs(fnname string, args Tuple, kwargs []Tuple, pairs ...interface{}) error { +// Examples: +// +// var ( +// a Value +// b = MakeInt(42) +// c Value = starlark.None +// ) +// +// // 1. mixed parameters, like def f(a, b=42, c=None). +// err := UnpackArgs("f", args, kwargs, "a", &a, "b?", &b, "c?", &c) +// +// // 2. keyword parameters only, like def f(*, a, b, c=None). +// if len(args) > 0 { +// return fmt.Errorf("f: unexpected positional arguments") +// } +// err := UnpackArgs("f", args, kwargs, "a", &a, "b?", &b, "c?", &c) +// +// // 3. positional parameters only, like def f(a, b=42, c=None, /) in Python 3.8. +// err := UnpackPositionalArgs("f", args, kwargs, 1, &a, &b, &c) +// +// More complex forms such as def f(a, b=42, *args, c, d=123, **kwargs) +// require additional logic, but their need in built-ins is exceedingly rare. +// +// In the examples above, the declaration of b with type Int causes UnpackArgs +// to require that b's argument value, if provided, is also an int. +// To allow arguments of any type, while retaining the default value of 42, +// declare b as a Value: +// +// var b Value = MakeInt(42) +// +// The zero value of a variable of type Value, such as 'a' in the +// examples above, is not a valid Starlark value, so if the parameter is +// optional, the caller must explicitly handle the default case by +// interpreting nil as None or some computed default. The same is true +// for the zero values of variables of type *List, *Dict, Callable, or +// Iterable. For example: +// +// // def myfunc(d=None, e=[], f={}) +// var ( +// d Value +// e *List +// f *Dict +// ) +// err := UnpackArgs("myfunc", args, kwargs, "d?", &d, "e?", &e, "f?", &f) +// if d == nil { d = None; } +// if e == nil { e = new(List); } +// if f == nil { f = new(Dict); } +func UnpackArgs(fnname string, args Tuple, kwargs []Tuple, pairs ...any) error { nparams := len(pairs) / 2 var defined intset defined.init(nparams) - paramName := func(x interface{}) string { // (no free variables) - name := x.(string) - if name[len(name)-1] == '?' { + paramName := func(x any) (name string, skipNone bool) { // (no free variables) + name = x.(string) + if strings.HasSuffix(name, "??") { + name = strings.TrimSuffix(name, "??") + skipNone = true + } else if name[len(name)-1] == '?' { name = name[:len(name)-1] } - return name + + return name, skipNone } // positional arguments @@ -48,8 +113,13 @@ func UnpackArgs(fnname string, args Tuple, kwargs []Tuple, pairs ...interface{}) } for i, arg := range args { defined.set(i) - if err := unpackOneArg(arg, pairs[2*i+1]); err != nil { - name := paramName(pairs[2*i]) + name, skipNone := paramName(pairs[2*i]) + if skipNone { + if _, isNone := arg.(NoneType); isNone { + continue + } + } + if err := UnpackArg(arg, pairs[2*i+1]); err != nil { return fmt.Errorf("%s: for parameter %s: %s", fnname, name, err) } } @@ -59,29 +129,49 @@ kwloop: for _, item := range kwargs { name, arg := item[0].(String), item[1] for i := 0; i < nparams; i++ { - if paramName(pairs[2*i]) == string(name) { + pName, skipNone := paramName(pairs[2*i]) + if pName == string(name) { // found it if defined.set(i) { return fmt.Errorf("%s: got multiple values for keyword argument %s", fnname, name) } + + if skipNone { + if _, isNone := arg.(NoneType); isNone { + continue kwloop + } + } + ptr := pairs[2*i+1] - if err := unpackOneArg(arg, ptr); err != nil { + if err := UnpackArg(arg, ptr); err != nil { return fmt.Errorf("%s: for parameter %s: %s", fnname, name, err) } continue kwloop } } - return fmt.Errorf("%s: unexpected keyword argument %s", fnname, name) + err := fmt.Errorf("%s: unexpected keyword argument %s", fnname, name) + names := make([]string, 0, nparams) + for i := 0; i < nparams; i += 2 { + param, _ := paramName(pairs[i]) + names = append(names, param) + } + if n := spell.Nearest(string(name), names); n != "" { + err = fmt.Errorf("%s (did you mean %s?)", err.Error(), n) + } + return err } // Check that all non-optional parameters are defined. - // (We needn't check the first len(args).) - for i := len(args); i < nparams; i++ { + for i := 0; i < nparams; i++ { name := pairs[2*i].(string) if strings.HasSuffix(name, "?") { break // optional } + // (We needn't check the first len(args).) + if i < len(args) { + continue + } if !defined.get(i) { return fmt.Errorf("%s: missing argument for %s", fnname, name) } @@ -97,7 +187,9 @@ kwloop: // UnpackPositionalArgs reports an error if the number of arguments is // less than min or greater than len(vars), if kwargs is nonempty, or if // any conversion fails. -func UnpackPositionalArgs(fnname string, args Tuple, kwargs []Tuple, min int, vars ...interface{}) error { +// +// See UnpackArgs for general comments. +func UnpackPositionalArgs(fnname string, args Tuple, kwargs []Tuple, min int, vars ...any) error { if len(kwargs) > 0 { return fmt.Errorf("%s: unexpected keyword arguments", fnname) } @@ -117,16 +209,25 @@ func UnpackPositionalArgs(fnname string, args Tuple, kwargs []Tuple, min int, va return fmt.Errorf("%s: got %d arguments, want %s%d", fnname, len(args), atmost, max) } for i, arg := range args { - if err := unpackOneArg(arg, vars[i]); err != nil { + if err := UnpackArg(arg, vars[i]); err != nil { return fmt.Errorf("%s: for parameter %d: %s", fnname, i+1, err) } } return nil } -func unpackOneArg(v Value, ptr interface{}) error { +// UnpackArg unpacks a Value v into the variable pointed to by ptr. +// See [UnpackArgs] for details, including which types of variable are supported. +// +// This function defines the unpack operation for a single value. +// The implementations of most built-in functions use [UnpackArgs] and +// [UnpackPositionalArgs] to unpack a sequence of positional and/or +// keyword arguments. +func UnpackArg(v Value, ptr any) error { // On failure, don't clobber *ptr. switch ptr := ptr.(type) { + case Unpacker: + return ptr.Unpack(v) case *Value: *ptr = v case *string: @@ -141,12 +242,15 @@ func unpackOneArg(v Value, ptr interface{}) error { return fmt.Errorf("got %s, want bool", v.Type()) } *ptr = bool(b) - case *int: - i, err := AsInt32(v) - if err != nil { - return err + case *int, *int8, *int16, *int32, *int64, + *uint, *uint8, *uint16, *uint32, *uint64, *uintptr: + return AsInt(v, ptr) + case *float64: + f, ok := v.(Float) + if !ok { + return fmt.Errorf("got %s, want float", v.Type()) } - *ptr = i + *ptr = float64(f) case **List: list, ok := v.(*List) if !ok { @@ -205,7 +309,9 @@ func unpackOneArg(v Value, ptr interface{}) error { // Attempt to call Value.Type method. func() { defer func() { recover() }() - paramType = paramVar.MethodByName("Type").Call(nil)[0].String() + if typer, _ := paramVar.Interface().(interface{ Type() string }); typer != nil { + paramType = typer.Type() + } }() return fmt.Errorf("got %s, want %s", v.Type(), paramType) } diff --git a/vendor/go.starlark.net/starlark/value.go b/vendor/go.starlark.net/starlark/value.go index 267465f8f..52bfa76d4 100644 --- a/vendor/go.starlark.net/starlark/value.go +++ b/vendor/go.starlark.net/starlark/value.go @@ -7,34 +7,36 @@ // Starlark values are represented by the Value interface. // The following built-in Value types are known to the evaluator: // -// NoneType -- NoneType -// Bool -- bool -// Int -- int -// Float -- float -// String -- string -// *List -- list -// Tuple -- tuple -// *Dict -- dict -// *Set -- set -// *Function -- function (implemented in Starlark) -// *Builtin -- builtin_function_or_method (function or method implemented in Go) +// NoneType -- NoneType +// Bool -- bool +// Bytes -- bytes +// Int -- int +// Float -- float +// String -- string +// *List -- list +// Tuple -- tuple +// *Dict -- dict +// *Set -- set +// *Function -- function (implemented in Starlark) +// *Builtin -- builtin_function_or_method (function or method implemented in Go) // // Client applications may define new data types that satisfy at least // the Value interface. Such types may provide additional operations by // implementing any of these optional interfaces: // -// Callable -- value is callable like a function -// Comparable -- value defines its own comparison operations -// Iterable -- value is iterable using 'for' loops -// Sequence -- value is iterable sequence of known length -// Indexable -- value is sequence with efficient random access -// Mapping -- value maps from keys to values, like a dictionary -// HasBinary -- value defines binary operations such as * and + -// HasAttrs -- value has readable fields or methods x.f -// HasSetField -- value has settable fields x.f -// HasSetIndex -- value supports element update using x[i]=y -// HasSetKey -- value supports map update using x[k]=v -// HasUnary -- value defines unary operations such as + and - +// Callable -- value is callable like a function +// Comparable -- value defines its own comparison operations +// Container -- value supports the 'in' operator +// Iterable -- value is iterable using 'for' loops +// Sequence -- value is iterable sequence of known length +// Indexable -- value is sequence with efficient random access +// Mapping -- value maps from keys to values, like a dictionary +// HasBinary -- value defines binary operations such as * and + +// HasAttrs -- value has readable fields or methods x.f +// HasSetField -- value has settable fields x.f +// HasSetIndex -- value supports element update using x[i]=y +// HasSetKey -- value supports map update using x[k]=v +// HasUnary -- value defines unary operations such as + and - // // Client applications may also define domain-specific functions in Go // and make them available to Starlark programs. Use NewBuiltin to @@ -62,7 +64,6 @@ // through Starlark code and into callbacks. When evaluation fails it // returns an EvalError from which the application may obtain a // backtrace of active Starlark calls. -// package starlark // import "go.starlark.net/starlark" // This file defines the data types of Starlark and their basic operations. @@ -95,6 +96,11 @@ type Value interface { // structure through this API will fail dynamically, making the // data structure immutable and safe for publishing to other // Starlark interpreters running concurrently. + // + // Implementations of Freeze must be defensive against + // reference cycles; this can be achieved by first checking + // the value's frozen state, then setting it, and only then + // visiting any other values that it references. Freeze() // Truth returns the truth value of an object. @@ -131,16 +137,41 @@ type Comparable interface { CompareSameType(op syntax.Token, y Value, depth int) (bool, error) } +// A TotallyOrdered is a type whose values form a total order: +// if x and y are of the same TotallyOrdered type, then x must be less than y, +// greater than y, or equal to y. +// +// It is simpler than Comparable and should be preferred in new code, +// but if a type implements both interfaces, Comparable takes precedence. +type TotallyOrdered interface { + Value + // Cmp compares two values x and y of the same totally ordered type. + // It returns negative if x < y, positive if x > y, and zero if the values are equal. + // + // Implementations that recursively compare subcomponents of + // the value should use the CompareDepth function, not Cmp, to + // avoid infinite recursion on cyclic structures. + // + // The depth parameter is used to bound comparisons of cyclic + // data structures. Implementations should decrement depth + // before calling CompareDepth and should return an error if depth + // < 1. + // + // Client code should not call this method. Instead, use the + // standalone Compare or Equals functions, which are defined for + // all pairs of operands. + Cmp(y Value, depth int) (int, error) +} + var ( - _ Comparable = None - _ Comparable = Int{} - _ Comparable = False - _ Comparable = Float(0) - _ Comparable = String("") - _ Comparable = (*Dict)(nil) - _ Comparable = (*List)(nil) - _ Comparable = Tuple(nil) - _ Comparable = (*Set)(nil) + _ TotallyOrdered = Int{} + _ TotallyOrdered = Float(0) + _ Comparable = False + _ Comparable = String("") + _ Comparable = (*Dict)(nil) + _ Comparable = (*List)(nil) + _ Comparable = Tuple(nil) + _ Comparable = (*Set)(nil) ) // A Callable value f may be the operand of a function call, f(x). @@ -204,6 +235,13 @@ type Sliceable interface { Slice(start, end, step int) Value } +// A Container is a value that supports the 'in' operator. +type Container interface { + Value + // Has reports whether the value contains the specified element. + Has(y Value) (bool, error) +} + // A HasSetIndex is an Indexable value whose elements may be assigned (x[i] = y). // // The implementation should not add Len to a negative index as the @@ -220,6 +258,10 @@ var ( _ Sliceable = Tuple(nil) _ Sliceable = String("") _ Sliceable = (*List)(nil) + _ Container = Tuple(nil) + _ Container = String("") + _ Container = (*List)(nil) + _ Container = (*Set)(nil) ) // An Iterator provides a sequence of values to the caller. @@ -229,13 +271,17 @@ var ( // // Example usage: // -// iter := iterable.Iterator() +// var seq Iterator = ... +// iter := seq.Iterate() // defer iter.Done() -// var x Value -// for iter.Next(&x) { +// var elem Value +// for iter.Next(elem) { // ... // } // +// Or, using go1.23 iterators: +// +// for elem := range Elements(seq) { ... } type Iterator interface { // If the iterator is exhausted, Next returns false. // Otherwise it sets *p to the current element of the sequence, @@ -259,6 +305,8 @@ type Mapping interface { } // An IterableMapping is a mapping that supports key enumeration. +// +// See [Entries] for example use. type IterableMapping interface { Mapping Iterate() Iterator // see Iterable interface @@ -276,7 +324,7 @@ type HasSetKey interface { var _ HasSetKey = (*Dict)(nil) // A HasBinary value may be used as either operand of these binary operators: -// + - * / // % in not in | & ^ << >> +// + - * / // % in not in | & ^ << >> // // The Side argument indicates whether the receiver is the left or right operand. // @@ -296,7 +344,7 @@ const ( ) // A HasUnary value may be used as the operand of these unary operators: -// + - ~ +// + - ~ // // An implementation may decline to handle an operation by returning (nil, nil). // For this reason, clients should always call the standalone Unary(op, x) @@ -354,9 +402,6 @@ func (NoneType) Type() string { return "NoneType" } func (NoneType) Freeze() {} // immutable func (NoneType) Truth() Bool { return False } func (NoneType) Hash() (uint32, error) { return 0, nil } -func (NoneType) CompareSameType(op syntax.Token, y Value, depth int) (bool, error) { - return threeway(op, 0), nil -} // Bool is the type of a Starlark bool. type Bool bool @@ -385,10 +430,47 @@ func (x Bool) CompareSameType(op syntax.Token, y_ Value, depth int) (bool, error // Float is the type of a Starlark float. type Float float64 -func (f Float) String() string { return strconv.FormatFloat(float64(f), 'g', 6, 64) } -func (f Float) Type() string { return "float" } -func (f Float) Freeze() {} // immutable -func (f Float) Truth() Bool { return f != 0.0 } +func (f Float) String() string { + var buf strings.Builder + f.format(&buf, 'g') + return buf.String() +} + +func (f Float) format(buf *strings.Builder, conv byte) { + ff := float64(f) + if !isFinite(ff) { + if math.IsInf(ff, +1) { + buf.WriteString("+inf") + } else if math.IsInf(ff, -1) { + buf.WriteString("-inf") + } else { + buf.WriteString("nan") + } + return + } + + // %g is the default format used by str. + // It uses the minimum precision to avoid ambiguity, + // and always includes a '.' or an 'e' so that the value + // is self-evidently a float, not an int. + if conv == 'g' || conv == 'G' { + s := strconv.FormatFloat(ff, conv, -1, 64) + buf.WriteString(s) + // Ensure result always has a decimal point if no exponent. + // "123" -> "123.0" + if strings.IndexByte(s, conv-'g'+'e') < 0 && strings.IndexByte(s, '.') < 0 { + buf.WriteString(".0") + } + return + } + + // %[eEfF] use 6-digit precision + buf.WriteString(strconv.FormatFloat(ff, conv, 6, 64)) +} + +func (f Float) Type() string { return "float" } +func (f Float) Freeze() {} // immutable +func (f Float) Truth() Bool { return f != 0.0 } func (f Float) Hash() (uint32, error) { // Equal float and int values must yield the same hash. // TODO(adonovan): opt: if f is non-integral, and thus not equal @@ -407,29 +489,38 @@ func isFinite(f float64) bool { return math.Abs(f) <= math.MaxFloat64 } -func (x Float) CompareSameType(op syntax.Token, y_ Value, depth int) (bool, error) { - y := y_.(Float) - switch op { - case syntax.EQL: - return x == y, nil - case syntax.NEQ: - return x != y, nil - case syntax.LE: - return x <= y, nil - case syntax.LT: - return x < y, nil - case syntax.GE: - return x >= y, nil - case syntax.GT: - return x > y, nil +// Cmp implements comparison of two Float values. +// Required by the TotallyOrdered interface. +func (f Float) Cmp(v Value, depth int) (int, error) { + g := v.(Float) + return floatCmp(f, g), nil +} + +// floatCmp performs a three-valued comparison on floats, +// which are totally ordered with NaN > +Inf. +func floatCmp(x, y Float) int { + if x > y { + return +1 + } else if x < y { + return -1 + } else if x == y { + return 0 } - panic(op) + + // At least one operand is NaN. + if x == x { + return -1 // y is NaN + } else if y == y { + return +1 // x is NaN + } + return 0 // both NaN } func (f Float) rational() *big.Rat { return new(big.Rat).SetFloat64(float64(f)) } // AsFloat returns the float64 value closest to x. -// The f result is undefined if x is not a float or int. +// The f result is undefined if x is not a float or Int. +// The result may be infinite if x is a very large Int. func AsFloat(x Value) (f float64, ok bool) { switch x := x.(type) { case Float: @@ -440,7 +531,13 @@ func AsFloat(x Value) (f float64, ok bool) { return 0, false } -func (x Float) Mod(y Float) Float { return Float(math.Mod(float64(x), float64(y))) } +func (x Float) Mod(y Float) Float { + z := Float(math.Mod(float64(x), float64(y))) + if (x < 0) != (y < 0) && z != 0 { + z += y + } + return z +} // Unary implements the operations +float and -float. func (f Float) Unary(op syntax.Token) (Value, error) { @@ -453,13 +550,20 @@ func (f Float) Unary(op syntax.Token) (Value, error) { return nil, nil } -// String is the type of a Starlark string. +// String is the type of a Starlark text string. // -// A String encapsulates an an immutable sequence of bytes, +// A String encapsulates an immutable sequence of bytes, // but strings are not directly iterable. Instead, iterate // over the result of calling one of these four methods: // codepoints, codepoint_ords, elems, elem_ords. // +// Strings typically contain text; use Bytes for binary strings. +// The Starlark spec defines text strings as sequences of UTF-k +// codes that encode Unicode code points. In this Go implementation, +// k=8, whereas in a Java implementation, k=16. For portability, +// operations on strings should aim to avoid assumptions about +// the value of k. +// // Warning: the contract of the Value interface's String method is that // it returns the value printed in Starlark notation, // so s.String() or fmt.Sprintf("%s", s) returns a quoted string. @@ -467,7 +571,7 @@ func (f Float) Unary(op syntax.Token) (Value, error) { // of a Starlark string as a Go string. type String string -func (s String) String() string { return strconv.Quote(string(s)) } +func (s String) String() string { return syntax.Quote(string(s), false) } func (s String) GoString() string { return string(s) } func (s String) Type() string { return "string" } func (s String) Freeze() {} // immutable @@ -497,99 +601,146 @@ func (x String) CompareSameType(op syntax.Token, y_ Value, depth int) (bool, err return threeway(op, strings.Compare(string(x), string(y))), nil } +func (s String) Has(y Value) (bool, error) { + needle, ok := y.(String) + if !ok { + return false, fmt.Errorf("'in ' requires string as left operand, not %s", y.Type()) + } + return strings.Contains(string(s), string(needle)), nil +} + func AsString(x Value) (string, bool) { v, ok := x.(String); return string(v), ok } -// A stringIterable is an iterable whose iterator yields a sequence of -// either Unicode code points or elements (bytes), -// either numerically or as successive substrings. -type stringIterable struct { - s String - ords bool - codepoints bool +// A stringElems is an iterable whose iterator yields a sequence of +// elements (bytes), either numerically or as successive substrings. +// It is an indexable sequence. +type stringElems struct { + s String + ords bool } -var _ Iterable = (*stringIterable)(nil) +var ( + _ Iterable = (*stringElems)(nil) + _ Indexable = (*stringElems)(nil) +) -func (si stringIterable) String() string { - var etype string - if si.codepoints { - etype = "codepoint" +func (si stringElems) String() string { + if si.ords { + return si.s.String() + ".elem_ords()" } else { - etype = "elem" + return si.s.String() + ".elems()" } +} +func (si stringElems) Type() string { return "string.elems" } +func (si stringElems) Freeze() {} // immutable +func (si stringElems) Truth() Bool { return True } +func (si stringElems) Hash() (uint32, error) { return 0, fmt.Errorf("unhashable: %s", si.Type()) } +func (si stringElems) Iterate() Iterator { return &stringElemsIterator{si, 0} } +func (si stringElems) Len() int { return len(si.s) } +func (si stringElems) Index(i int) Value { if si.ords { - return si.s.String() + "." + etype + "_ords()" + return MakeInt(int(si.s[i])) } else { - return si.s.String() + "." + etype + "s()" + // TODO(adonovan): opt: preallocate canonical 1-byte strings + // to avoid interface allocation. + return si.s[i : i+1] + } +} + +type stringElemsIterator struct { + si stringElems + i int +} + +func (it *stringElemsIterator) Next(p *Value) bool { + if it.i == len(it.si.s) { + return false } + *p = it.si.Index(it.i) + it.i++ + return true } -func (si stringIterable) Type() string { - if si.codepoints { - return "codepoints" + +func (*stringElemsIterator) Done() {} + +// A stringCodepoints is an iterable whose iterator yields a sequence of +// Unicode code points, either numerically or as successive substrings. +// It is not indexable. +type stringCodepoints struct { + s String + ords bool +} + +var _ Iterable = (*stringCodepoints)(nil) + +func (si stringCodepoints) String() string { + if si.ords { + return si.s.String() + ".codepoint_ords()" } else { - return "elems" + return si.s.String() + ".codepoints()" } } -func (si stringIterable) Freeze() {} // immutable -func (si stringIterable) Truth() Bool { return True } -func (si stringIterable) Hash() (uint32, error) { return 0, fmt.Errorf("unhashable: %s", si.Type()) } -func (si stringIterable) Iterate() Iterator { return &stringIterator{si, 0} } +func (si stringCodepoints) Type() string { return "string.codepoints" } +func (si stringCodepoints) Freeze() {} // immutable +func (si stringCodepoints) Truth() Bool { return True } +func (si stringCodepoints) Hash() (uint32, error) { return 0, fmt.Errorf("unhashable: %s", si.Type()) } +func (si stringCodepoints) Iterate() Iterator { return &stringCodepointsIterator{si, 0} } -type stringIterator struct { - si stringIterable +type stringCodepointsIterator struct { + si stringCodepoints i int } -func (it *stringIterator) Next(p *Value) bool { +func (it *stringCodepointsIterator) Next(p *Value) bool { s := it.si.s[it.i:] if s == "" { return false } - if it.si.codepoints { - r, sz := utf8.DecodeRuneInString(string(s)) - if !it.si.ords { - *p = s[:sz] + r, sz := utf8.DecodeRuneInString(string(s)) + if !it.si.ords { + if r == utf8.RuneError { + *p = String(r) } else { - *p = MakeInt(int(r)) + *p = s[:sz] } - it.i += sz } else { - b := int(s[0]) - if !it.si.ords { - *p = s[:1] - } else { - *p = MakeInt(b) - } - it.i += 1 + *p = MakeInt(int(r)) } + it.i += sz return true } -func (*stringIterator) Done() {} +func (*stringCodepointsIterator) Done() {} // A Function is a function defined by a Starlark def statement or lambda expression. // The initialization behavior of a Starlark module is also represented by a Function. type Function struct { funcode *compile.Funcode - module *module + module *Module defaults Tuple freevars Tuple } -// A module is the dynamic counterpart to a Program. -// All functions in the same program share a module. -type module struct { - program *compile.Program +// A Module represents an evaluated Starlark module. +// It is the dynamic counterpart to a [Program]. +// All functions in the same program share a Module. +type Module struct { + program *Program predeclared StringDict globals []Value constants []Value } -// makeGlobalDict returns a new, unfrozen StringDict containing all global +// Program returns the program from which this module was constructed. +func (m *Module) Program() *Program { + return m.program +} + +// Globals returns a new StringDict containing all global // variables so far defined in the module. -func (m *module) makeGlobalDict() StringDict { - r := make(StringDict, len(m.program.Globals)) - for i, id := range m.program.Globals { +func (m *Module) Globals() StringDict { + r := make(StringDict, len(m.program.compiled.Globals)) + for i, id := range m.program.compiled.Globals { if v := m.globals[i]; v != nil { r[id.Name] = v } @@ -597,6 +748,12 @@ func (m *module) makeGlobalDict() StringDict { return r } +// Predeclared returns the predeclared environment used +// to construct this module. +func (m *Module) Predeclared() StringDict { + return m.predeclared +} + func (fn *Function) Name() string { return fn.funcode.Name } // "lambda" for anonymous functions func (fn *Function) Doc() string { return fn.funcode.Doc } func (fn *Function) Hash() (uint32, error) { return hashString(fn.funcode.Name), nil } @@ -604,10 +761,13 @@ func (fn *Function) Freeze() { fn.defaults.Freeze(); fn.freevars.F func (fn *Function) String() string { return toString(fn) } func (fn *Function) Type() string { return "function" } func (fn *Function) Truth() Bool { return true } +func (fn *Function) Module() *Module { return fn.module } -// Globals returns a new, unfrozen StringDict containing all global +// Globals returns a new StringDict containing all global // variables so far defined in the function's module. -func (fn *Function) Globals() StringDict { return fn.module.makeGlobalDict() } +// +// fn.Globals() is equivalent to fn.Module().Globals(). +func (fn *Function) Globals() StringDict { return fn.module.Globals() } func (fn *Function) Position() syntax.Position { return fn.funcode.Pos } func (fn *Function) NumParams() int { return fn.funcode.NumParams } @@ -624,9 +784,46 @@ func (fn *Function) Param(i int) (string, syntax.Position) { id := fn.funcode.Locals[i] return id.Name, id.Pos } + +// ParamDefault returns the default value of the specified parameter +// (0 <= i < NumParams()), or nil if the parameter is not optional. +func (fn *Function) ParamDefault(i int) Value { + if i < 0 || i >= fn.NumParams() { + panic(i) + } + + // fn.defaults omits all required params up to the first optional param. It + // also does not include *args or **kwargs at the end. + firstOptIdx := fn.NumParams() - len(fn.defaults) + if fn.HasVarargs() { + firstOptIdx-- + } + if fn.HasKwargs() { + firstOptIdx-- + } + if i < firstOptIdx || i >= firstOptIdx+len(fn.defaults) { + return nil + } + + dflt := fn.defaults[i-firstOptIdx] + if _, ok := dflt.(mandatory); ok { + return nil + } + return dflt +} + func (fn *Function) HasVarargs() bool { return fn.funcode.HasVarargs } func (fn *Function) HasKwargs() bool { return fn.funcode.HasKwargs } +// NumFreeVars returns the number of free variables of this function. +func (fn *Function) NumFreeVars() int { return len(fn.funcode.FreeVars) } + +// FreeVar returns the binding (name and binding position) and value +// of the i'th free variable of function fn. +func (fn *Function) FreeVar(i int) (Binding, Value) { + return Binding(fn.funcode.FreeVars[i]), fn.freevars[i].(*cell).v +} + // A Builtin is a function implemented in Go. type Builtin struct { name string @@ -667,13 +864,12 @@ func NewBuiltin(name string, fn func(thread *Thread, fn *Builtin, args Tuple, kw // In the example below, the value of f is the string.index // built-in method bound to the receiver value "abc": // -// f = "abc".index; f("a"); f("b") +// f = "abc".index; f("a"); f("b") // // In the common case, the receiver is bound only during the call, // but this still results in the creation of a temporary method closure: // -// "abc".index("a") -// +// "abc".index("a") func (b *Builtin) BindReceiver(recv Value) *Builtin { return &Builtin{name: b.name, fn: b.fn, recv: recv} } @@ -708,6 +904,14 @@ func (d *Dict) Freeze() { d.ht.freeze() func (d *Dict) Truth() Bool { return d.Len() > 0 } func (d *Dict) Hash() (uint32, error) { return 0, fmt.Errorf("unhashable type: dict") } +func (x *Dict) Union(y *Dict) *Dict { + z := new(Dict) + z.ht.init(x.Len()) // a lower bound + z.ht.addAll(&x.ht) // can't fail + z.ht.addAll(&y.ht) // can't fail + return z +} + func (d *Dict) Attr(name string) (Value, error) { return builtinAttr(d, name, dictMethods) } func (d *Dict) AttrNames() []string { return builtinAttrNames(dictMethods) } @@ -729,8 +933,8 @@ func dictsEqual(x, y *Dict, depth int) (bool, error) { if x.Len() != y.Len() { return false, nil } - for _, xitem := range x.Items() { - key, xval := xitem[0], xitem[1] + for e := x.ht.head; e != nil; e = e.next { + key, xval := e.key, e.value if yval, found, _ := y.Get(key); !found { return false, nil @@ -806,6 +1010,17 @@ func (l *List) Iterate() Iterator { return &listIterator{l: l} } +func (l *List) Has(y Value) (bool, error) { + for _, x := range l.elems { + if eq, err := Equal(x, y); err != nil { + return false, err + } else if eq { + return true, nil + } + } + return false, nil +} + func (x *List) CompareSameType(op syntax.Token, y_ Value, depth int) (bool, error) { y := y_.(*List) // It's tempting to check x == y as an optimization here, @@ -905,6 +1120,7 @@ func (t Tuple) Slice(start, end, step int) Value { } func (t Tuple) Iterate() Iterator { return &tupleIterator{elems: t} } + func (t Tuple) Freeze() { for _, elem := range t { elem.Freeze() @@ -919,6 +1135,17 @@ func (x Tuple) CompareSameType(op syntax.Token, y_ Value, depth int) (bool, erro return sliceCompare(op, x, y, depth) } +func (t Tuple) Has(y Value) (bool, error) { + for _, x := range t { + if eq, err := Equal(x, y); err != nil { + return false, err + } else if eq { + return true, nil + } + } + return false, nil +} + func (t Tuple) Hash() (uint32, error) { // Use same algorithm as Python. var x, mult uint32 = 0x345678, 1000003 @@ -970,7 +1197,6 @@ func (s *Set) Len() int { return int(s.ht.len) } func (s *Set) Iterate() Iterator { return s.ht.iterate() } func (s *Set) String() string { return toString(s) } func (s *Set) Type() string { return "set" } -func (s *Set) elems() []Value { return s.ht.keys() } func (s *Set) Freeze() { s.ht.freeze() } func (s *Set) Hash() (uint32, error) { return 0, fmt.Errorf("unhashable type: set") } func (s *Set) Truth() Bool { return s.Len() > 0 } @@ -987,6 +1213,34 @@ func (x *Set) CompareSameType(op syntax.Token, y_ Value, depth int) (bool, error case syntax.NEQ: ok, err := setsEqual(x, y, depth) return !ok, err + case syntax.GE: // superset + if x.Len() < y.Len() { + return false, nil + } + iter := y.Iterate() + defer iter.Done() + return x.IsSuperset(iter) + case syntax.LE: // subset + if x.Len() > y.Len() { + return false, nil + } + iter := y.Iterate() + defer iter.Done() + return x.IsSubset(iter) + case syntax.GT: // proper superset + if x.Len() <= y.Len() { + return false, nil + } + iter := y.Iterate() + defer iter.Done() + return x.IsSuperset(iter) + case syntax.LT: // proper subset + if x.Len() >= y.Len() { + return false, nil + } + iter := y.Iterate() + defer iter.Done() + return x.IsSubset(iter) default: return false, fmt.Errorf("%s %s %s not implemented", x.Type(), op, y.Type()) } @@ -996,19 +1250,36 @@ func setsEqual(x, y *Set, depth int) (bool, error) { if x.Len() != y.Len() { return false, nil } - for _, elem := range x.elems() { - if found, _ := y.Has(elem); !found { + for e := x.ht.head; e != nil; e = e.next { + if found, _ := y.Has(e.key); !found { return false, nil } } return true, nil } -func (s *Set) Union(iter Iterator) (Value, error) { +func setFromIterator(iter Iterator) (*Set, error) { + var x Value set := new(Set) - for _, elem := range s.elems() { - set.Insert(elem) // can't fail + for iter.Next(&x) { + err := set.Insert(x) + if err != nil { + return set, err + } + } + return set, nil +} + +func (s *Set) clone() *Set { + set := new(Set) + for e := s.ht.head; e != nil; e = e.next { + set.Insert(e.key) // can't fail } + return set +} + +func (s *Set) Union(iter Iterator) (Value, error) { + set := s.clone() var x Value for iter.Next(&x) { if err := set.Insert(x); err != nil { @@ -1018,6 +1289,82 @@ func (s *Set) Union(iter Iterator) (Value, error) { return set, nil } +func (s *Set) InsertAll(iter Iterator) error { + var x Value + for iter.Next(&x) { + if err := s.Insert(x); err != nil { + return err + } + } + return nil +} + +func (s *Set) Difference(other Iterator) (Value, error) { + diff := s.clone() + var x Value + for other.Next(&x) { + if _, err := diff.Delete(x); err != nil { + return nil, err + } + } + return diff, nil +} + +func (s *Set) IsSuperset(other Iterator) (bool, error) { + var x Value + for other.Next(&x) { + found, err := s.Has(x) + if err != nil { + return false, err + } + if !found { + return false, nil + } + } + return true, nil +} + +func (s *Set) IsSubset(other Iterator) (bool, error) { + if count, err := s.ht.count(other); err != nil { + return false, err + } else { + return count == s.Len(), nil + } +} + +func (s *Set) Intersection(other Iterator) (Value, error) { + intersect := new(Set) + var x Value + for other.Next(&x) { + found, err := s.Has(x) + if err != nil { + return nil, err + } + if found { + err = intersect.Insert(x) + if err != nil { + return nil, err + } + } + } + return intersect, nil +} + +func (s *Set) SymmetricDifference(other Iterator) (Value, error) { + diff := s.clone() + var x Value + for other.Next(&x) { + found, err := diff.Delete(x) + if err != nil { + return nil, err + } + if !found { + diff.Insert(x) + } + } + return diff, nil +} + // toString returns the string form of value v. // It may be more efficient than v.String() for larger values. func toString(v Value) string { @@ -1038,6 +1385,7 @@ func writeValue(out *strings.Builder, x Value, path []Value) { case nil: out.WriteString("") // indicates a bug + // These four cases are duplicates of T.String(), for efficiency. case NoneType: out.WriteString("None") @@ -1052,7 +1400,7 @@ func writeValue(out *strings.Builder, x Value, path []Value) { } case String: - fmt.Fprintf(out, "%q", string(x)) + out.WriteString(syntax.Quote(string(x), false)) case *List: out.WriteByte('[') @@ -1097,8 +1445,8 @@ func writeValue(out *strings.Builder, x Value, path []Value) { out.WriteString("...") // dict contains itself } else { sep := "" - for _, item := range x.Items() { - k, v := item[0], item[1] + for e := x.ht.head; e != nil; e = e.next { + k, v := e.key, e.value out.WriteString(sep) writeValue(out, k, path) out.WriteString(": ") @@ -1110,11 +1458,11 @@ func writeValue(out *strings.Builder, x Value, path []Value) { case *Set: out.WriteString("set([") - for i, elem := range x.elems() { - if i > 0 { + for e := x.ht.head; e != nil; e = e.next { + if e != x.ht.head { out.WriteString(", ") } - writeValue(out, elem, path) + writeValue(out, e.key, path) } out.WriteString("])") @@ -1132,14 +1480,16 @@ func pathContains(path []Value, x Value) bool { return false } -const maxdepth = 10 +// CompareLimit is the depth limit on recursive comparison operations such as == and <. +// Comparison of data structures deeper than this limit may fail. +var CompareLimit = 10 // Equal reports whether two Starlark values are equal. func Equal(x, y Value) (bool, error) { if x, ok := x.(String); ok { return x == y, nil // fast path for an important special case } - return EqualDepth(x, y, maxdepth) + return EqualDepth(x, y, CompareLimit) } // EqualDepth reports whether two Starlark values are equal. @@ -1158,7 +1508,7 @@ func EqualDepth(x, y Value, depth int) (bool, error) { // Recursive comparisons by implementations of Value.CompareSameType // should use CompareDepth to prevent infinite recursion. func Compare(op syntax.Token, x, y Value) (bool, error) { - return CompareDepth(op, x, y, maxdepth) + return CompareDepth(op, x, y, CompareLimit) } // CompareDepth compares two Starlark values. @@ -1177,6 +1527,14 @@ func CompareDepth(op syntax.Token, x, y Value, depth int) (bool, error) { return xcomp.CompareSameType(op, y, depth) } + if xcomp, ok := x.(TotallyOrdered); ok { + t, err := xcomp.Cmp(y, depth) + if err != nil { + return false, err + } + return threeway(op, t), nil + } + // use identity comparison switch op { case syntax.EQL: @@ -1193,11 +1551,10 @@ func CompareDepth(op syntax.Token, x, y Value, depth int) (bool, error) { switch x := x.(type) { case Int: if y, ok := y.(Float); ok { - if y != y { - return false, nil // y is NaN - } var cmp int - if !math.IsInf(float64(y), 0) { + if y != y { + cmp = -1 // y is NaN + } else if !math.IsInf(float64(y), 0) { cmp = x.rational().Cmp(y.rational()) // y is finite } else if y > 0 { cmp = -1 // y is +Inf @@ -1208,16 +1565,15 @@ func CompareDepth(op syntax.Token, x, y Value, depth int) (bool, error) { } case Float: if y, ok := y.(Int); ok { - if x != x { - return false, nil // x is NaN - } var cmp int - if !math.IsInf(float64(x), 0) { + if x != x { + cmp = +1 // x is NaN + } else if !math.IsInf(float64(x), 0) { cmp = x.rational().Cmp(y.rational()) // x is finite } else if x > 0 { - cmp = -1 // x is +Inf + cmp = +1 // x is +Inf } else { - cmp = +1 // x is -Inf + cmp = -1 // x is -Inf } return threeway(op, cmp), nil } @@ -1274,6 +1630,8 @@ func Len(x Value) int { switch x := x.(type) { case String: return x.Len() + case Indexable: + return x.Len() case Sequence: return x.Len() } @@ -1291,3 +1649,70 @@ func Iterate(x Value) Iterator { } return nil } + +// Bytes is the type of a Starlark binary string. +// +// A Bytes encapsulates an immutable sequence of bytes. +// It is comparable, indexable, and sliceable, but not directly iterable; +// use bytes.elems() for an iterable view. +// +// In this Go implementation, the elements of 'string' and 'bytes' are +// both bytes, but in other implementations, notably Java, the elements +// of a 'string' are UTF-16 codes (Java chars). The spec abstracts text +// strings as sequences of UTF-k codes that encode Unicode code points, +// and operations that convert from text to binary incur UTF-k-to-UTF-8 +// transcoding; conversely, conversion from binary to text incurs +// UTF-8-to-UTF-k transcoding. Because k=8 for Go, these operations +// are the identity function, at least for valid encodings of text. +type Bytes string + +var ( + _ Comparable = Bytes("") + _ Sliceable = Bytes("") + _ Indexable = Bytes("") + _ Container = Bytes("") +) + +func (b Bytes) String() string { return syntax.Quote(string(b), true) } +func (b Bytes) Type() string { return "bytes" } +func (b Bytes) Freeze() {} // immutable +func (b Bytes) Truth() Bool { return len(b) > 0 } +func (b Bytes) Hash() (uint32, error) { return String(b).Hash() } +func (b Bytes) Len() int { return len(b) } +func (b Bytes) Index(i int) Value { return b[i : i+1] } + +func (b Bytes) Attr(name string) (Value, error) { return builtinAttr(b, name, bytesMethods) } +func (b Bytes) AttrNames() []string { return builtinAttrNames(bytesMethods) } + +func (b Bytes) Slice(start, end, step int) Value { + if step == 1 { + return b[start:end] + } + + sign := signum(step) + var str []byte + for i := start; signum(end-i) == sign; i += step { + str = append(str, b[i]) + } + return Bytes(str) +} + +func (x Bytes) CompareSameType(op syntax.Token, y_ Value, depth int) (bool, error) { + y := y_.(Bytes) + return threeway(op, strings.Compare(string(x), string(y))), nil +} + +func (b Bytes) Has(y Value) (bool, error) { + switch needle := y.(type) { + case Bytes: + return strings.Contains(string(b), string(needle)), nil + case Int: + var by byte + if err := AsInt(needle, &by); err != nil { + return false, fmt.Errorf("int in bytes: %s", err) + } + return strings.IndexByte(string(b), by) >= 0, nil + default: + return false, fmt.Errorf("'in bytes' requires bytes or int as left operand, not %s", y.Type()) + } +} diff --git a/vendor/go.starlark.net/starlarkstruct/struct.go b/vendor/go.starlark.net/starlarkstruct/struct.go index 1982cc085..44612d28e 100644 --- a/vendor/go.starlark.net/starlarkstruct/struct.go +++ b/vendor/go.starlark.net/starlarkstruct/struct.go @@ -4,7 +4,6 @@ // Package starlarkstruct defines the Starlark types 'struct' and // 'module', both optional language extensions. -// package starlarkstruct // import "go.starlark.net/starlarkstruct" // It is tempting to introduce a variant of Struct that is a wrapper @@ -36,10 +35,9 @@ import ( // // An application can add 'struct' to the Starlark environment like so: // -// globals := starlark.StringDict{ -// "struct": starlark.NewBuiltin("struct", starlarkstruct.Make), -// } -// +// globals := starlark.StringDict{ +// "struct": starlark.NewBuiltin("struct", starlarkstruct.Make), +// } func Make(_ *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { if len(args) > 0 { return nil, fmt.Errorf("struct: unexpected positional arguments") @@ -66,7 +64,7 @@ func FromKeywords(constructor starlark.Value, kwargs []starlark.Tuple) *Struct { return s } -// FromStringDict returns a whose elements are those of d. +// FromStringDict returns a new struct instance whose elements are those of d. // The constructor parameter specifies the constructor; use Default for an ordinary struct. func FromStringDict(constructor starlark.Value, d starlark.StringDict) *Struct { if constructor == nil { @@ -101,6 +99,7 @@ func FromStringDict(constructor starlark.Value, d starlark.StringDict) *Struct { type Struct struct { constructor starlark.Value entries entries // sorted by name + frozen bool } // Default is the default constructor for structs. @@ -132,11 +131,12 @@ func (s *Struct) ToStringDict(d starlark.StringDict) { func (s *Struct) String() string { buf := new(strings.Builder) - if s.constructor == Default { + switch constructor := s.constructor.(type) { + case starlark.String: // NB: The Java implementation always prints struct // even for Bazel provider instances. - buf.WriteString("struct") // avoid String()'s quotation - } else { + buf.WriteString(constructor.GoString()) // avoid String()'s quotation + default: buf.WriteString(s.constructor.String()) } buf.WriteByte('(') @@ -173,8 +173,11 @@ func (s *Struct) Hash() (uint32, error) { return x, nil } func (s *Struct) Freeze() { - for _, e := range s.entries { - e.value.Freeze() + if !s.frozen { + s.frozen = true + for _, e := range s.entries { + e.value.Freeze() + } } } diff --git a/vendor/go.starlark.net/syntax/options.go b/vendor/go.starlark.net/syntax/options.go new file mode 100644 index 000000000..109aaf630 --- /dev/null +++ b/vendor/go.starlark.net/syntax/options.go @@ -0,0 +1,63 @@ +// Copyright 2023 The Bazel Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package syntax + +import _ "unsafe" // for linkname + +// FileOptions specifies various per-file options that affect static +// aspects of an individual file such as parsing, name resolution, and +// code generation. (Options that affect global dynamics are typically +// controlled through [go.starlark.net/starlark.Thread].) +// +// The zero value of FileOptions is the default behavior. +// +// Many functions in this package come in two versions: the legacy +// standalone function (such as [Parse]) uses [LegacyFileOptions], +// whereas the more recent method (such as [FileOptions.Parse]) honors the +// provided options. The second form is preferred. In other packages, +// the modern version is a standalone function with a leading +// FileOptions parameter and the name suffix "Options", such as +// [go.starlark.net/starlark.ExecFileOptions]. +type FileOptions struct { + // resolver + Set bool // allow references to the 'set' built-in function + While bool // allow 'while' statements + TopLevelControl bool // allow if/for/while statements at top-level + GlobalReassign bool // allow reassignment to top-level names + LoadBindsGlobally bool // load creates global not file-local bindings (deprecated) + + // compiler + Recursion bool // disable recursion check for functions in this file +} + +// TODO(adonovan): provide a canonical flag parser for FileOptions. +// (And use it in the testdata "options:" strings.) + +// LegacyFileOptions returns a new FileOptions containing the current +// values of the resolver package's legacy global variables such as +// [resolve.AllowRecursion], etc. +// These variables may be associated with command-line flags. +func LegacyFileOptions() *FileOptions { + return &FileOptions{ + Set: resolverAllowSet, + While: resolverAllowGlobalReassign, + TopLevelControl: resolverAllowGlobalReassign, + GlobalReassign: resolverAllowGlobalReassign, + Recursion: resolverAllowRecursion, + LoadBindsGlobally: resolverLoadBindsGlobally, + } +} + +// Access resolver (legacy) flags, if they are linked in; false otherwise. +var ( + //go:linkname resolverAllowSet go.starlark.net/resolve.AllowSet + resolverAllowSet bool + //go:linkname resolverAllowGlobalReassign go.starlark.net/resolve.AllowGlobalReassign + resolverAllowGlobalReassign bool + //go:linkname resolverAllowRecursion go.starlark.net/resolve.AllowRecursion + resolverAllowRecursion bool + //go:linkname resolverLoadBindsGlobally go.starlark.net/resolve.LoadBindsGlobally + resolverLoadBindsGlobally bool +) diff --git a/vendor/go.starlark.net/syntax/parse.go b/vendor/go.starlark.net/syntax/parse.go index 0281e4b87..52bdb3875 100644 --- a/vendor/go.starlark.net/syntax/parse.go +++ b/vendor/go.starlark.net/syntax/parse.go @@ -23,19 +23,27 @@ const ( RetainComments Mode = 1 << iota // retain comments in AST; see Node.Comments ) +// Parse calls the Parse method of LegacyFileOptions(). +// +// Deprecated: use [FileOptions.Parse] instead, +// because this function relies on legacy global variables. +func Parse(filename string, src interface{}, mode Mode) (f *File, err error) { + return LegacyFileOptions().Parse(filename, src, mode) +} + // Parse parses the input data and returns the corresponding parse tree. // -// If src != nil, ParseFile parses the source from src and the filename +// If src != nil, Parse parses the source from src and the filename // is only used when recording position information. // The type of the argument for the src parameter must be string, -// []byte, or io.Reader. -// If src == nil, ParseFile parses the file specified by filename. -func Parse(filename string, src interface{}, mode Mode) (f *File, err error) { +// []byte, io.Reader, or FilePortion. +// If src == nil, Parse parses the file specified by filename. +func (opts *FileOptions) Parse(filename string, src interface{}, mode Mode) (f *File, err error) { in, err := newScanner(filename, src, mode&RetainComments != 0) if err != nil { return nil, err } - p := parser{in: in} + p := parser{options: opts, in: in} defer p.in.recover(&err) p.nextToken() // read first lookahead token @@ -47,6 +55,14 @@ func Parse(filename string, src interface{}, mode Mode) (f *File, err error) { return f, nil } +// ParseCompoundStmt calls the ParseCompoundStmt method of LegacyFileOptions(). +// +// Deprecated: use [FileOptions.ParseCompoundStmt] instead, +// because this function relies on legacy global variables. +func ParseCompoundStmt(filename string, readline func() ([]byte, error)) (f *File, err error) { + return LegacyFileOptions().ParseCompoundStmt(filename, readline) +} + // ParseCompoundStmt parses a single compound statement: // a blank line, a def, for, while, or if statement, or a // semicolon-separated list of simple statements followed @@ -54,13 +70,13 @@ func Parse(filename string, src interface{}, mode Mode) (f *File, err error) { // ParseCompoundStmt does not consume any following input. // The parser calls the readline function each // time it needs a new line of input. -func ParseCompoundStmt(filename string, readline func() ([]byte, error)) (f *File, err error) { +func (opts *FileOptions) ParseCompoundStmt(filename string, readline func() ([]byte, error)) (f *File, err error) { in, err := newScanner(filename, readline, false) if err != nil { return nil, err } - p := parser{in: in} + p := parser{options: opts, in: in} defer p.in.recover(&err) p.nextToken() // read first lookahead token @@ -79,18 +95,26 @@ func ParseCompoundStmt(filename string, readline func() ([]byte, error)) (f *Fil } } - return &File{Path: filename, Stmts: stmts}, nil + return &File{Options: opts, Path: filename, Stmts: stmts}, nil +} + +// ParseExpr calls the ParseExpr method of LegacyFileOptions(). +// +// Deprecated: use [FileOptions.ParseExpr] instead, +// because this function relies on legacy global variables. +func ParseExpr(filename string, src interface{}, mode Mode) (expr Expr, err error) { + return LegacyFileOptions().ParseExpr(filename, src, mode) } // ParseExpr parses a Starlark expression. // A comma-separated list of expressions is parsed as a tuple. // See Parse for explanation of parameters. -func ParseExpr(filename string, src interface{}, mode Mode) (expr Expr, err error) { +func (opts *FileOptions) ParseExpr(filename string, src interface{}, mode Mode) (expr Expr, err error) { in, err := newScanner(filename, src, mode&RetainComments != 0) if err != nil { return nil, err } - p := parser{in: in} + p := parser{options: opts, in: in} defer p.in.recover(&err) p.nextToken() // read first lookahead token @@ -112,9 +136,10 @@ func ParseExpr(filename string, src interface{}, mode Mode) (expr Expr, err erro } type parser struct { - in *scanner - tok Token - tokval tokenValue + options *FileOptions + in *scanner + tok Token + tokval tokenValue } // nextToken advances the scanner and returns the position of the @@ -139,7 +164,7 @@ func (p *parser) parseFile() *File { } stmts = p.parseStmt(stmts) } - return &File{Stmts: stmts} + return &File{Options: p.options, Stmts: stmts} } func (p *parser) parseStmt(stmts []Stmt) []Stmt { @@ -158,15 +183,17 @@ func (p *parser) parseStmt(stmts []Stmt) []Stmt { func (p *parser) parseDefStmt() Stmt { defpos := p.nextToken() // consume DEF id := p.parseIdent() - p.consume(LPAREN) + lparen := p.consume(LPAREN) params := p.parseParams() - p.consume(RPAREN) + rparen := p.consume(RPAREN) p.consume(COLON) body := p.parseSuite() return &DefStmt{ Def: defpos, Name: id, + Lparen: lparen, Params: params, + Rparen: rparen, Body: body, } } @@ -275,10 +302,11 @@ func (p *parser) parseSimpleStmt(stmts []Stmt, consumeNL bool) []Stmt { } // small_stmt = RETURN expr? -// | PASS | BREAK | CONTINUE -// | LOAD ... -// | expr ('=' | '+=' | '-=' | '*=' | '/=' | '%=' | '&=' | '|=' | '^=' | '<<=' | '>>=') expr // assign -// | expr +// +// | PASS | BREAK | CONTINUE +// | LOAD ... +// | expr ('=' | '+=' | '-=' | '*=' | '/=' | '%=' | '&=' | '|=' | '^=' | '<<=' | '>>=') expr // assign +// | expr func (p *parser) parseSmallStmt() Stmt { switch p.tok { case RETURN: @@ -357,9 +385,6 @@ func (p *parser) parseLoadStmt() *LoadStmt { Name: lit.Value.(string), }) - case RPAREN: - p.in.errorf(p.in.pos, "trailing comma in load statement") - default: p.in.errorf(p.in.pos, `load operand must be "name" or localname="name" (got %#v)`, p.tok) } @@ -415,21 +440,23 @@ func (p *parser) consume(t Token) Position { } // params = (param COMMA)* param COMMA? -// | +// +// | // // param = IDENT -// | IDENT EQ test -// | STAR -// | STAR IDENT -// | STARSTAR IDENT +// +// | IDENT EQ test +// | STAR +// | STAR IDENT +// | STARSTAR IDENT // // parseParams parses a parameter list. The resulting expressions are of the form: // -// *Ident x -// *Binary{Op: EQ, X: *Ident, Y: Expr} x=y -// *Unary{Op: STAR} * -// *Unary{Op: STAR, X: *Ident} *args -// *Unary{Op: STARSTAR, X: *Ident} **kwargs +// *Ident x +// *Binary{Op: EQ, X: *Ident, Y: Expr} x=y +// *Unary{Op: STAR} * +// *Unary{Op: STAR, X: *Ident} *args +// *Unary{Op: STARSTAR, X: *Ident} **kwargs func (p *parser) parseParams() []Expr { var params []Expr for p.tok != RPAREN && p.tok != COLON && p.tok != EOF { @@ -532,7 +559,7 @@ func (p *parser) parseTest() Expr { return x } -// parseTestNoCond parses a a single-component expression without +// parseTestNoCond parses a single-component expression without // consuming a trailing 'if expr else expr'. func (p *parser) parseTestNoCond() Expr { if p.tok == LAMBDA { @@ -651,9 +678,10 @@ func init() { } // primary_with_suffix = primary -// | primary '.' IDENT -// | primary slice_suffix -// | primary call_suffix +// +// | primary '.' IDENT +// | primary slice_suffix +// | primary call_suffix func (p *parser) parsePrimaryWithSuffix() Expr { x := p.parsePrimary() for { @@ -770,19 +798,19 @@ func (p *parser) parseArgs() []Expr { return args } -// primary = IDENT -// | INT | FLOAT -// | STRING -// | '[' ... // list literal or comprehension -// | '{' ... // dict literal or comprehension -// | '(' ... // tuple or parenthesized expression -// | ('-'|'+'|'~') primary_with_suffix +// primary = IDENT +// +// | INT | FLOAT | STRING | BYTES +// | '[' ... // list literal or comprehension +// | '{' ... // dict literal or comprehension +// | '(' ... // tuple or parenthesized expression +// | ('-'|'+'|'~') primary_with_suffix func (p *parser) parsePrimary() Expr { switch p.tok { case IDENT: return p.parseIdent() - case INT, FLOAT, STRING: + case INT, FLOAT, STRING, BYTES: var val interface{} tok := p.tok switch tok { @@ -794,7 +822,7 @@ func (p *parser) parsePrimary() Expr { } case FLOAT: val = p.tokval.float - case STRING: + case STRING, BYTES: val = p.tokval.string } raw := p.tokval.raw @@ -832,14 +860,17 @@ func (p *parser) parsePrimary() Expr { X: x, } } - p.in.errorf(p.in.pos, "got %#v, want primary expression", p.tok) + + // Report start pos of final token as it may be a NEWLINE (#532). + p.in.errorf(p.tokval.pos, "got %#v, want primary expression", p.tok) panic("unreachable") } // list = '[' ']' -// | '[' expr ']' -// | '[' expr expr_list ']' -// | '[' expr (FOR loop_variables IN expr)+ ']' +// +// | '[' expr ']' +// | '[' expr expr_list ']' +// | '[' expr (FOR loop_variables IN expr)+ ']' func (p *parser) parseList() Expr { lbrack := p.nextToken() if p.tok == RBRACK { @@ -866,8 +897,9 @@ func (p *parser) parseList() Expr { } // dict = '{' '}' -// | '{' dict_entry_list '}' -// | '{' dict_entry FOR loop_variables IN expr '}' +// +// | '{' dict_entry_list '}' +// | '{' dict_entry FOR loop_variables IN expr '}' func (p *parser) parseDict() Expr { lbrace := p.nextToken() if p.tok == RBRACE { @@ -905,8 +937,9 @@ func (p *parser) parseDictEntry() *DictEntry { } // comp_suffix = FOR loopvars IN expr comp_suffix -// | IF expr comp_suffix -// | ']' or ')' (end) +// +// | IF expr comp_suffix +// | ']' or ')' (end) // // There can be multiple FOR/IF clauses; the first is always a FOR. func (p *parser) parseComprehensionSuffix(lbrace Position, body Expr, endBrace Token) Expr { diff --git a/vendor/go.starlark.net/syntax/quote.go b/vendor/go.starlark.net/syntax/quote.go index cc9a8d0ae..741e106ad 100644 --- a/vendor/go.starlark.net/syntax/quote.go +++ b/vendor/go.starlark.net/syntax/quote.go @@ -10,6 +10,8 @@ import ( "fmt" "strconv" "strings" + "unicode" + "unicode/utf8" ) // unesc maps single-letter chars following \ to their actual values. @@ -40,23 +42,21 @@ var esc = [256]byte{ '"': '"', } -// notEsc is a list of characters that can follow a \ in a string value -// without having to escape the \. That is, since ( is in this list, we -// quote the Go string "foo\\(bar" as the Python literal "foo\(bar". -// This really does happen in BUILD files, especially in strings -// being used as shell arguments containing regular expressions. -const notEsc = " !#$%&()*+,-./:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ{|}~" - // unquote unquotes the quoted string, returning the actual -// string value, whether the original was triple-quoted, and -// an error describing invalid input. -func unquote(quoted string) (s string, triple bool, err error) { +// string value, whether the original was triple-quoted, +// whether it was a byte string, and an error describing invalid input. +func unquote(quoted string) (s string, triple, isByte bool, err error) { // Check for raw prefix: means don't interpret the inner \. raw := false if strings.HasPrefix(quoted, "r") { raw = true quoted = quoted[1:] } + // Check for bytes prefix. + if strings.HasPrefix(quoted, "b") { + isByte = true + quoted = quoted[1:] + } if len(quoted) < 2 { err = fmt.Errorf("string literal too short") @@ -127,22 +127,25 @@ func unquote(quoted string) (s string, triple bool, err error) { switch quoted[1] { default: - // In Python, if \z (for some byte z) is not a known escape sequence - // then it appears as literal text in the string. - buf.WriteString(quoted[:2]) - quoted = quoted[2:] + // In Starlark, like Go, a backslash must escape something. + // (Python still treats unnecessary backslashes literally, + // but since 3.6 has emitted a deprecation warning.) + err = fmt.Errorf("invalid escape sequence \\%c", quoted[1]) + return case '\n': // Ignore the escape and the line break. quoted = quoted[2:] case 'a', 'b', 'f', 'n', 'r', 't', 'v', '\\', '\'', '"': - // One-char escape + // One-char escape. + // Escapes are allowed for both kinds of quotation + // mark, not just the kind in use. buf.WriteByte(unesc[quoted[1]]) quoted = quoted[2:] case '0', '1', '2', '3', '4', '5', '6', '7': - // Octal escape, up to 3 digits. + // Octal escape, up to 3 digits, \OOO. n := int(quoted[1] - '0') quoted = quoted[2:] for i := 1; i < 3; i++ { @@ -152,6 +155,10 @@ func unquote(quoted string) (s string, triple bool, err error) { n = n*8 + int(quoted[0]-'0') quoted = quoted[1:] } + if !isByte && n > 127 { + err = fmt.Errorf(`non-ASCII octal escape \%o (use \u%04X for the UTF-8 encoding of U+%04X)`, n, n, n) + return + } if n >= 256 { // NOTE: Python silently discards the high bit, // so that '\541' == '\141' == 'a'. @@ -162,7 +169,7 @@ func unquote(quoted string) (s string, triple bool, err error) { buf.WriteByte(byte(n)) case 'x': - // Hexadecimal escape, exactly 2 digits. + // Hexadecimal escape, exactly 2 digits, \xXX. [0-127] if len(quoted) < 4 { err = fmt.Errorf(`truncated escape sequence %s`, quoted) return @@ -172,8 +179,41 @@ func unquote(quoted string) (s string, triple bool, err error) { err = fmt.Errorf(`invalid escape sequence %s`, quoted[:4]) return } + if !isByte && n > 127 { + err = fmt.Errorf(`non-ASCII hex escape %s (use \u%04X for the UTF-8 encoding of U+%04X)`, + quoted[:4], n, n) + return + } buf.WriteByte(byte(n)) quoted = quoted[4:] + + case 'u', 'U': + // Unicode code point, 4 (\uXXXX) or 8 (\UXXXXXXXX) hex digits. + sz := 6 + if quoted[1] == 'U' { + sz = 10 + } + if len(quoted) < sz { + err = fmt.Errorf(`truncated escape sequence %s`, quoted) + return + } + n, err1 := strconv.ParseUint(quoted[2:sz], 16, 0) + if err1 != nil { + err = fmt.Errorf(`invalid escape sequence %s`, quoted[:sz]) + return + } + if n > unicode.MaxRune { + err = fmt.Errorf(`code point out of range: %s (max \U%08x)`, + quoted[:sz], n) + return + } + // As in Go, surrogates are disallowed. + if 0xD800 <= n && n < 0xE000 { + err = fmt.Errorf(`invalid Unicode code point U+%04X`, n) + return + } + buf.WriteRune(rune(n)) + quoted = quoted[sz:] } } @@ -191,79 +231,79 @@ func indexByte(s string, b byte) int { return -1 } -// hex is a list of the hexadecimal digits, for use in quoting. -// We always print lower-case hexadecimal. -const hex = "0123456789abcdef" +// Quote returns a Starlark literal that denotes s. +// If b, it returns a bytes literal. +func Quote(s string, b bool) string { + const hex = "0123456789abcdef" + var runeTmp [utf8.UTFMax]byte -// quote returns the quoted form of the string value "x". -// If triple is true, quote uses the triple-quoted form """x""". -func quote(unquoted string, triple bool) string { - q := `"` - if triple { - q = `"""` + buf := make([]byte, 0, 3*len(s)/2) + if b { + buf = append(buf, 'b') } - - buf := new(strings.Builder) - buf.WriteString(q) - - for i := 0; i < len(unquoted); i++ { - c := unquoted[i] - if c == '"' && triple && (i+1 < len(unquoted) && unquoted[i+1] != '"' || i+2 < len(unquoted) && unquoted[i+2] != '"') { - // Can pass up to two quotes through, because they are followed by a non-quote byte. - buf.WriteByte(c) - if i+1 < len(unquoted) && unquoted[i+1] == '"' { - buf.WriteByte(c) - i++ - } - continue + buf = append(buf, '"') + for width := 0; len(s) > 0; s = s[width:] { + r := rune(s[0]) + width = 1 + if r >= utf8.RuneSelf { + r, width = utf8.DecodeRuneInString(s) } - if triple && c == '\n' { - // Can allow newline in triple-quoted string. - buf.WriteByte(c) + if width == 1 && r == utf8.RuneError { + // String (!b) literals accept \xXX escapes only for ASCII, + // but we must use them here to represent invalid bytes. + // The result is not a legal literal. + buf = append(buf, `\x`...) + buf = append(buf, hex[s[0]>>4]) + buf = append(buf, hex[s[0]&0xF]) continue } - if c == '\'' { - // Can allow ' since we always use ". - buf.WriteByte(c) + if r == '"' || r == '\\' { // always backslashed + buf = append(buf, '\\') + buf = append(buf, byte(r)) continue } - if c == '\\' { - if i+1 < len(unquoted) && indexByte(notEsc, unquoted[i+1]) >= 0 { - // Can pass \ through when followed by a byte that - // known not to be a valid escape sequence and also - // that does not trigger an escape sequence of its own. - // Use this, because various BUILD files do. - buf.WriteByte('\\') - buf.WriteByte(unquoted[i+1]) - i++ - continue - } - } - if esc[c] != 0 { - buf.WriteByte('\\') - buf.WriteByte(esc[c]) + if strconv.IsPrint(r) { + n := utf8.EncodeRune(runeTmp[:], r) + buf = append(buf, runeTmp[:n]...) continue } - if c < 0x20 || c >= 0x80 { - // BUILD files are supposed to be Latin-1, so escape all control and high bytes. - // I'd prefer to use \x here, but Blaze does not implement - // \x in quoted strings (b/7272572). - buf.WriteByte('\\') - buf.WriteByte(hex[c>>6]) // actually octal but reusing hex digits 0-7. - buf.WriteByte(hex[(c>>3)&7]) - buf.WriteByte(hex[c&7]) - /* - buf.WriteByte('\\') - buf.WriteByte('x') - buf.WriteByte(hex[c>>4]) - buf.WriteByte(hex[c&0xF]) - */ - continue + switch r { + case '\a': + buf = append(buf, `\a`...) + case '\b': + buf = append(buf, `\b`...) + case '\f': + buf = append(buf, `\f`...) + case '\n': + buf = append(buf, `\n`...) + case '\r': + buf = append(buf, `\r`...) + case '\t': + buf = append(buf, `\t`...) + case '\v': + buf = append(buf, `\v`...) + default: + switch { + case r < ' ' || r == 0x7f: + buf = append(buf, `\x`...) + buf = append(buf, hex[byte(r)>>4]) + buf = append(buf, hex[byte(r)&0xF]) + case r > utf8.MaxRune: + r = 0xFFFD + fallthrough + case r < 0x10000: + buf = append(buf, `\u`...) + for s := 12; s >= 0; s -= 4 { + buf = append(buf, hex[r>>uint(s)&0xF]) + } + default: + buf = append(buf, `\U`...) + for s := 28; s >= 0; s -= 4 { + buf = append(buf, hex[r>>uint(s)&0xF]) + } + } } - buf.WriteByte(c) - continue } - - buf.WriteString(q) - return buf.String() + buf = append(buf, '"') + return string(buf) } diff --git a/vendor/go.starlark.net/syntax/scan.go b/vendor/go.starlark.net/syntax/scan.go index 51cf8855f..4a6dac641 100644 --- a/vendor/go.starlark.net/syntax/scan.go +++ b/vendor/go.starlark.net/syntax/scan.go @@ -9,7 +9,6 @@ package syntax import ( "fmt" "io" - "io/ioutil" "log" "math/big" "os" @@ -35,6 +34,7 @@ const ( INT // 123 FLOAT // 1.23e45 STRING // "foo" or 'foo' or '''foo''' or r'foo' or r"foo" + BYTES // b"foo", etc // Punctuation PLUS // + @@ -98,6 +98,26 @@ const ( RETURN WHILE + // Reserved words (following Python); unused in Starlark + + AS + // ASSERT // heavily used in our tests + ASYNC + AWAIT + CLASS + DEL + EXCEPT + FINALLY + FROM + GLOBAL + IMPORT + IS + NONLOCAL + RAISE + TRY + WITH + YIELD + maxToken ) @@ -180,6 +200,35 @@ var tokenNames = [...]string{ PASS: "pass", RETURN: "return", WHILE: "while", + + // Reserved words (following Python); unused in Starlark + + AS: "as", + // ASSERT: "assert", // heavily used in our tests + ASYNC: "async", + AWAIT: "await", + CLASS: "class", + DEL: "del", + EXCEPT: "except", + FINALLY: "finally", + FROM: "from", + GLOBAL: "global", + IMPORT: "import", + IS: "is", + NONLOCAL: "nonlocal", + RAISE: "raise", + TRY: "try", + WITH: "with", + YIELD: "yield", +} + +// A FilePortion describes the content of a portion of a file. +// Callers may provide a FilePortion for the src argument of Parse +// when the desired initial line and column numbers are not (1, 1), +// such as when an expression is parsed from within larger file. +type FilePortion struct { + Content []byte + FirstLine, FirstCol int32 } // A Position describes the location of a rune of input. @@ -249,13 +298,17 @@ type scanner struct { } func newScanner(filename string, src interface{}, keepComments bool) (*scanner, error) { + var firstLine, firstCol int32 = 1, 1 + if portion, ok := src.(FilePortion); ok { + firstLine, firstCol = portion.FirstLine, portion.FirstCol + } sc := &scanner{ - pos: Position{file: &filename, Line: 1, Col: 1}, + pos: MakePosition(&filename, firstLine, firstCol), indentstk: make([]int, 1, 10), // []int{0} + spare capacity lineStart: true, keepComments: keepComments, } - sc.readline, _ = src.(func() ([]byte, error)) // REPL only + sc.readline, _ = src.(func() ([]byte, error)) // ParseCompoundStmt (REPL) only if sc.readline == nil { data, err := readSource(filename, src) if err != nil { @@ -273,14 +326,16 @@ func readSource(filename string, src interface{}) ([]byte, error) { case []byte: return src, nil case io.Reader: - data, err := ioutil.ReadAll(src) + data, err := io.ReadAll(src) if err != nil { err = &os.PathError{Op: "read", Path: filename, Err: err} return nil, err } return data, nil + case FilePortion: + return src.Content, nil case nil: - return ioutil.ReadFile(filename) + return os.ReadFile(filename) default: return nil, fmt.Errorf("invalid source: %T", src) } @@ -407,7 +462,7 @@ type tokenValue struct { int int64 // decoded int bigInt *big.Int // decoded integers > int64 float float64 // decoded float - string string // decoded string + string string // decoded string or bytes pos Position // start position of token } @@ -627,8 +682,15 @@ start: // identifier or keyword if isIdentStart(c) { - // raw string literal - if c == 'r' && len(sc.rest) > 1 && (sc.rest[1] == '"' || sc.rest[1] == '\'') { + if (c == 'r' || c == 'b') && len(sc.rest) > 1 && (sc.rest[1] == '"' || sc.rest[1] == '\'') { + // r"..." + // b"..." + sc.readRune() + c = sc.peekRune() + return sc.scanString(val, c) + } else if c == 'r' && len(sc.rest) > 2 && sc.rest[1] == 'b' && (sc.rest[2] == '"' || sc.rest[2] == '\'') { + // rb"..." + sc.readRune() sc.readRune() c = sc.peekRune() return sc.scanString(val, c) @@ -805,13 +867,26 @@ func (sc *scanner) scanString(val *tokenValue, quote rune) Token { start := sc.pos triple := len(sc.rest) >= 3 && sc.rest[0] == byte(quote) && sc.rest[1] == byte(quote) && sc.rest[2] == byte(quote) sc.readRune() + + // String literals may contain escaped or unescaped newlines, + // causing them to span multiple lines (gulps) of REPL input; + // they are the only such token. Thus we cannot call endToken, + // as it assumes sc.rest is unchanged since startToken. + // Instead, buffer the token here. + // TODO(adonovan): opt: buffer only if we encounter a newline. + raw := new(strings.Builder) + + // Copy the prefix, e.g. r' or " (see startToken). + raw.Write(sc.token[:len(sc.token)-len(sc.rest)]) + if !triple { - // Precondition: startToken was already called. + // single-quoted string literal for { if sc.eof() { sc.error(val.pos, "unexpected EOF in string") } c := sc.readRune() + raw.WriteRune(c) if c == quote { break } @@ -822,22 +897,16 @@ func (sc *scanner) scanString(val *tokenValue, quote rune) Token { if sc.eof() { sc.error(val.pos, "unexpected EOF in string") } - sc.readRune() + c = sc.readRune() + raw.WriteRune(c) } } - sc.endToken(val) } else { // triple-quoted string literal sc.readRune() + raw.WriteRune(quote) sc.readRune() - - // A triple-quoted string literal may span multiple - // gulps of REPL input; it is the only such token. - // Thus we must avoid {start,end}Token. - raw := new(strings.Builder) - - // Copy the prefix, e.g. r''' or """ (see startToken). - raw.Write(sc.token[:len(sc.token)-len(sc.rest)]) + raw.WriteRune(quote) quoteCount := 0 for { @@ -862,15 +931,19 @@ func (sc *scanner) scanString(val *tokenValue, quote rune) Token { raw.WriteRune(c) } } - val.raw = raw.String() } + val.raw = raw.String() - s, _, err := unquote(val.raw) + s, _, isByte, err := unquote(val.raw) if err != nil { sc.error(start, err.Error()) } val.string = s - return STRING + if isByte { + return BYTES + } else { + return STRING + } } func (sc *scanner) scanNumber(val *tokenValue, c rune) Token { @@ -1071,19 +1144,21 @@ var keywordToken = map[string]Token{ "while": WHILE, // reserved words: - "as": ILLEGAL, - // "assert": ILLEGAL, // heavily used by our tests - "class": ILLEGAL, - "del": ILLEGAL, - "except": ILLEGAL, - "finally": ILLEGAL, - "from": ILLEGAL, - "global": ILLEGAL, - "import": ILLEGAL, - "is": ILLEGAL, - "nonlocal": ILLEGAL, - "raise": ILLEGAL, - "try": ILLEGAL, - "with": ILLEGAL, - "yield": ILLEGAL, + "as": AS, + // "assert": ASSERT, // heavily used by our tests + "async": ASYNC, + "await": AWAIT, + "class": CLASS, + "del": DEL, + "except": EXCEPT, + "finally": FINALLY, + "from": FROM, + "global": GLOBAL, + "import": IMPORT, + "is": IS, + "nonlocal": NONLOCAL, + "raise": RAISE, + "try": TRY, + "with": WITH, + "yield": YIELD, } diff --git a/vendor/go.starlark.net/syntax/syntax.go b/vendor/go.starlark.net/syntax/syntax.go index b4817c1a2..5bfbcec39 100644 --- a/vendor/go.starlark.net/syntax/syntax.go +++ b/vendor/go.starlark.net/syntax/syntax.go @@ -70,7 +70,8 @@ type File struct { Path string Stmts []Stmt - Module interface{} // a *resolve.Module, set by resolver + Module interface{} // a *resolve.Module, set by resolver + Options *FileOptions } func (x *File) Span() (start, end Position) { @@ -99,9 +100,10 @@ func (*LoadStmt) stmt() {} func (*ReturnStmt) stmt() {} // An AssignStmt represents an assignment: +// // x = 0 // x, y = y, x -// x += 1 +// x += 1 type AssignStmt struct { commentsRef OpPos Position @@ -121,7 +123,9 @@ type DefStmt struct { commentsRef Def Position Name *Ident + Lparen Position Params []Expr // param = ident | ident=expr | * | *ident | **ident + Rparen Position Body []Stmt Function interface{} // a *resolve.Function, set by resolver @@ -251,10 +255,10 @@ func (x *Ident) Span() (start, end Position) { // A Literal represents a literal string or number. type Literal struct { commentsRef - Token Token // = STRING | INT + Token Token // = STRING | BYTES | INT | FLOAT TokenPos Position Raw string // uninterpreted text - Value interface{} // = string | int64 | *big.Int + Value interface{} // = string | int64 | *big.Int | float64 } func (x *Literal) Span() (start, end Position) { @@ -398,10 +402,6 @@ func (x *DictEntry) Span() (start, end Position) { } // A LambdaExpr represents an inline function abstraction. -// -// Although they may be added in future, lambda expressions are not -// currently part of the Starlark spec, so their use is controlled by the -// resolver.AllowLambda flag. type LambdaExpr struct { commentsRef Lambda Position diff --git a/vendor/go.starlark.net/syntax/walk.go b/vendor/go.starlark.net/syntax/walk.go index 1491149c6..65f68083d 100644 --- a/vendor/go.starlark.net/syntax/walk.go +++ b/vendor/go.starlark.net/syntax/walk.go @@ -49,6 +49,10 @@ func Walk(n Node, f func(Node) bool) { Walk(n.X, f) walkStmts(n.Body, f) + case *WhileStmt: + Walk(n.Cond, f) + walkStmts(n.Body, f) + case *ReturnStmt: if n.Result != nil { Walk(n.Result, f) @@ -119,9 +123,7 @@ func Walk(n Node, f func(Node) bool) { case *DictExpr: for _, entry := range n.List { - entry := entry.(*DictEntry) - Walk(entry.Key, f) - Walk(entry.Value, f) + Walk(entry, f) } case *UnaryExpr: diff --git a/vendor/modules.txt b/vendor/modules.txt index 67e0b3228..35f51641d 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -362,8 +362,8 @@ go.opencensus.io/trace go.opencensus.io/trace/internal go.opencensus.io/trace/propagation go.opencensus.io/trace/tracestate -# go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 -## explicit; go 1.13 +# go.starlark.net v0.0.0-20250804182900-3c9dc17c5f2e +## explicit; go 1.19 go.starlark.net/internal/compile go.starlark.net/internal/spell go.starlark.net/resolve