Skip to content

Commit 2b1ac5d

Browse files
jmgomezjmgomez
authored
Implements the declarative parser (#1342)
* Implements the declarative parser * Fixes unrelated test issues
1 parent ef7b8c9 commit 2b1ac5d

15 files changed

+322
-34
lines changed

‎config.nims

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,3 +6,4 @@ switch("define", "ssl")
66
switch("path", "vendor" / "zippy" / "src")
77
switch("path", "vendor" / "sat" / "src")
88
switch("path", "vendor" / "checksums" / "src")
9+
switch("define", "zippyNoSimd")

‎src/nimble.nim

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ import nimblepkg/packageinfotypes, nimblepkg/packageinfo, nimblepkg/version,
1919
nimblepkg/nimbledatafile, nimblepkg/packagemetadatafile,
2020
nimblepkg/displaymessages, nimblepkg/sha1hashes, nimblepkg/syncfile,
2121
nimblepkg/deps, nimblepkg/nimblesat, nimblepkg/forge_aliases, nimblepkg/nimenv,
22-
nimblepkg/downloadnim
22+
nimblepkg/downloadnim, nimblepkg/declarativeparser
2323

2424
const
2525
nimblePathsFileName* = "nimble.paths"
@@ -87,7 +87,11 @@ proc processFreeDependenciesSAT(rootPkgInfo: PackageInfo, options: Options): Has
8787
var pkgsToInstall: seq[(string, Version)] = @[]
8888
var rootPkgInfo = rootPkgInfo
8989
rootPkgInfo.requires &= options.extraRequires
90-
var pkgList = initPkgList(rootPkgInfo, options).mapIt(it.toFullInfo(options))
90+
var pkgList = initPkgList(rootPkgInfo, options)
91+
if options.useDeclarativeParser:
92+
pkgList = pkgList.mapIt(it.toRequiresInfo(options))
93+
else:
94+
pkgList = pkgList.mapIt(it.toFullInfo(options))
9195
var allPkgsInfo: seq[PackageInfo] = pkgList & rootPkgInfo
9296
#Remove from the pkglist the packages that exists in lock file and has a different vcsRevision
9397
var upgradeVersions = initTable[string, VersionRange]()

‎src/nimble.nim.cfg

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,3 +7,4 @@
77
--path:"../vendor/checksums/src"
88
-d:ssl
99
-d:nimcore # Enable 'gorge' in Nim's VM. See https://github.com/nim-lang/Nim/issues/8096
10+
-d:zippyNoSimd

‎src/nimblepkg/declarativeparser.nim

Lines changed: 186 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,186 @@
1+
## Utility API for Nim package managers.
2+
## (c) 2021 Andreas Rumpf
3+
4+
import std/strutils
5+
6+
import compiler/[ast, idents, msgs, syntaxes, options, pathutils, lineinfos]
7+
import version, packageinfotypes, packageinfo, options, packageparser
8+
9+
type NimbleFileInfo* = object
10+
requires*: seq[string]
11+
srcDir*: string
12+
version*: string
13+
tasks*: seq[(string, string)]
14+
hasInstallHooks*: bool
15+
hasErrors*: bool
16+
17+
proc eqIdent(a, b: string): bool {.inline.} =
18+
cmpIgnoreCase(a, b) == 0 and a[0] == b[0]
19+
20+
proc extract(n: PNode, conf: ConfigRef, result: var NimbleFileInfo) =
21+
case n.kind
22+
of nkStmtList, nkStmtListExpr:
23+
for child in n:
24+
extract(child, conf, result)
25+
of nkCallKinds:
26+
if n[0].kind == nkIdent:
27+
case n[0].ident.s
28+
of "requires":
29+
for i in 1 ..< n.len:
30+
var ch = n[i]
31+
while ch.kind in {nkStmtListExpr, nkStmtList} and ch.len > 0:
32+
ch = ch.lastSon
33+
if ch.kind in {nkStrLit .. nkTripleStrLit}:
34+
result.requires.add ch.strVal
35+
else:
36+
localError(conf, ch.info, "'requires' takes string literals")
37+
result.hasErrors = true
38+
of "task":
39+
if n.len >= 3 and n[1].kind == nkIdent and
40+
n[2].kind in {nkStrLit .. nkTripleStrLit}:
41+
result.tasks.add((n[1].ident.s, n[2].strVal))
42+
of "before", "after":
43+
#[
44+
before install do:
45+
exec "git submodule update --init"
46+
var make = "make"
47+
when defined(windows):
48+
make = "mingw32-make"
49+
exec make
50+
]#
51+
if n.len >= 3 and n[1].kind == nkIdent and n[1].ident.s == "install":
52+
result.hasInstallHooks = true
53+
else:
54+
discard
55+
of nkAsgn, nkFastAsgn:
56+
if n[0].kind == nkIdent and eqIdent(n[0].ident.s, "srcDir"):
57+
if n[1].kind in {nkStrLit .. nkTripleStrLit}:
58+
result.srcDir = n[1].strVal
59+
else:
60+
localError(conf, n[1].info, "assignments to 'srcDir' must be string literals")
61+
result.hasErrors = true
62+
elif n[0].kind == nkIdent and eqIdent(n[0].ident.s, "version"):
63+
if n[1].kind in {nkStrLit .. nkTripleStrLit}:
64+
result.version = n[1].strVal
65+
else:
66+
localError(conf, n[1].info, "assignments to 'version' must be string literals")
67+
result.hasErrors = true
68+
else:
69+
discard
70+
71+
proc extractRequiresInfo*(nimbleFile: string): NimbleFileInfo =
72+
## Extract the `requires` information from a Nimble file. This does **not**
73+
## evaluate the Nimble file. Errors are produced on stderr/stdout and are
74+
## formatted as the Nim compiler does it. The parser uses the Nim compiler
75+
## as an API. The result can be empty, this is not an error, only parsing
76+
## errors are reported.
77+
var conf = newConfigRef()
78+
conf.foreignPackageNotes = {}
79+
conf.notes = {}
80+
conf.mainPackageNotes = {}
81+
conf.errorMax = high(int)
82+
conf.structuredErrorHook = proc(
83+
config: ConfigRef, info: TLineInfo, msg: string, severity: Severity
84+
) {.gcsafe.} =
85+
localError(config, info, warnUser, msg)
86+
87+
let fileIdx = fileInfoIdx(conf, AbsoluteFile nimbleFile)
88+
var parser: Parser
89+
if setupParser(parser, fileIdx, newIdentCache(), conf):
90+
extract(parseAll(parser), conf, result)
91+
closeParser(parser)
92+
result.hasErrors = result.hasErrors or conf.errorCounter > 0
93+
94+
type PluginInfo* = object
95+
builderPatterns*: seq[(string, string)]
96+
97+
proc extractPlugin(
98+
nimscriptFile: string, n: PNode, conf: ConfigRef, result: var PluginInfo
99+
) =
100+
case n.kind
101+
of nkStmtList, nkStmtListExpr:
102+
for child in n:
103+
extractPlugin(nimscriptFile, child, conf, result)
104+
of nkCallKinds:
105+
if n[0].kind == nkIdent:
106+
case n[0].ident.s
107+
of "builder":
108+
if n.len >= 3 and n[1].kind in {nkStrLit .. nkTripleStrLit}:
109+
result.builderPatterns.add((n[1].strVal, nimscriptFile))
110+
else:
111+
discard
112+
else:
113+
discard
114+
115+
proc extractPluginInfo*(nimscriptFile: string, info: var PluginInfo) =
116+
var conf = newConfigRef()
117+
conf.foreignPackageNotes = {}
118+
conf.notes = {}
119+
conf.mainPackageNotes = {}
120+
121+
let fileIdx = fileInfoIdx(conf, AbsoluteFile nimscriptFile)
122+
var parser: Parser
123+
if setupParser(parser, fileIdx, newIdentCache(), conf):
124+
extractPlugin(nimscriptFile, parseAll(parser), conf, info)
125+
closeParser(parser)
126+
127+
const Operators* = {'<', '>', '=', '&', '@', '!', '^'}
128+
129+
proc token(s: string, idx: int, lit: var string): int =
130+
var i = idx
131+
if i >= s.len:
132+
return i
133+
while s[i] in Whitespace:
134+
inc(i)
135+
case s[i]
136+
of Letters, '#':
137+
lit.add s[i]
138+
inc i
139+
while i < s.len and s[i] notin (Whitespace + {'@', '#'}):
140+
lit.add s[i]
141+
inc i
142+
of '0' .. '9':
143+
while i < s.len and s[i] in {'0' .. '9', '.'}:
144+
lit.add s[i]
145+
inc i
146+
of '"':
147+
inc i
148+
while i < s.len and s[i] != '"':
149+
lit.add s[i]
150+
inc i
151+
inc i
152+
of Operators:
153+
while i < s.len and s[i] in Operators:
154+
lit.add s[i]
155+
inc i
156+
else:
157+
lit.add s[i]
158+
inc i
159+
result = i
160+
161+
iterator tokenizeRequires*(s: string): string =
162+
var start = 0
163+
var tok = ""
164+
while start < s.len:
165+
tok.setLen 0
166+
start = token(s, start, tok)
167+
yield tok
168+
169+
proc getRequires*(nimbleFileInfo: NimbleFileInfo): seq[PkgTuple] =
170+
for require in nimbleFileInfo.requires:
171+
result.add(parseRequires(require))
172+
173+
proc toRequiresInfo*(pkgInfo: PackageInfo, options: Options): PackageInfo =
174+
#For nim we only need the version. Since version is usually in the form of `version = $NimMajor & "." & $NimMinor & "." & $NimPatch
175+
#we need to use the vm to get the version. Another option could be to use the binary and ask for the version
176+
if pkgInfo.basicInfo.name.isNim:
177+
return pkgInfo.toFullInfo(options)
178+
179+
let nimbleFileInfo = extractRequiresInfo(pkgInfo.myPath)
180+
result = pkgInfo
181+
result.requires = getRequires(nimbleFileInfo)
182+
result.infoKind = pikRequires
183+
184+
when isMainModule:
185+
for x in tokenizeRequires("jester@#head >= 1.5 & <= 1.8"):
186+
echo x

‎src/nimblepkg/nimblesat.nim

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import sat/[sat, satvars]
22
import version, packageinfotypes, download, packageinfo, packageparser, options,
3-
sha1hashes, tools, downloadnim, cli
3+
sha1hashes, tools, downloadnim, cli, declarativeparser
44

55
import std/[tables, sequtils, algorithm, sets, strutils, options, strformat, os, json, jsonutils]
66

@@ -110,6 +110,15 @@ proc getMinimalInfo*(pkg: PackageInfo, options: Options): PackageMinimalInfo =
110110
if options.action.typ in {actionLock, actionDeps} or options.hasNimInLockFile():
111111
result.requires = result.requires.filterIt(not it.isNim)
112112

113+
proc getMinimalInfo*(nimbleFile: string, pkgName: string, options: Options): PackageMinimalInfo =
114+
assert options.useDeclarativeParser, "useDeclarativeParser must be set"
115+
let nimbleFileInfo = extractRequiresInfo(nimbleFile)
116+
result.name = if pkgName.isNim: "nim" else: pkgName
117+
result.version = nimbleFileInfo.version.newVersion()
118+
result.requires = nimbleFileInfo.getRequires() #TODO if package is Nim do not parse the file. Just get the version from the binary.
119+
if options.action.typ in {actionLock, actionDeps} or options.hasNimInLockFile():
120+
result.requires = result.requires.filterIt(not it.isNim)
121+
113122
proc hasVersion*(packageVersions: PackageVersions, pv: PkgTuple): bool =
114123
for pkg in packageVersions.versions:
115124
if pkg.name == pv.name and pkg.version.withinRange(pv.ver):
@@ -561,8 +570,11 @@ proc getPackageMinimalVersionsFromRepo*(repoDir: string, name: string, version:
561570
try:
562571
doCheckout(downloadMethod, tempDir, tag)
563572
let nimbleFile = findNimbleFile(tempDir, true, options)
564-
let pkgInfo = getPkgInfoFromFile(nimbleFile, options, useCache=false)
565-
result.addUnique pkgInfo.getMinimalInfo(options)
573+
if options.useDeclarativeParser:
574+
result.addUnique getMinimalInfo(nimbleFile, name, options)
575+
else:
576+
let pkgInfo = getPkgInfoFromFile(nimbleFile, options, useCache=false)
577+
result.addUnique pkgInfo.getMinimalInfo(options)
566578
except CatchableError as e:
567579
displayWarning(
568580
&"Error reading tag {tag}: for package {name}. This may not be relevant as it could be an old version of the package. \n {e.msg}",

‎src/nimblepkg/options.nim

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@ type
6464
nimBinariesDir*: string # Directory where nim binaries are stored. Separated from nimbleDir as it can be changed by the user/tests
6565
disableNimBinaries*: bool # Whether to disable the use of nim binaries
6666
maxTaggedVersions*: int # Maximum number of tags to check for a package when discovering versions in a local repo
67+
useDeclarativeParser*: bool # Whether to use the declarative parser for parsing nimble files (only when solver is SAT)
6768

6869
ActionType* = enum
6970
actionNil, actionRefresh, actionInit, actionDump, actionPublish, actionUpgrade
@@ -276,6 +277,7 @@ Nimble Options:
276277
--requires Add extra packages to the dependency resolution. Uses the same syntax as the Nimble file. Example: nimble install --requires "pkg1; pkg2 >= 1.2".
277278
--disableNimBinaries Disable the use of nim precompiled binaries. Note in some platforms precompiled binaries are not available but the flag can still be used to avoid compile the Nim version once and reuse it.
278279
--maximumTaggedVersions Maximum number of tags to check for a package when discovering versions for the SAT solver. 0 means all.
280+
--parser:declarative|nimvm Use the declarative parser or the nimvm parser (default).
279281
For more information read the GitHub readme:
280282
https://github.com/nim-lang/nimble#readme
281283
"""
@@ -662,6 +664,13 @@ proc parseFlag*(flag, val: string, result: var Options, kind = cmdLongOption) =
662664
result.useSatSolver = false
663665
else:
664666
raise nimbleError("Unknown solver option: " & val)
667+
of "parser":
668+
if val == "declarative":
669+
result.useDeclarativeParser = true
670+
elif val == "nimvm":
671+
result.useDeclarativeParser = false
672+
else:
673+
raise nimbleError("Unknown parser option: " & val)
665674
of "requires":
666675
result.extraRequires = val.split(";").mapIt(it.strip.parseRequires())
667676
of "disablenimbinaries":

‎src/nimblepkg/packageinfo.nim

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -296,7 +296,7 @@ proc setNameVersionChecksum*(pkgInfo: var PackageInfo, pkgDir: string) =
296296
proc getInstalledPackageMin*(options: Options, pkgDir, nimbleFilePath: string): PackageInfo =
297297
result = initPackageInfo(options, nimbleFilePath)
298298
setNameVersionChecksum(result, pkgDir)
299-
result.isMinimal = true
299+
result.infoKind = pikMinimal
300300
result.isInstalled = true
301301
try:
302302
fillMetaData(result, pkgDir, true, options)

‎src/nimblepkg/packageinfotypes.nim

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,10 +41,16 @@ type
4141
version: Version
4242
checksum: Sha1Hash
4343

44+
PackageInfoKind* = enum
45+
pikNone #No info
46+
pikMinimal #Minimal info, previous isMinimal
47+
pikRequires #Declarative parser only Minimal + requires (No vm involved)
48+
pikFull #Full info
49+
4450
PackageInfo* = object
4551
myPath*: string ## The path of this .nimble file
4652
isNimScript*: bool ## Determines if this pkg info was read from a nims file
47-
isMinimal*: bool
53+
infoKind*: PackageInfoKind
4854
isInstalled*: bool ## Determines if the pkg this info belongs to is installed
4955
nimbleTasks*: HashSet[string] ## All tasks defined in the Nimble file
5056
postHooks*: HashSet[string] ## Useful to know so that Nimble doesn't execHook unnecessarily
@@ -88,5 +94,8 @@ type
8894

8995
PackageDependenciesInfo* = tuple[deps: HashSet[PackageInfo], pkg: PackageInfo]
9096

97+
proc isMinimal*(pkg: PackageInfo): bool =
98+
pkg.infoKind == pikMinimal
99+
91100
const noTask* = "" # Means that noTask is being ran. Use this as key for base dependencies
92101
var satProccesedPackages*: HashSet[PackageInfo]

‎src/nimblepkg/packageparser.nim

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -310,7 +310,7 @@ proc readPackageInfo(pkgInfo: var PackageInfo, nf: NimbleFile, options: Options,
310310
if not success:
311311
if onlyMinimalInfo:
312312
pkgInfo.isNimScript = true
313-
pkgInfo.isMinimal = true
313+
pkgInfo.infoKind = pikMinimal
314314
else:
315315
try:
316316
readPackageInfoFromNims(nf, options, pkgInfo)

‎tests/issue27/a/issue27a.babel renamed to ‎tests/issue27/a/issue27a.nimble

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,6 @@
1-
[Package]
2-
name = "issue27a"
31
version = "0.1.0"
42
author = "Dominik Picheta"
53
description = "Dependency A for Issue 27"
64
license = "BSD"
75

8-
[Deps]
9-
Requires: "nimrod >= 0.9.3, issue27b"
6+
requires "nimrod >= 0.9.3, issue27b"

‎tests/issue27/b/issue27b.babel renamed to ‎tests/issue27/b/issue27b.nimble

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,7 @@
1-
[Package]
2-
name = "issue27b"
1+
32
version = "0.1.0"
43
author = "Dominik Picheta"
54
description = "Dependency B for Issue 27"
65
license = "BSD"
76

8-
[Deps]
9-
Requires: "nimrod >= 0.9.3"
7+
requires "nimrod >= 0.9.3"

‎tests/issue27/issue27.babel renamed to ‎tests/issue27/issue27.nimble

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,8 @@
1-
[Package]
2-
name = "issue27"
31
version = "0.1.0"
42
author = "Dominik Picheta"
53
description = "Test package for Issue 27"
64
license = "BSD"
75

8-
bin = "issue27"
6+
bin = @["issue27"]
97

10-
[Deps]
11-
Requires: "nimrod >= 0.9.3, issue27a"
8+
requires "nimrod >= 0.9.3, issue27a"

0 commit comments

Comments
 (0)