Implements the declarative parser

config.nims

@@ -6,3 +6,4 @@ switch("define", "ssl")
 switch("path", "vendor" / "zippy" / "src")
 switch("path", "vendor" / "sat" / "src")
 switch("path", "vendor" / "checksums" / "src")
+switch("define", "zippyNoSimd")

src/nimble.nim

@@ -19,7 +19,7 @@ import nimblepkg/packageinfotypes, nimblepkg/packageinfo, nimblepkg/version,
        nimblepkg/nimbledatafile, nimblepkg/packagemetadatafile,
        nimblepkg/displaymessages, nimblepkg/sha1hashes, nimblepkg/syncfile,
        nimblepkg/deps, nimblepkg/nimblesat, nimblepkg/forge_aliases, nimblepkg/nimenv,
-       nimblepkg/downloadnim
+       nimblepkg/downloadnim, nimblepkg/declarativeparser
 
 const
   nimblePathsFileName* = "nimble.paths"
@@ -87,7 +87,11 @@ proc processFreeDependenciesSAT(rootPkgInfo: PackageInfo, options: Options): Has
   var pkgsToInstall: seq[(string, Version)] = @[]
   var rootPkgInfo = rootPkgInfo
   rootPkgInfo.requires &= options.extraRequires
-  var pkgList = initPkgList(rootPkgInfo, options).mapIt(it.toFullInfo(options))
+  var pkgList = initPkgList(rootPkgInfo, options)
+  if options.useDeclarativeParser:
+    pkgList = pkgList.mapIt(it.toRequiresInfo(options))
+  else:
+    pkgList = pkgList.mapIt(it.toFullInfo(options))
   var allPkgsInfo: seq[PackageInfo] = pkgList & rootPkgInfo
   #Remove from the pkglist the packages that exists in lock file and has a different vcsRevision
   var upgradeVersions = initTable[string, VersionRange]()

src/nimble.nim.cfg

@@ -7,3 +7,4 @@
 --path:"../vendor/checksums/src"
 -d:ssl
 -d:nimcore # Enable 'gorge' in Nim's VM. See https://github.com/nim-lang/Nim/issues/8096
+-d:zippyNoSimd

src/nimblepkg/declarativeparser.nim

@@ -0,0 +1,186 @@
+## Utility API for Nim package managers.
+## (c) 2021 Andreas Rumpf
+
+import std/strutils
+
+import compiler/[ast, idents, msgs, syntaxes, options, pathutils, lineinfos]
+import version, packageinfotypes, packageinfo, options, packageparser
+
+type NimbleFileInfo* = object
+  requires*: seq[string]
+  srcDir*: string
+  version*: string
+  tasks*: seq[(string, string)]
+  hasInstallHooks*: bool
+  hasErrors*: bool
+
+proc eqIdent(a, b: string): bool {.inline.} =
+  cmpIgnoreCase(a, b) == 0 and a[0] == b[0]
+
+proc extract(n: PNode, conf: ConfigRef, result: var NimbleFileInfo) =
+  case n.kind
+  of nkStmtList, nkStmtListExpr:
+    for child in n:
+      extract(child, conf, result)
+  of nkCallKinds:
+    if n[0].kind == nkIdent:
+      case n[0].ident.s
+      of "requires":
+        for i in 1 ..< n.len:
+          var ch = n[i]
+          while ch.kind in {nkStmtListExpr, nkStmtList} and ch.len > 0:
+            ch = ch.lastSon
+          if ch.kind in {nkStrLit .. nkTripleStrLit}:
+            result.requires.add ch.strVal
+          else:
+            localError(conf, ch.info, "'requires' takes string literals")
+            result.hasErrors = true
+      of "task":
+        if n.len >= 3 and n[1].kind == nkIdent and
+            n[2].kind in {nkStrLit .. nkTripleStrLit}:
+          result.tasks.add((n[1].ident.s, n[2].strVal))
+      of "before", "after":
+        #[
+          before install do:
+            exec "git submodule update --init"
+            var make = "make"
+            when defined(windows):
+              make = "mingw32-make"
+            exec make
+        ]#
+        if n.len >= 3 and n[1].kind == nkIdent and n[1].ident.s == "install":
+          result.hasInstallHooks = true
+      else:
+        discard
+  of nkAsgn, nkFastAsgn:
+    if n[0].kind == nkIdent and eqIdent(n[0].ident.s, "srcDir"):
+      if n[1].kind in {nkStrLit .. nkTripleStrLit}:
+        result.srcDir = n[1].strVal
+      else:
+        localError(conf, n[1].info, "assignments to 'srcDir' must be string literals")
+        result.hasErrors = true
+    elif n[0].kind == nkIdent and eqIdent(n[0].ident.s, "version"):
+      if n[1].kind in {nkStrLit .. nkTripleStrLit}:
+        result.version = n[1].strVal
+      else:
+        localError(conf, n[1].info, "assignments to 'version' must be string literals")
+        result.hasErrors = true
+  else:
+    discard
+
+proc extractRequiresInfo*(nimbleFile: string): NimbleFileInfo =
+  ## Extract the `requires` information from a Nimble file. This does **not**
+  ## evaluate the Nimble file. Errors are produced on stderr/stdout and are
+  ## formatted as the Nim compiler does it. The parser uses the Nim compiler
+  ## as an API. The result can be empty, this is not an error, only parsing
+  ## errors are reported.
+  var conf = newConfigRef()
+  conf.foreignPackageNotes = {}
+  conf.notes = {}
+  conf.mainPackageNotes = {}
+  conf.errorMax = high(int)
+  conf.structuredErrorHook = proc(
+      config: ConfigRef, info: TLineInfo, msg: string, severity: Severity
+  ) {.gcsafe.} =
+    localError(config, info, warnUser, msg)
+
+  let fileIdx = fileInfoIdx(conf, AbsoluteFile nimbleFile)
+  var parser: Parser
+  if setupParser(parser, fileIdx, newIdentCache(), conf):
+    extract(parseAll(parser), conf, result)
+    closeParser(parser)
+  result.hasErrors = result.hasErrors or conf.errorCounter > 0
+
+type PluginInfo* = object
+  builderPatterns*: seq[(string, string)]
+
+proc extractPlugin(
+    nimscriptFile: string, n: PNode, conf: ConfigRef, result: var PluginInfo
+) =
+  case n.kind
+  of nkStmtList, nkStmtListExpr:
+    for child in n:
+      extractPlugin(nimscriptFile, child, conf, result)
+  of nkCallKinds:
+    if n[0].kind == nkIdent:
+      case n[0].ident.s
+      of "builder":
+        if n.len >= 3 and n[1].kind in {nkStrLit .. nkTripleStrLit}:
+          result.builderPatterns.add((n[1].strVal, nimscriptFile))
+      else:
+        discard
+  else:
+    discard
+
+proc extractPluginInfo*(nimscriptFile: string, info: var PluginInfo) =
+  var conf = newConfigRef()
+  conf.foreignPackageNotes = {}
+  conf.notes = {}
+  conf.mainPackageNotes = {}
+
+  let fileIdx = fileInfoIdx(conf, AbsoluteFile nimscriptFile)
+  var parser: Parser
+  if setupParser(parser, fileIdx, newIdentCache(), conf):
+    extractPlugin(nimscriptFile, parseAll(parser), conf, info)
+    closeParser(parser)
+
+const Operators* = {'<', '>', '=', '&', '@', '!', '^'}
+
+proc token(s: string, idx: int, lit: var string): int =
+  var i = idx
+  if i >= s.len:
+    return i
+  while s[i] in Whitespace:
+    inc(i)
+  case s[i]
+  of Letters, '#':
+    lit.add s[i]
+    inc i
+    while i < s.len and s[i] notin (Whitespace + {'@', '#'}):
+      lit.add s[i]
+      inc i
+  of '0' .. '9':
+    while i < s.len and s[i] in {'0' .. '9', '.'}:
+      lit.add s[i]
+      inc i
+  of '"':
+    inc i
+    while i < s.len and s[i] != '"':
+      lit.add s[i]
+      inc i
+    inc i
+  of Operators:
+    while i < s.len and s[i] in Operators:
+      lit.add s[i]
+      inc i
+  else:
+    lit.add s[i]
+    inc i
+  result = i
+
+iterator tokenizeRequires*(s: string): string =
+  var start = 0
+  var tok = ""
+  while start < s.len:
+    tok.setLen 0
+    start = token(s, start, tok)
+    yield tok
+
+proc getRequires*(nimbleFileInfo: NimbleFileInfo): seq[PkgTuple] =
+  for require in nimbleFileInfo.requires:
+    result.add(parseRequires(require))
+
+proc toRequiresInfo*(pkgInfo: PackageInfo, options: Options): PackageInfo =
+  #For nim we only need the version. Since version is usually in the form of `version = $NimMajor & "." & $NimMinor & "." & $NimPatch
+  #we need to use the vm to get the version. Another option could be to use the binary and ask for the version
+  if pkgInfo.basicInfo.name.isNim:
+    return pkgInfo.toFullInfo(options)
+
+  let nimbleFileInfo = extractRequiresInfo(pkgInfo.myPath)
+  result = pkgInfo
+  result.requires = getRequires(nimbleFileInfo)
+  result.infoKind = pikRequires
+
+when isMainModule:
+  for x in tokenizeRequires("jester@#head >= 1.5 & <= 1.8"):
+    echo x

src/nimblepkg/nimblesat.nim

@@ -1,6 +1,6 @@
 import sat/[sat, satvars] 
 import version, packageinfotypes, download, packageinfo, packageparser, options, 
-  sha1hashes, tools, downloadnim, cli
+  sha1hashes, tools, downloadnim, cli, declarativeparser
 
 import std/[tables, sequtils, algorithm, sets, strutils, options, strformat, os, json, jsonutils]
 
@@ -110,6 +110,15 @@ proc getMinimalInfo*(pkg: PackageInfo, options: Options): PackageMinimalInfo =
   if options.action.typ in {actionLock, actionDeps} or options.hasNimInLockFile():
     result.requires = result.requires.filterIt(not it.isNim)
 
+proc getMinimalInfo*(nimbleFile: string, pkgName: string, options: Options): PackageMinimalInfo =
+  assert options.useDeclarativeParser, "useDeclarativeParser must be set"
+  let nimbleFileInfo = extractRequiresInfo(nimbleFile)
+  result.name =  if pkgName.isNim: "nim" else: pkgName
+  result.version = nimbleFileInfo.version.newVersion()
+  result.requires = nimbleFileInfo.getRequires() #TODO if package is Nim do not parse the file. Just get the version from the binary.
+  if options.action.typ in {actionLock, actionDeps} or options.hasNimInLockFile():
+    result.requires = result.requires.filterIt(not it.isNim)
+
 proc hasVersion*(packageVersions: PackageVersions, pv: PkgTuple): bool =
   for pkg in packageVersions.versions:
     if pkg.name == pv.name and pkg.version.withinRange(pv.ver):
@@ -561,8 +570,11 @@ proc getPackageMinimalVersionsFromRepo*(repoDir: string, name: string, version:
       try:
         doCheckout(downloadMethod, tempDir, tag)
         let nimbleFile = findNimbleFile(tempDir, true, options)
-        let pkgInfo = getPkgInfoFromFile(nimbleFile, options, useCache=false)
-        result.addUnique pkgInfo.getMinimalInfo(options)
+        if options.useDeclarativeParser:
+          result.addUnique getMinimalInfo(nimbleFile, name, options)
+        else:
+          let pkgInfo = getPkgInfoFromFile(nimbleFile, options, useCache=false)
+          result.addUnique pkgInfo.getMinimalInfo(options)
       except CatchableError as e:
         displayWarning(
           &"Error reading tag {tag}: for package {name}. This may not be relevant as it could be an old version of the package. \n {e.msg}",

src/nimblepkg/options.nim

@@ -64,6 +64,7 @@ type
     nimBinariesDir*: string # Directory where nim binaries are stored. Separated from nimbleDir as it can be changed by the user/tests
     disableNimBinaries*: bool # Whether to disable the use of nim binaries
     maxTaggedVersions*: int # Maximum number of tags to check for a package when discovering versions in a local repo
+    useDeclarativeParser*: bool # Whether to use the declarative parser for parsing nimble files (only when solver is SAT)
 
   ActionType* = enum
     actionNil, actionRefresh, actionInit, actionDump, actionPublish, actionUpgrade
@@ -276,6 +277,7 @@ Nimble Options:
       --requires                  Add extra packages to the dependency resolution. Uses the same syntax as the Nimble file. Example: nimble install --requires "pkg1; pkg2 >= 1.2".
       --disableNimBinaries        Disable the use of nim precompiled binaries. Note in some platforms precompiled binaries are not available but the flag can still be used to avoid compile the Nim version once and reuse it.
       --maximumTaggedVersions     Maximum number of tags to check for a package when discovering versions for the SAT solver. 0 means all.
+      --parser:declarative|nimvm  Use the declarative parser or the nimvm parser (default).
 For more information read the GitHub readme:
   https://github.com/nim-lang/nimble#readme
 """
@@ -662,6 +664,13 @@ proc parseFlag*(flag, val: string, result: var Options, kind = cmdLongOption) =
       result.useSatSolver = false
     else:
       raise nimbleError("Unknown solver option: " & val)
+  of "parser":
+    if val == "declarative":
+      result.useDeclarativeParser = true
+    elif val == "nimvm":
+      result.useDeclarativeParser = false
+    else:
+      raise nimbleError("Unknown parser option: " & val)
   of "requires":
     result.extraRequires = val.split(";").mapIt(it.strip.parseRequires())
   of "disablenimbinaries":

src/nimblepkg/packageinfo.nim

@@ -296,7 +296,7 @@ proc setNameVersionChecksum*(pkgInfo: var PackageInfo, pkgDir: string) =
 proc getInstalledPackageMin*(options: Options, pkgDir, nimbleFilePath: string): PackageInfo =
   result = initPackageInfo(options, nimbleFilePath)
   setNameVersionChecksum(result, pkgDir)
-  result.isMinimal = true
+  result.infoKind = pikMinimal
   result.isInstalled = true
   try:
     fillMetaData(result, pkgDir, true, options)

src/nimblepkg/packageinfotypes.nim

@@ -41,10 +41,16 @@ type
     version: Version
     checksum: Sha1Hash
 
+  PackageInfoKind* = enum
+    pikNone #No info
+    pikMinimal #Minimal info, previous isMinimal
+    pikRequires #Declarative parser only Minimal + requires (No vm involved)
+    pikFull #Full info
+
   PackageInfo* = object
     myPath*: string ## The path of this .nimble file
     isNimScript*: bool ## Determines if this pkg info was read from a nims file
-    isMinimal*: bool
+    infoKind*: PackageInfoKind
     isInstalled*: bool ## Determines if the pkg this info belongs to is installed
     nimbleTasks*: HashSet[string] ## All tasks defined in the Nimble file
     postHooks*: HashSet[string] ## Useful to know so that Nimble doesn't execHook unnecessarily
@@ -88,5 +94,8 @@ type
 
   PackageDependenciesInfo* = tuple[deps: HashSet[PackageInfo], pkg: PackageInfo]
 
+proc isMinimal*(pkg: PackageInfo): bool =
+  pkg.infoKind == pikMinimal
+
 const noTask* = "" # Means that noTask is being ran. Use this as key for base dependencies
 var satProccesedPackages*: HashSet[PackageInfo]

src/nimblepkg/packageparser.nim

@@ -310,7 +310,7 @@ proc readPackageInfo(pkgInfo: var PackageInfo, nf: NimbleFile, options: Options,
   if not success:
     if onlyMinimalInfo:
       pkgInfo.isNimScript = true
-      pkgInfo.isMinimal = true
+      pkgInfo.infoKind = pikMinimal
     else:
       try:
         readPackageInfoFromNims(nf, options, pkgInfo)

tests/issue27/a/issue27a.babel → tests/issue27/a/issue27a.nimble

@@ -1,9 +1,6 @@
-[Package]
-name          = "issue27a"
 version       = "0.1.0"
 author        = "Dominik Picheta"
 description   = "Dependency A for Issue 27"
 license       = "BSD"
 
-[Deps]
-Requires: "nimrod >= 0.9.3, issue27b"
+requires "nimrod >= 0.9.3, issue27b"

tests/issue27/b/issue27b.babel → tests/issue27/b/issue27b.nimble

@@ -1,9 +1,7 @@
-[Package]
-name          = "issue27b"
+
 version       = "0.1.0"
 author        = "Dominik Picheta"
 description   = "Dependency B for Issue 27"
 license       = "BSD"
 
-[Deps]
-Requires: "nimrod >= 0.9.3"
+requires "nimrod >= 0.9.3"

tests/issue27/issue27.babel → tests/issue27/issue27.nimble

@@ -1,11 +1,8 @@
-[Package]
-name          = "issue27"
 version       = "0.1.0"
 author        = "Dominik Picheta"
 description   = "Test package for Issue 27"
 license       = "BSD"
 
-bin = "issue27"
+bin = @["issue27"]
 
-[Deps]
-Requires: "nimrod >= 0.9.3, issue27a"
+requires "nimrod >= 0.9.3, issue27a"