Skip to content

Commit d2ae352

Browse files
authored
refactor(disk): templates catalog (#5914)
* refactor(disk): templates catalog Signed-off-by: Dwi Siswanto <git@dw1.io> * feat(disk): drying err Signed-off-by: Dwi Siswanto <git@dw1.io> * feat(disk): simplify `DiskCatalog.OpenFile` method since `BackwardsCompatiblePaths` func is already deprecated. Signed-off-by: Dwi Siswanto <git@dw1.io> * test: update functional test cases Signed-off-by: Dwi Siswanto <git@dw1.io> * feat: reuse error Signed-off-by: Dwi Siswanto <git@dw1.io> * fix(disk): handle glob errors consistently Signed-off-by: Dwi Siswanto <git@dw1.io> * fix(disk): use forward slashes for fs.FS path ops to fix Windows compat. The io/fs package requires forward slashes ("/") as path separators regardless of the OS. Using [filepath.Separator] or [os.PathSeparator] breaks [fs.Open] and [fs.Glob] ops on Windows where the separator is backslash ("\"). Signed-off-by: Dwi Siswanto <git@dw1.io> --------- Signed-off-by: Dwi Siswanto <git@dw1.io>
1 parent f329832 commit d2ae352

File tree

8 files changed

+75
-111
lines changed

8 files changed

+75
-111
lines changed

cmd/functional-test/testcases.txt

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -36,24 +36,24 @@
3636
{{binary}} -tags cve,dos,fuzz
3737
{{binary}} -tags cve -include-tags dos,fuzz
3838
{{binary}} -tags cve -exclude-tags cve2020
39-
{{binary}} -tags cve -exclude-templates cves/2020/
40-
{{binary}} -tags cve -exclude-templates cves/2020/CVE-2020-9757.yaml
41-
{{binary}} -tags cve -exclude-templates cves/2020/CVE-2020-9757.yaml -exclude-templates cves/2021/
42-
{{binary}} -t cves/
43-
{{binary}} -t cves/ -t exposures/
44-
{{binary}} -t cves/ -t exposures/ -tags config
45-
{{binary}} -t cves/ -t exposures/ -tags config,ssrf
46-
{{binary}} -t cves/ -t exposures/ -tags config -severity high,critical
47-
{{binary}} -t cves/ -t exposures/ -tags config -severity high,critical -author geeknik,pdteam
48-
{{binary}} -t cves/ -t exposures/ -tags config -severity high,critical -author geeknik,pdteam -etags sqli
49-
{{binary}} -t cves/ -t exposures/ -tags config -severity high,critical -author geeknik,pdteam -etags sqli -exclude-templates cves/2021/
50-
{{binary}} -t cves/ -t exposures/ -tags config -severity high,critical -author geeknik,pdteam -etags sqli -exclude-templates cves/2017/CVE-2017-7269.yaml
51-
{{binary}} -t cves/ -t exposures/ -tags config -severity high,critical -author geeknik,pdteam -etags sqli -include-templates cves/2017/CVE-2017-7269.yaml
39+
{{binary}} -tags cve -exclude-templates http/cves/2020/
40+
{{binary}} -tags cve -exclude-templates http/cves/2020/CVE-2020-9757.yaml
41+
{{binary}} -tags cve -exclude-templates http/cves/2020/CVE-2020-9757.yaml -exclude-templates http/cves/2021/
42+
{{binary}} -t http/cves/
43+
{{binary}} -t http/cves/ -t http/exposures/
44+
{{binary}} -t http/cves/ -t http/exposures/ -tags config
45+
{{binary}} -t http/cves/ -t http/exposures/ -tags config,ssrf
46+
{{binary}} -t http/cves/ -t http/exposures/ -tags config -severity high,critical
47+
{{binary}} -t http/cves/ -t http/exposures/ -tags config -severity high,critical -author geeknik,pdteam
48+
{{binary}} -t http/cves/ -t http/exposures/ -tags config -severity high,critical -author geeknik,pdteam -etags sqli
49+
{{binary}} -t http/cves/ -t http/exposures/ -tags config -severity high,critical -author geeknik,pdteam -etags sqli -exclude-templates http/cves/2021/
50+
{{binary}} -t http/cves/ -t http/exposures/ -tags config -severity high,critical -author geeknik,pdteam -etags sqli -exclude-templates http/cves/2017/CVE-2017-7269.yaml
51+
{{binary}} -t http/cves/ -t http/exposures/ -tags config -severity high,critical -author geeknik,pdteam -etags sqli -include-templates http/cves/2017/CVE-2017-7269.yaml
5252

5353
# Advanced Filtering
5454
{{binary}} -tags cve -author geeknik,pdteam -tc severity=='high'
5555
{{binary}} -tc contains(authors,'pdteam')
56-
{{binary}} -t cves/ -t exposures/ -tc contains(tags,'cve') -exclude-templates cves/2020/CVE-2020-9757.yaml
56+
{{binary}} -t http/cves/ -t http/exposures/ -tc contains(tags,'cve') -exclude-templates http/cves/2020/CVE-2020-9757.yaml
5757
{{binary}} -tc protocol=='dns'
5858
{{binary}} -tc contains(http_method,'GET')
5959
{{binary}} -tc len(body)>0
@@ -65,7 +65,7 @@
6565
{{binary}} -w workflows
6666
{{binary}} -w workflows -author geeknik,pdteam
6767
{{binary}} -w workflows -severity high,critical
68-
{{binary}} -w workflows -author geeknik,pdteam -severity high,critical
68+
{{binary}} -w workflows -author geeknik,pdteam -severity high,critical
6969

7070
# Input Types
7171
# http protocol

internal/runner/lazy.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import (
77

88
"github.com/projectdiscovery/nuclei/v3/pkg/authprovider/authx"
99
"github.com/projectdiscovery/nuclei/v3/pkg/catalog"
10+
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/disk"
1011
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/loader"
1112
"github.com/projectdiscovery/nuclei/v3/pkg/output"
1213
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
@@ -68,7 +69,7 @@ func GetLazyAuthFetchCallback(opts *AuthLazyFetchOptions) authx.LazyFetchSecret
6869
return func(d *authx.Dynamic) error {
6970
tmpls := opts.TemplateStore.LoadTemplates([]string{d.TemplatePath})
7071
if len(tmpls) == 0 {
71-
return fmt.Errorf("no templates found for path: %s", d.TemplatePath)
72+
return fmt.Errorf("%w for path: %s", disk.ErrNoTemplatesFound, d.TemplatePath)
7273
}
7374
if len(tmpls) > 1 {
7475
return fmt.Errorf("multiple templates found for path: %s", d.TemplatePath)

internal/runner/runner.go

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -655,7 +655,6 @@ func (r *Runner) RunEnumeration() error {
655655
}
656656
store.Load()
657657
// TODO: remove below functions after v3 or update warning messages
658-
disk.PrintDeprecatedPathsMsgIfApplicable(r.options.Silent)
659658
templates.PrintDeprecatedProtocolNameMsgIfApplicable(r.options.Silent, r.options.Verbose)
660659

661660
// add the hosts from the metadata queries of loaded templates into input provider

pkg/catalog/disk/catalog.go

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -38,13 +38,7 @@ func NewFSCatalog(fs fs.FS, directory string) *DiskCatalog {
3838
// It is used to read template and payload files based on catalog responses.
3939
func (d *DiskCatalog) OpenFile(filename string) (io.ReadCloser, error) {
4040
if d.templatesFS == nil {
41-
file, err := os.Open(filename)
42-
if err != nil {
43-
if file, errx := os.Open(BackwardsCompatiblePaths(d.templatesDirectory, filename)); errx == nil {
44-
return file, nil
45-
}
46-
}
47-
return file, err
41+
return os.Open(filename)
4842
}
4943

5044
return d.templatesFS.Open(filename)

pkg/catalog/disk/errors.go

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
package disk
2+
3+
import "errors"
4+
5+
var (
6+
ErrNoTemplatesFound = errors.New("no templates found")
7+
)

pkg/catalog/disk/find.go

Lines changed: 43 additions & 86 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package disk
22

33
import (
4+
"fmt"
45
"io/fs"
56
"os"
67
"path/filepath"
@@ -65,41 +66,34 @@ func (c *DiskCatalog) GetTemplatesPath(definitions []string) ([]string, map[stri
6566
// or folders provided as in.
6667
func (c *DiskCatalog) GetTemplatePath(target string) ([]string, error) {
6768
processed := make(map[string]struct{})
68-
// Template input includes a wildcard
69-
if strings.Contains(target, "*") {
70-
matches, findErr := c.findGlobPathMatches(target, processed)
71-
if findErr != nil {
72-
return nil, errors.Wrap(findErr, "could not find glob matches")
73-
}
74-
if len(matches) == 0 {
75-
return nil, errors.Errorf("no templates found for path")
76-
}
77-
return matches, nil
78-
}
7969

80-
// try to handle deprecated template paths
81-
absPath := target
8270
if c.templatesFS == nil {
83-
absPath = BackwardsCompatiblePaths(c.templatesDirectory, target)
84-
if absPath != target && strings.TrimPrefix(absPath, c.templatesDirectory+string(filepath.Separator)) != target {
85-
if config.DefaultConfig.LogAllEvents {
86-
config.DefaultConfig.Logger.Print().Msgf("[%v] requested Template path %s is deprecated, please update to %s\n", aurora.Yellow("WRN").String(), target, absPath)
87-
}
88-
deprecatedPathsCounter++
89-
}
90-
9171
var err error
92-
absPath, err = c.convertPathToAbsolute(absPath)
72+
target, err = c.convertPathToAbsolute(target)
9373
if err != nil {
9474
return nil, errors.Wrapf(err, "could not find template file")
9575
}
9676
}
9777

98-
// Template input is either a file or a directory
99-
match, file, err := c.findFileMatches(absPath, processed)
78+
if strings.Contains(target, "*") {
79+
globMatches, err := c.findGlobPathMatches(target, processed)
80+
if err != nil {
81+
return nil, errors.Wrap(err, "could not globbing path")
82+
}
83+
84+
if len(globMatches) > 0 {
85+
return globMatches, nil
86+
} else {
87+
return globMatches, fmt.Errorf("%w in path %q", ErrNoTemplatesFound, target)
88+
}
89+
}
90+
91+
// `target` is either a file or a directory
92+
match, file, err := c.findFileMatches(target, processed)
10093
if err != nil {
10194
return nil, errors.Wrap(err, "could not find file")
10295
}
96+
10397
if file {
10498
if match != "" {
10599
return []string{match}, nil
@@ -109,13 +103,15 @@ func (c *DiskCatalog) GetTemplatePath(target string) ([]string, error) {
109103

110104
// Recursively walk down the Templates directory and run all
111105
// the template file checks
112-
matches, err := c.findDirectoryMatches(absPath, processed)
106+
matches, err := c.findDirectoryMatches(target, processed)
113107
if err != nil {
114108
return nil, errors.Wrap(err, "could not find directory matches")
115109
}
110+
116111
if len(matches) == 0 {
117-
return nil, errors.Errorf("no templates found in path %s", absPath)
112+
return nil, fmt.Errorf("%w in path %q", ErrNoTemplatesFound, target)
118113
}
114+
119115
return matches, nil
120116
}
121117

@@ -135,79 +131,37 @@ func (c *DiskCatalog) convertPathToAbsolute(t string) (string, error) {
135131

136132
// findGlobPathMatches returns the matched files from a glob path
137133
func (c *DiskCatalog) findGlobPathMatches(absPath string, processed map[string]struct{}) ([]string, error) {
138-
// to support globbing on old paths we use brute force to find matches with exit on first match
139134
// trim templateDir if any
140135
relPath := strings.TrimPrefix(absPath, c.templatesDirectory)
141136
// trim leading slash if any
142-
relPath = strings.TrimPrefix(relPath, string(os.PathSeparator))
143-
144-
OldPathsResolver := func(inputGlob string) []string {
145-
templateDir := c.templatesDirectory
146-
if c.templatesDirectory == "" {
147-
templateDir = "./"
148-
}
149-
150-
if c.templatesFS == nil {
151-
matches, _ := fs.Glob(os.DirFS(filepath.Join(templateDir, "http")), inputGlob)
152-
if len(matches) != 0 {
153-
return matches
154-
}
155-
156-
// condition to support network cve related globs
157-
matches, _ = fs.Glob(os.DirFS(filepath.Join(templateDir, "network")), inputGlob)
158-
return matches
159-
} else {
160-
sub, err := fs.Sub(c.templatesFS, filepath.Join(templateDir, "http"))
161-
if err != nil {
162-
return nil
163-
}
164-
matches, _ := fs.Glob(sub, inputGlob)
165-
if len(matches) != 0 {
166-
return matches
167-
}
168-
169-
// condition to support network cve related globs
170-
sub, err = fs.Sub(c.templatesFS, filepath.Join(templateDir, "network"))
171-
if err != nil {
172-
return nil
173-
}
174-
matches, _ = fs.Glob(sub, inputGlob)
175-
return matches
176-
}
137+
if c.templatesFS != nil {
138+
// fs.FS always uses forward slashes
139+
relPath = strings.TrimPrefix(relPath, "/")
140+
} else {
141+
relPath = strings.TrimPrefix(relPath, string(os.PathSeparator))
177142
}
178143

179-
var matched []string
144+
var err error
180145
var matches []string
181-
if c.templatesFS == nil {
182-
var err error
183-
matches, err = filepath.Glob(relPath)
184-
if len(matches) != 0 {
185-
matched = append(matched, matches...)
186-
} else {
187-
matched = append(matched, OldPathsResolver(relPath)...)
188-
}
189-
if err != nil && len(matched) == 0 {
190-
return nil, errors.Errorf("wildcard found, but unable to glob: %s\n", err)
191-
}
192-
} else {
193-
var err error
146+
147+
if c.templatesFS != nil {
194148
matches, err = fs.Glob(c.templatesFS, relPath)
195-
if len(matches) != 0 {
196-
matched = append(matched, matches...)
197-
} else {
198-
matched = append(matched, OldPathsResolver(relPath)...)
199-
}
200-
if err != nil && len(matched) == 0 {
201-
return nil, errors.Errorf("wildcard found, but unable to glob: %s\n", err)
202-
}
149+
} else {
150+
matches, err = filepath.Glob(absPath)
151+
}
152+
153+
if err != nil {
154+
return nil, err
203155
}
156+
204157
results := make([]string, 0, len(matches))
205158
for _, match := range matches {
206159
if _, ok := processed[match]; !ok {
207160
processed[match] = struct{}{}
208161
results = append(results, match)
209162
}
210163
}
164+
211165
return results, nil
212166
}
213167

@@ -294,8 +248,11 @@ func (c *DiskCatalog) findDirectoryMatches(absPath string, processed map[string]
294248
return results, err
295249
}
296250

297-
// PrintDeprecatedPathsMsgIfApplicable prints a warning message if any deprecated paths are found
298-
// Unless mode is silent warning message is printed
251+
// PrintDeprecatedPathsMsgIfApplicable prints a warning message if any
252+
// deprecated paths are found. Unless mode is silent warning message is printed.
253+
//
254+
// Deprecated: No longer used since the official Nuclei Templates repository
255+
// have restructured this a long time ago.
299256
func PrintDeprecatedPathsMsgIfApplicable(isSilent bool) {
300257
if !updateutils.IsOutdated("v9.4.3", config.DefaultConfig.TemplateVersion) {
301258
return

pkg/catalog/disk/path.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,9 @@ func (c *DiskCatalog) tryResolve(fullPath string) (string, error) {
7272

7373
// BackwardsCompatiblePaths returns new paths for all old/legacy template paths
7474
// Note: this is a temporary function and will be removed in the future release
75+
//
76+
// Deprecated: No longer used since the official Nuclei Templates repository
77+
// have restructured this a long time ago.
7578
func BackwardsCompatiblePaths(templateDir string, oldPath string) string {
7679
// TODO: remove this function in the future release
7780
// 1. all http related paths are now moved at path /http

pkg/protocols/common/automaticscan/util.go

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,11 @@
11
package automaticscan
22

33
import (
4+
"fmt"
5+
46
"github.com/pkg/errors"
57
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
8+
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/disk"
69
"github.com/projectdiscovery/nuclei/v3/pkg/templates"
710
"github.com/projectdiscovery/nuclei/v3/pkg/types"
811
sliceutil "github.com/projectdiscovery/utils/slice"
@@ -27,7 +30,7 @@ func getTemplateDirs(opts Options) ([]string, error) {
2730
}
2831
allTemplates = sliceutil.Dedupe(allTemplates)
2932
if len(allTemplates) == 0 {
30-
return nil, errors.New("no templates found for given input")
33+
return nil, fmt.Errorf("%w for given input", disk.ErrNoTemplatesFound)
3134
}
3235
return allTemplates, nil
3336
}

0 commit comments

Comments
 (0)