fix(trackers): paginate gitea to find all issues when searching for duplicates (#6707)

* (fix) paginate to find all issues when searching for duplicates

* (feat) add configurable limits for perpage and total pages

Author: leonjza

cmd/nuclei/issue-tracker-config.yaml

@@ -81,6 +81,10 @@
 #    severity: low
 #  # duplicate-issue-check (optional) flag to enable duplicate tracking issue check
 #  duplicate-issue-check: false
+#  # duplicate-issue-page-size (optional) controls how many issues to fetch per page when searching for duplicates
+#  duplicate-issue-page-size: 100
+#  # duplicate-issue-max-pages (optional) limits how many pages to fetch when searching for duplicates (0 = no limit)
+#  duplicate-issue-max-pages: 0
 #
 # Jira contains configuration options for Jira issue tracker
 #jira:
@@ -118,10 +122,10 @@
 # status-not: Closed
 #  # Customfield supports name, id and freeform. name and id are to be used when the custom field is a dropdown.
 #  # freeform can be used if the custom field is just a text entry
-#  # Variables can be used to pull various pieces of data from the finding itself. 
+#  # Variables can be used to pull various pieces of data from the finding itself.
 #  # Supported variables: $CVSSMetrics, $CVEID, $CWEID, $Host, $Severity, $CVSSScore, $Name
 # custom-fields:
-#  customfield_00001: 
+#  customfield_00001:
 #    name: "Nuclei"
 #  customfield_00002:
 #    freeform: $CVSSMetrics
@@ -172,4 +176,4 @@
 #  omit-raw: false
 #  # determines the number of results to be kept in memory before writing it to the database or 0 to
 #  # persist all in memory and write all results at the end (default)
-#  batch-size: 0
+#  batch-size: 0

pkg/reporting/trackers/gitea/gitea.go

@@ -42,6 +42,12 @@ type Options struct {
 	DenyList *filters.Filter `yaml:"deny-list"`
 	// DuplicateIssueCheck is a bool to enable duplicate tracking issue check and update the newest
 	DuplicateIssueCheck bool `yaml:"duplicate-issue-check" default:"false"`
+	// DuplicateIssuePageSize controls how many issues are fetched per page when searching for duplicates.
+	// If unset or <=0, a default of 100 is used.
+	DuplicateIssuePageSize int `yaml:"duplicate-issue-page-size" default:"100"`
+	// DuplicateIssueMaxPages limits how many pages are fetched when searching for duplicates.
+	// If unset or <=0, all pages are fetched until exhaustion.
+	DuplicateIssueMaxPages int `yaml:"duplicate-issue-max-pages" default:"0"`
 
 	HttpClient *retryablehttp.Client `yaml:"-"`
 	OmitRaw    bool                  `yaml:"-"`
@@ -151,21 +157,44 @@ func (i *Integration) ShouldFilter(event *output.ResultEvent) bool {
 }
 
 func (i *Integration) findIssueByTitle(title string) (*gitea.Issue, error) {
+	// Fetch issues in pages to ensure older issues are also checked for duplicates.
+	pageSize := i.options.DuplicateIssuePageSize
+	if pageSize <= 0 {
+		pageSize = 100
+	}
+	maxPages := i.options.DuplicateIssueMaxPages
 
-	issueList, _, err := i.client.ListRepoIssues(i.options.ProjectOwner, i.options.ProjectName, gitea.ListIssueOption{
+	opts := gitea.ListIssueOption{
 		State: "all",
-	})
-	if err != nil {
-		return nil, err
+		ListOptions: gitea.ListOptions{
+			Page:     1,
+			PageSize: pageSize,
+		},
 	}
 
-	for _, issue := range issueList {
-		if issue.Title == title {
-			return issue, nil
+	for {
+		if maxPages > 0 && opts.Page > maxPages {
+			return nil, nil
 		}
-	}
 
-	return nil, nil
+		issueList, _, err := i.client.ListRepoIssues(i.options.ProjectOwner, i.options.ProjectName, opts)
+		if err != nil {
+			return nil, err
+		}
+
+		for _, issue := range issueList {
+			if issue.Title == title {
+				return issue, nil
+			}
+		}
+
+		if len(issueList) < opts.PageSize {
+			// Last page reached.
+			return nil, nil
+		}
+
+		opts.Page++
+	}
 }
 
 func (i *Integration) getLabelIDsByNames(labels []string) ([]int64, error) {