fix(headless): merge extra headers (#6376)

* headless: fix extra headers overwrite

* headless: set Accept-Language when no custom headers

Author: ysokolovsky

pkg/protocols/headless/engine/engine.go

@@ -20,12 +20,13 @@ import (
 
 // Browser is a browser structure for nuclei headless module
 type Browser struct {
-	customAgent  string
-	tempDir      string
-	previousPIDs map[int32]struct{} // track already running PIDs
-	engine       *rod.Browser
-	options      *types.Options
-	launcher     *launcher.Launcher
+	customAgent    string
+	defaultHeaders map[string]string
+	tempDir        string
+	previousPIDs   map[int32]struct{} // track already running PIDs
+	engine         *rod.Browser
+	options        *types.Options
+	launcher       *launcher.Launcher
 
 	// use getHTTPClient to get the http client
 	httpClient     *http.Client
@@ -95,6 +96,7 @@ func New(options *types.Options) (*Browser, error) {
 	if browserErr := browser.Connect(); browserErr != nil {
 		return nil, browserErr
 	}
+	defaultHeaders := make(map[string]string)
 	customAgent := ""
 	for _, option := range options.CustomHeaders {
 		parts := strings.SplitN(option, ":", 2)
@@ -103,12 +105,20 @@ func New(options *types.Options) (*Browser, error) {
 		}
 		if strings.EqualFold(parts[0], "User-Agent") {
 			customAgent = parts[1]
+		} else {
+			k := strings.TrimSpace(parts[0])
+			v := strings.TrimSpace(parts[1])
+			if k == "" || v == "" {
+				continue
+			}
+			defaultHeaders[k] = v
 		}
 	}
 
 	engine := &Browser{
 		tempDir:        dataStore,
 		customAgent:    customAgent,
+		defaultHeaders: defaultHeaders,
 		engine:         browser,
 		options:        options,
 		httpClientOnce: &sync.Once{},
@@ -135,6 +145,30 @@ func (b *Browser) UserAgent() string {
 	return b.customAgent
 }
 
+// applyDefaultHeaders setsheaders passed via cli -H flag
+func (b *Browser) applyDefaultHeaders(p *rod.Page) error {
+	pairs := make([]string, 0, len(b.defaultHeaders)*2+2)
+
+	hasAcceptLanguage := false
+	for k := range b.defaultHeaders {
+		if strings.EqualFold(k, "Accept-Language") {
+			hasAcceptLanguage = true
+			break
+		}
+	}
+	if !hasAcceptLanguage {
+		pairs = append(pairs, "Accept-Language", "en, en-GB, en-us;")
+	}
+	for k, v := range b.defaultHeaders {
+		pairs = append(pairs, k, v)
+	}
+	if len(pairs) == 0 {
+		return nil
+	}
+	_, err := p.SetExtraHeaders(pairs)
+	return err
+}
+
 func (b *Browser) getHTTPClient() (*http.Client, error) {
 	var err error
 	b.httpClientOnce.Do(func() {

pkg/protocols/headless/engine/page.go

@@ -61,6 +61,10 @@ func (i *Instance) Run(ctx *contextargs.Context, actions []*Action, payloads map
 	}
 	page = page.Timeout(options.Timeout)
 
+	if err = i.browser.applyDefaultHeaders(page); err != nil {
+		return nil, nil, err
+	}
+
 	if i.browser.customAgent != "" {
 		if userAgentErr := page.SetUserAgent(&proto.NetworkSetUserAgentOverride{UserAgent: i.browser.customAgent}); userAgentErr != nil {
 			return nil, nil, userAgentErr
@@ -130,10 +134,6 @@ func (i *Instance) Run(ctx *contextargs.Context, actions []*Action, payloads map
 		return nil, nil, err
 	}
 
-	if _, err := page.SetExtraHeaders([]string{"Accept-Language", "en, en-GB, en-us;"}); err != nil {
-		return nil, nil, err
-	}
-
 	// inject cookies
 	// each http request is performed via the native go http client
 	// we first inject the shared cookies