Merge pull request #6511 from matejsmycka/http-probing-optimizations

feat: http(s) probing optimization

Author: Mzack9999

pkg/utils/http_probe.go

@@ -2,22 +2,46 @@ package utils
 
 import (
 	"fmt"
+	"net"
 	"net/http"
 
 	"github.com/projectdiscovery/httpx/common/httpx"
 	"github.com/projectdiscovery/nuclei/v3/pkg/input/types"
 	"github.com/projectdiscovery/useragent"
+	sliceutil "github.com/projectdiscovery/utils/slice"
 )
 
-var (
-	HttpSchemes = []string{"https", "http"}
-)
+var commonHttpPorts = []string{
+	"80",
+	"8080",
+}
+var defaultHttpSchemes = []string{
+	"https",
+	"http",
+}
+var httpFirstSchemes = []string{
+	"http",
+	"https",
+}
+
+// determineSchemeOrder for the input
+func determineSchemeOrder(input string) []string {
+	// if input has port that is commonly used for HTTP, return http then https
+	if _, port, err := net.SplitHostPort(input); err == nil {
+		if sliceutil.Contains(commonHttpPorts, port) {
+			return httpFirstSchemes
+		}
+	}
+
+	return defaultHttpSchemes
+}
 
-// ProbeURL probes the scheme for a URL. first HTTPS is tried
-// and if any errors occur http is tried. If none succeeds, probing
-// is abandoned for such URLs.
+// ProbeURL probes the scheme for a URL.
+// http schemes are selected with heuristics
+// If none succeeds, probing is abandoned for such URLs.
 func ProbeURL(input string, httpxclient *httpx.HTTPX) string {
-	for _, scheme := range HttpSchemes {
+	schemes := determineSchemeOrder(input)
+	for _, scheme := range schemes {
 		formedURL := fmt.Sprintf("%s://%s", scheme, input)
 		req, err := httpxclient.NewRequest(http.MethodHead, formedURL)
 		if err != nil {
@@ -39,7 +63,7 @@ type inputLivenessChecker struct {
 	client *httpx.HTTPX
 }
 
-// ProbeURL probes the scheme for a URL. first HTTPS is tried
+// ProbeURL probes the scheme for a URL.
 func (i *inputLivenessChecker) ProbeURL(input string) (string, error) {
 	return ProbeURL(input, i.client), nil
 }

pkg/utils/http_probe_test.go

@@ -0,0 +1,37 @@
+package utils
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestDetermineSchemeOrder(t *testing.T) {
+	type testCase struct {
+		input    string
+		expected []string
+	}
+
+	tests := []testCase{
+		// No port or uncommon ports should return https first
+		{"example.com", []string{"https", "http"}},
+		{"example.com:443", []string{"https", "http"}},
+		{"127.0.0.1", []string{"https", "http"}},
+		{"[fe80::1]:443", []string{"https", "http"}},
+		// Common HTTP ports should return http first
+		{"example.com:80", []string{"http", "https"}},
+		{"example.com:8080", []string{"http", "https"}},
+		{"127.0.0.1:80", []string{"http", "https"}},
+		{"127.0.0.1:8080", []string{"http", "https"}},
+		{"fe80::1", []string{"https", "http"}},
+		{"[fe80::1]:80", []string{"http", "https"}},
+		{"[fe80::1]:8080", []string{"http", "https"}},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.input, func(t *testing.T) {
+			actual := determineSchemeOrder(tc.input)
+			require.Equal(t, tc.expected, actual)
+		})
+	}
+}