Reduce string allocations (#687)

* reduce string allocations

* Pre-parse remap rules per run

* fix logging in unknownRemapLanguage

* microbenchmark for DetermineLanguage

* microbenchmarks for remapping functions

Author: qwerty8811

processor/detector.go

@@ -3,6 +3,7 @@
 package processor
 
 import (
+	"bytes"
 	"cmp"
 	"errors"
 	"slices"
@@ -166,11 +167,9 @@ func DetermineLanguage(filename string, fallbackLanguage string, possibleLanguag
 
 	startTime := makeTimestampNano()
 
-	var toCheck string
+	toCheck := content
 	if len(content) > 20_000 {
-		toCheck = string(content)[:20_000]
-	} else {
-		toCheck = string(content)
+		toCheck = content[:20_000]
 	}
 
 	primary := ""
@@ -182,8 +181,8 @@ func DetermineLanguage(filename string, fallbackLanguage string, possibleLanguag
 		LanguageFeaturesMutex.Unlock()
 
 		count := 0
-		for _, key := range langFeatures.Keywords {
-			if strings.Contains(toCheck, key) {
+		for _, key := range langFeatures.KeywordBytes {
+			if bytes.Contains(toCheck, key) {
 				count++
 			}
 		}

processor/detector_test.go

@@ -3,6 +3,7 @@
 package processor
 
 import (
+	"strings"
 	"testing"
 )
 
@@ -418,3 +419,34 @@ func BenchmarkScanSheBangReal(b *testing.B) {
 		_, _ = scanForSheBang([]byte("#!  /usr/bin/env   perl   -w"))
 	}
 }
+
+func BenchmarkDetermineLanguage(b *testing.B) {
+	ProcessConstants()
+
+	coqContent := []byte("Require Hypothesis Inductive\n")
+	systemVerilogContent := []byte("endmodule posedge edge always wire\n")
+	largeCoqContent := []byte("Require Hypothesis Inductive\n" + strings.Repeat("x", 25_000))
+	largeSystemVerilogContent := []byte("endmodule posedge edge always wire\n" + strings.Repeat("y", 25_000))
+	possibleLanguages := []string{"Coq", "SystemVerilog"}
+
+	benchmarks := []struct {
+		name    string
+		content []byte
+	}{
+		{name: "small_coq", content: coqContent},
+		{name: "small_systemverilog", content: systemVerilogContent},
+		{name: "large_coq_over_cutoff", content: largeCoqContent},
+		{name: "large_systemverilog_over_cutoff", content: largeSystemVerilogContent},
+	}
+
+	for _, benchmark := range benchmarks {
+		b.Run(benchmark.name, func(b *testing.B) {
+			b.ReportAllocs()
+			b.SetBytes(int64(len(benchmark.content)))
+
+			for i := 0; i < b.N; i++ {
+				_ = DetermineLanguage("", "", possibleLanguages, benchmark.content)
+			}
+		})
+	}
+}

processor/processor.go

@@ -145,6 +145,43 @@ var RemapUnknown = ""
 // RemapAll allows remapping of all files with a string to search the content for
 var RemapAll = ""
 
+type remapRule struct {
+	pattern  []byte
+	language string
+}
+
+type remapConfig struct {
+	all     []remapRule
+	unknown []remapRule
+}
+
+type processorContext struct {
+	remap remapConfig
+}
+
+func parseRemapRules(value string) []remapRule {
+	rules := []remapRule{}
+
+	for s := range strings.SplitSeq(value, ",") {
+		t := strings.Split(s, ":")
+		if len(t) == 2 {
+			rules = append(rules, remapRule{
+				pattern:  []byte(t[0]),
+				language: t[1],
+			})
+		}
+	}
+
+	return rules
+}
+
+func newRemapConfig(remapAll string, remapUnknown string) remapConfig {
+	return remapConfig{
+		all:     parseRemapRules(remapAll),
+		unknown: parseRemapRules(remapUnknown),
+	}
+}
+
 // CurrencySymbol allows setting the currency symbol for cocomo project cost estimation
 var CurrencySymbol = ""
 
@@ -425,6 +462,7 @@ func processLanguageFeature(name string, value Language) {
 	mlCommentTrie := &Trie{}
 	stringTrie := &Trie{}
 	tokenTrie := &Trie{}
+	keywordBytes := make([][]byte, 0, len(value.Keywords))
 
 	complexityMask := byte(0)
 	singleLineCommentMask := byte(0)
@@ -464,6 +502,10 @@ func processLanguageFeature(name string, value Language) {
 	}
 	processMask |= stringMask
 
+	for _, v := range value.Keywords {
+		keywordBytes = append(keywordBytes, []byte(v))
+	}
+
 	LanguageFeaturesMutex.Lock()
 	LanguageFeatures[name] = LanguageFeature{
 		Complexity:            complexityTrie,
@@ -480,6 +522,7 @@ func processLanguageFeature(name string, value Language) {
 		StringCheckMask:       stringMask,
 		ProcessMask:           processMask,
 		Keywords:              value.Keywords,
+		KeywordBytes:          keywordBytes,
 		Quotes:                value.Quotes,
 	}
 	LanguageFeaturesMutex.Unlock()
@@ -608,6 +651,7 @@ func Process() {
 	}
 
 	SortBy = strings.ToLower(SortBy)
+	ctx := processorContext{remap: newRemapConfig(RemapAll, RemapUnknown)}
 
 	printDebugF("NumCPU: %d", runtime.NumCPU())
 	printDebugF("SortBy: %s", SortBy)
@@ -692,7 +736,7 @@ func Process() {
 		close(fileListQueue)
 	}()
 
-	go fileProcessorWorker(fileListQueue, fileSummaryJobQueue)
+	go ctx.fileProcessorWorker(fileListQueue, fileSummaryJobQueue)
 
 	result := fileSummarize(fileSummaryJobQueue)
 	if FileOutput == "" {

processor/result.go

@@ -42,6 +42,7 @@ func ProcessResult() ([]LanguageSummary, error) {
 	}
 
 	SortBy = strings.ToLower(SortBy)
+	ctx := processorContext{remap: newRemapConfig(RemapAll, RemapUnknown)}
 
 	printDebugF("NumCPU: %d", runtime.NumCPU())
 	printDebugF("SortBy: %s", SortBy)
@@ -126,7 +127,7 @@ func ProcessResult() ([]LanguageSummary, error) {
 		close(fileListQueue)
 	}()
 
-	go fileProcessorWorker(fileListQueue, fileSummaryJobQueue)
+	go ctx.fileProcessorWorker(fileListQueue, fileSummaryJobQueue)
 
 	language := aggregateLanguageSummary(fileSummaryJobQueue)
 	language = sortLanguageSummary(language)

processor/structs.go

@@ -65,6 +65,7 @@ type LanguageFeature struct {
 	StringCheckMask       byte
 	ProcessMask           byte
 	Keywords              []string
+	KeywordBytes          [][]byte
 	Quotes                []Quote
 }
 

processor/workers.go

@@ -688,7 +688,7 @@ func checkBomSkip(fileJob *FileJob) int {
 
 // Reads and processes files from input chan in parallel, and sends results to
 // output chan
-func fileProcessorWorker(input chan *FileJob, output chan *FileJob) {
+func (ctx processorContext) fileProcessorWorker(input chan *FileJob, output chan *FileJob) {
 	var startTime int64
 	var fileCount int64
 	var gcEnabled int64
@@ -721,7 +721,7 @@ func fileProcessorWorker(input chan *FileJob, output chan *FileJob) {
 
 				if err == nil {
 					job.Content = content
-					if processFile(job) {
+					if ctx.processFile(job) {
 						output <- job
 					}
 				} else {
@@ -743,7 +743,7 @@ func fileProcessorWorker(input chan *FileJob, output chan *FileJob) {
 
 // Process a single file
 // File must have been read to job.Content already
-func processFile(job *FileJob) bool {
+func (ctx processorContext) processFile(job *FileJob) bool {
 	fileStartTime := makeTimestampNano()
 
 	contents := job.Content
@@ -752,14 +752,14 @@ func processFile(job *FileJob) bool {
 	job.Language = DetermineLanguage(job.Filename, job.Language, job.PossibleLanguages, job.Content)
 
 	remapped := false
-	if RemapAll != "" {
-		hardRemapLanguage(job)
+	if len(ctx.remap.all) != 0 {
+		ctx.hardRemapLanguage(job)
 	}
 
 	// If the type is #! we should check to see if we can identify
 	if job.Language == SheBang {
-		if RemapUnknown != "" {
-			remapped = unknownRemapLanguage(job)
+		if len(ctx.remap.unknown) != 0 {
+			remapped = ctx.unknownRemapLanguage(job)
 		}
 
 		// if we didn't remap we then want to see if it's a #! map
@@ -835,36 +835,30 @@ func processFile(job *FileJob) bool {
 	return true
 }
 
-func hardRemapLanguage(job *FileJob) bool {
+func (ctx processorContext) hardRemapLanguage(job *FileJob) bool {
 	remapped := false
-	for s := range strings.SplitSeq(RemapAll, ",") {
-		t := strings.Split(s, ":")
-		if len(t) == 2 {
-			cutoff := min(1000, len(job.Content)) // at most 1000 bytes into the file to look
-
-			if strings.Contains(string(job.Content[:cutoff]), t[0]) {
-				job.Language = t[1]
-				remapped = true
-				printWarnF("hard remapping: %s to %s", job.Location, job.Language)
-			}
+	cutoff := min(1000, len(job.Content)) // at most 1000 bytes into the file to look
+
+	for _, rule := range ctx.remap.all {
+		if bytes.Contains(job.Content[:cutoff], rule.pattern) {
+			job.Language = rule.language
+			remapped = true
+			printWarnF("hard remapping: %s to %s", job.Location, job.Language)
 		}
 	}
 
 	return remapped
 }
 
-func unknownRemapLanguage(job *FileJob) bool {
+func (ctx processorContext) unknownRemapLanguage(job *FileJob) bool {
 	remapped := false
-	for s := range strings.SplitSeq(RemapUnknown, ",") {
-		t := strings.Split(s, ":")
-		if len(t) == 2 {
-			cutoff := min(1000, len(job.Content)) // at most 1000 bytes into the file to look
-
-			if strings.Contains(string(job.Content[:cutoff]), t[0]) {
-				printWarnF("unknown remapping: %s to %s", job.Location, job.Language)
-				job.Language = t[1]
-				remapped = true
-			}
+	cutoff := min(1000, len(job.Content)) // at most 1000 bytes into the file to look
+
+	for _, rule := range ctx.remap.unknown {
+		if bytes.Contains(job.Content[:cutoff], rule.pattern) {
+			job.Language = rule.language
+			remapped = true
+			printWarnF("unknown remapping: %s to %s", job.Location, job.Language)
 		}
 	}