Merge pull request #106 from unhappychoice/perf/parallelize-challenge-conversion

perf(extracto(: parallelize challenge conversion and zen file processor

Author: unhappychoice

src/extractor/challenge_converter.rs

@@ -1,5 +1,10 @@
 use super::{CodeChunk, ProgressReporter};
 use crate::game::Challenge;
+use rayon::prelude::*;
+use std::sync::{
+    atomic::{AtomicUsize, Ordering},
+    Arc,
+};
 use uuid::Uuid;
 
 struct NoOpProgressReporter;
@@ -44,35 +49,40 @@ impl ChallengeConverter {
         chunks: Vec<CodeChunk>,
         progress: &dyn ProgressReporter,
     ) -> Vec<Challenge> {
-        let mut all_challenges = Vec::new();
-        let total_chunks = chunks.len();
-
-        for (chunk_index, chunk) in chunks.iter().enumerate() {
-            // Update progress
-            let chunk_progress = chunk_index as f64 / total_chunks as f64;
-            progress.set_progress(chunk_progress);
-            progress.set_file_counts(chunk_index, total_chunks);
-
-            // Generate challenges for Easy (~100), Normal (~200), Hard (~500), Wild (full chunks) only
-            let difficulties = [
-                super::super::game::stage_builder::DifficultyLevel::Easy,
-                super::super::game::stage_builder::DifficultyLevel::Normal,
-                super::super::game::stage_builder::DifficultyLevel::Hard,
-                super::super::game::stage_builder::DifficultyLevel::Wild,
-            ];
-
-            for difficulty in &difficulties {
-                let split_challenges = self.split_chunk_by_difficulty(chunk, difficulty);
-                all_challenges.extend(split_challenges);
-            }
-        }
+        // Parallelize per-chunk conversion with atomic progress updates
+        let total = chunks.len();
+        let processed = Arc::new(AtomicUsize::new(0));
+
+        let difficulties = [
+            super::super::game::stage_builder::DifficultyLevel::Easy,
+            super::super::game::stage_builder::DifficultyLevel::Normal,
+            super::super::game::stage_builder::DifficultyLevel::Hard,
+            super::super::game::stage_builder::DifficultyLevel::Wild,
+        ];
+
+        // Initialize progress bounds (0 processed yet)
+        progress.set_file_counts(0, total);
+
+        let all: Vec<Challenge> = chunks
+            .into_par_iter()
+            .flat_map(|chunk| {
+                let mut local = Vec::new();
+                for difficulty in &difficulties {
+                    let split = self.split_chunk_by_difficulty(&chunk, difficulty);
+                    local.extend(split);
+                }
 
-        // Final progress update
+                // Track progress count without calling reporter from parallel context
+                let _ = processed.fetch_add(1, Ordering::Relaxed);
+                local
+            })
+            .collect();
+
+        // Final progress update only (avoid Sync bound on ProgressReporter)
+        progress.set_file_counts(total, total);
         progress.set_progress(1.0);
-        progress.set_file_counts(total_chunks, total_chunks);
 
-        // Zen challenges are now handled separately in main.rs
-        all_challenges
+        all
     }
 
     pub fn convert_with_filter<F>(&self, chunks: Vec<CodeChunk>, filter: F) -> Vec<Challenge>
@@ -91,40 +101,35 @@ impl ChallengeConverter {
         chunks: Vec<CodeChunk>,
         difficulty: &super::super::game::stage_builder::DifficultyLevel,
     ) -> Vec<Challenge> {
-        let mut challenges = Vec::new();
-
-        for chunk in chunks {
-            let split_challenges = self.split_chunk_by_difficulty(&chunk, difficulty);
-            challenges.extend(split_challenges);
-        }
-
-        challenges
+        chunks
+            .into_par_iter()
+            .flat_map(|chunk| self.split_chunk_by_difficulty(&chunk, difficulty))
+            .collect()
     }
 
     pub fn convert_whole_files_to_challenges(
         &self,
         file_paths: Vec<std::path::PathBuf>,
     ) -> Vec<Challenge> {
         use super::super::game::stage_builder::DifficultyLevel;
-        let mut challenges = Vec::new();
-
-        for file_path in file_paths {
-            if let Ok(content) = std::fs::read_to_string(&file_path) {
+        file_paths
+            .into_par_iter()
+            .filter_map(|file_path| {
+                std::fs::read_to_string(&file_path)
+                    .ok()
+                    .map(|c| (file_path, c))
+            })
+            .map(|(file_path, content)| {
                 let id = Uuid::new_v4().to_string();
                 let language = super::Language::detect_from_path(&file_path);
                 let file_path_str = file_path.to_string_lossy().to_string();
-
                 let line_count = content.lines().count();
-                let challenge = Challenge::new(id, content)
+                Challenge::new(id, content)
                     .with_source_info(file_path_str, 1, line_count)
                     .with_language(language)
-                    .with_difficulty_level(DifficultyLevel::Zen);
-
-                challenges.push(challenge);
-            }
-        }
-
-        challenges
+                    .with_difficulty_level(DifficultyLevel::Zen)
+            })
+            .collect()
     }
 
     fn split_chunk_by_difficulty(
@@ -336,24 +341,23 @@ impl ChallengeConverter {
         &self,
         file_paths: Vec<std::path::PathBuf>,
     ) -> Vec<Challenge> {
-        let mut zen_challenges = Vec::new();
-
-        for file_path in file_paths {
-            if let Ok(content) = std::fs::read_to_string(&file_path) {
+        file_paths
+            .into_par_iter()
+            .filter_map(|file_path| {
+                std::fs::read_to_string(&file_path)
+                    .ok()
+                    .map(|c| (file_path, c))
+            })
+            .map(|(file_path, content)| {
                 let id = uuid::Uuid::new_v4().to_string();
                 let language = super::Language::detect_from_path(&file_path);
                 let file_path_str = file_path.to_string_lossy().to_string();
-
                 let line_count = content.lines().count();
-                let challenge = Challenge::new(id, content)
+                Challenge::new(id, content)
                     .with_source_info(file_path_str, 1, line_count)
                     .with_language(language)
-                    .with_difficulty_level(super::super::game::stage_builder::DifficultyLevel::Zen);
-
-                zen_challenges.push(challenge);
-            }
-        }
-
-        zen_challenges
+                    .with_difficulty_level(super::super::game::stage_builder::DifficultyLevel::Zen)
+            })
+            .collect()
     }
 }

src/extractor/core/extractor.rs

@@ -1,5 +1,5 @@
 use crate::extractor::models::{CodeChunk, Language};
-use crate::extractor::parsers::get_parser_registry;
+use crate::extractor::parsers::{get_parser_registry, parse_with_thread_local};
 use crate::{GitTypeError, Result};
 use std::fs;
 use std::path::Path;
@@ -46,10 +46,8 @@ impl CommonExtractor {
 
     pub fn extract_from_file(file_path: &Path, language: Language) -> Result<Vec<CodeChunk>> {
         let content = fs::read_to_string(file_path)?;
-        let registry = get_parser_registry();
-        let mut parser = registry.create_parser(language)?;
-
-        let tree = parser.parse(&content, None).ok_or_else(|| {
+        // Reuse per-thread parser instance for the language
+        let tree = parse_with_thread_local(language, &content).ok_or_else(|| {
             GitTypeError::ExtractionFailed(format!("Failed to parse file: {:?}", file_path))
         })?;
 

src/extractor/parser.rs

@@ -29,6 +29,18 @@ impl CodeExtractor {
     ) -> Result<Vec<CodeChunk>> {
         progress.set_step(crate::game::models::loading_steps::StepType::Scanning);
 
+        // Compile glob patterns once for faster matching
+        let include_patterns: Vec<glob::Pattern> = _options
+            .include_patterns
+            .iter()
+            .filter_map(|p| glob::Pattern::new(p).ok())
+            .collect();
+        let exclude_patterns: Vec<glob::Pattern> = _options
+            .exclude_patterns
+            .iter()
+            .filter_map(|p| glob::Pattern::new(p).ok())
+            .collect();
+
         // First pass: count total files to process
         let walker_count = WalkBuilder::new(repo_path)
             .hidden(false) // Include hidden files
@@ -50,7 +62,11 @@ impl CodeExtractor {
 
             if let Some(extension) = path.extension().and_then(|e| e.to_str()) {
                 if let Some(_language) = Language::from_extension(extension) {
-                    if Self::should_process_file_static(path, &_options) {
+                    if Self::should_process_file_compiled(
+                        path,
+                        &include_patterns,
+                        &exclude_patterns,
+                    ) {
                         total_files_to_process += 1;
                     }
                 }
@@ -79,7 +95,11 @@ impl CodeExtractor {
 
             if let Some(extension) = path.extension().and_then(|e| e.to_str()) {
                 if let Some(language) = Language::from_extension(extension) {
-                    if Self::should_process_file_static(path, &_options) {
+                    if Self::should_process_file_compiled(
+                        path,
+                        &include_patterns,
+                        &exclude_patterns,
+                    ) {
                         files_to_process.push((path.to_path_buf(), language));
                         processed_count += 1;
 
@@ -142,30 +162,33 @@ impl CodeExtractor {
         Ok(all_chunks)
     }
 
+    #[allow(dead_code)]
     fn should_process_file_static(path: &Path, _options: &ExtractionOptions) -> bool {
-        let path_str = path.to_string_lossy();
+        // Kept for backward compatibility; build patterns on the fly (slower)
+        let include_patterns: Vec<glob::Pattern> = _options
+            .include_patterns
+            .iter()
+            .filter_map(|p| glob::Pattern::new(p).ok())
+            .collect();
+        let exclude_patterns: Vec<glob::Pattern> = _options
+            .exclude_patterns
+            .iter()
+            .filter_map(|p| glob::Pattern::new(p).ok())
+            .collect();
+        Self::should_process_file_compiled(path, &include_patterns, &exclude_patterns)
+    }
 
-        // Check exclude patterns first
-        for pattern in &_options.exclude_patterns {
-            if glob::Pattern::new(pattern)
-                .map(|p| p.matches(&path_str))
-                .unwrap_or(false)
-            {
-                return false;
-            }
-        }
+    fn should_process_file_compiled(
+        path: &Path,
+        include_patterns: &[glob::Pattern],
+        exclude_patterns: &[glob::Pattern],
+    ) -> bool {
+        let path_str = path.to_string_lossy();
 
-        // Check include patterns
-        for pattern in &_options.include_patterns {
-            if glob::Pattern::new(pattern)
-                .map(|p| p.matches(&path_str))
-                .unwrap_or(false)
-            {
-                return true;
-            }
+        if exclude_patterns.iter().any(|p| p.matches(&path_str)) {
+            return false;
         }
-
-        false
+        include_patterns.iter().any(|p| p.matches(&path_str))
     }
 
     pub fn extract_from_file(

src/extractor/parsers/mod.rs

@@ -1,8 +1,9 @@
 use crate::extractor::models::{ChunkType, Language};
 use crate::{GitTypeError, Result};
 use once_cell::sync::Lazy;
+use std::cell::RefCell;
 use std::collections::HashMap;
-use tree_sitter::{Node, Parser, Query};
+use tree_sitter::{Node, Parser, Query, Tree};
 
 pub mod csharp;
 pub mod go;
@@ -165,3 +166,28 @@ static REGISTRY: Lazy<ParserRegistry> = Lazy::new(ParserRegistry::new);
 pub fn get_parser_registry() -> &'static ParserRegistry {
     &REGISTRY
 }
+
+thread_local! {
+    static TL_PARSERS: RefCell<std::collections::HashMap<Language, Parser>> = RefCell::new(std::collections::HashMap::new());
+}
+
+/// Parse source using a thread-local parser per language to avoid re-allocations.
+pub fn parse_with_thread_local(language: Language, content: &str) -> Option<Tree> {
+    TL_PARSERS.with(|cell| {
+        let mut map = cell.borrow_mut();
+        let parser = match map.get_mut(&language) {
+            Some(p) => p,
+            None => {
+                // Create and insert parser if not exists
+                match REGISTRY.create_parser(language) {
+                    Ok(p) => {
+                        map.insert(language, p);
+                        map.get_mut(&language).unwrap()
+                    }
+                    Err(_) => return None,
+                }
+            }
+        };
+        parser.parse(content, None)
+    })
+}