Merge pull request #105 from unhappychoice/feature/github-repo-support

feat: comprehensive loading system improvements with GitHub repository support

Author: unhappychoice

Cargo.lock

@@ -58,6 +58,8 @@ once_cell = "1.19"
 rayon = "1.8"
 open = "5.0"
 urlencoding = "2.1"
+git2 = "0.18"
+dirs = "5.0"
 
 [dev-dependencies]
 tempfile = "3.8"

Cargo.toml

@@ -28,6 +28,12 @@ pub enum GitTypeError {
 
     #[error("Walk directory error: {0}")]
     WalkDirError(#[from] walkdir::Error),
+
+    #[error("Repository clone error: {0}")]
+    RepositoryCloneError(#[from] git2::Error),
+
+    #[error("Invalid repository format: {0}")]
+    InvalidRepositoryFormat(String),
 }
 
 pub type Result<T> = std::result::Result<T, GitTypeError>;

src/error.rs

@@ -1,7 +1,16 @@
-use super::CodeChunk;
+use super::{CodeChunk, ProgressReporter};
 use crate::game::Challenge;
 use uuid::Uuid;
 
+struct NoOpProgressReporter;
+
+impl ProgressReporter for NoOpProgressReporter {
+    fn set_step(&self, _step_type: crate::game::models::loading_steps::StepType) {}
+    fn set_progress(&self, _progress: f64) {}
+    fn set_current_file(&self, _file: Option<String>) {}
+    fn set_file_counts(&self, _processed: usize, _total: usize) {}
+}
+
 pub struct ChallengeConverter;
 
 impl Default for ChallengeConverter {
@@ -27,9 +36,23 @@ impl ChallengeConverter {
     }
 
     pub fn convert_chunks_to_challenges(&self, chunks: Vec<CodeChunk>) -> Vec<Challenge> {
+        self.convert_chunks_to_challenges_with_progress(chunks, &NoOpProgressReporter)
+    }
+
+    pub fn convert_chunks_to_challenges_with_progress(
+        &self,
+        chunks: Vec<CodeChunk>,
+        progress: &dyn ProgressReporter,
+    ) -> Vec<Challenge> {
         let mut all_challenges = Vec::new();
+        let total_chunks = chunks.len();
+
+        for (chunk_index, chunk) in chunks.iter().enumerate() {
+            // Update progress
+            let chunk_progress = chunk_index as f64 / total_chunks as f64;
+            progress.set_progress(chunk_progress);
+            progress.set_file_counts(chunk_index, total_chunks);
 
-        for chunk in &chunks {
             // Generate challenges for Easy (~100), Normal (~200), Hard (~500), Wild (full chunks) only
             let difficulties = [
                 super::super::game::stage_builder::DifficultyLevel::Easy,
@@ -44,6 +67,10 @@ impl ChallengeConverter {
             }
         }
 
+        // Final progress update
+        progress.set_progress(1.0);
+        progress.set_file_counts(total_chunks, total_chunks);
+
         // Zen challenges are now handled separately in main.rs
         all_challenges
     }

src/extractor/challenge_converter.rs

@@ -4,12 +4,11 @@ pub mod git_info;
 pub mod models;
 pub mod parser;
 pub mod parsers;
-pub mod progress;
 pub mod repository_loader;
 
+pub use crate::game::screens::loading_screen::{NoOpProgressReporter, ProgressReporter};
 pub use challenge_converter::ChallengeConverter;
 pub use git_info::{GitInfoExtractor, GitRepositoryInfo};
 pub use models::{ChunkType, CodeChunk, ExtractionOptions, Language};
 pub use parser::CodeExtractor;
-pub use progress::{ConsoleProgressReporter, NoOpProgressReporter, ProgressReporter};
 pub use repository_loader::RepositoryLoader;

src/extractor/mod.rs

@@ -21,24 +21,53 @@ impl CodeExtractor {
         self.extract_chunks_with_progress(repo_path, _options, &NoOpProgressReporter)
     }
 
-    pub fn extract_chunks_with_progress<P: ProgressReporter + ?Sized>(
+    pub fn extract_chunks_with_progress<P: ProgressReporter>(
         &mut self,
         repo_path: &Path,
         _options: ExtractionOptions,
         progress: &P,
     ) -> Result<Vec<CodeChunk>> {
-        progress.set_phase("Scanning repository".to_string());
+        progress.set_step(crate::game::models::loading_steps::StepType::Scanning);
 
-        // Use ignore crate to respect .gitignore files
+        // First pass: count total files to process
+        let walker_count = WalkBuilder::new(repo_path)
+            .hidden(false) // Include hidden files
+            .git_ignore(true) // Respect .gitignore
+            .git_global(true) // Respect global gitignore
+            .git_exclude(true) // Respect .git/info/exclude
+            .build();
+
+        let mut total_files_to_process = 0;
+
+        for entry in walker_count {
+            let entry =
+                entry.map_err(|e| GitTypeError::ExtractionFailed(format!("Walk error: {}", e)))?;
+            let path = entry.path();
+
+            if !path.is_file() {
+                continue;
+            }
+
+            if let Some(extension) = path.extension().and_then(|e| e.to_str()) {
+                if let Some(_language) = Language::from_extension(extension) {
+                    if Self::should_process_file_static(path, &_options) {
+                        total_files_to_process += 1;
+                    }
+                }
+            }
+        }
+
+        // Second pass: actually collect files with proper progress
         let walker = WalkBuilder::new(repo_path)
             .hidden(false) // Include hidden files
             .git_ignore(true) // Respect .gitignore
             .git_global(true) // Respect global gitignore
             .git_exclude(true) // Respect .git/info/exclude
             .build();
 
-        // Collect all files to process first to get total count
         let mut files_to_process = Vec::new();
+        let mut processed_count = 0;
+
         for entry in walker {
             let entry =
                 entry.map_err(|e| GitTypeError::ExtractionFailed(format!("Walk error: {}", e)))?;
@@ -52,13 +81,22 @@ impl CodeExtractor {
                 if let Some(language) = Language::from_extension(extension) {
                     if Self::should_process_file_static(path, &_options) {
                         files_to_process.push((path.to_path_buf(), language));
+                        processed_count += 1;
+
+                        // Update progress with known total
+                        if processed_count % 10 == 0 || processed_count == total_files_to_process {
+                            progress.set_file_counts(processed_count, total_files_to_process);
+                        }
                     }
                 }
             }
         }
 
         let total_files = files_to_process.len();
-        progress.set_phase("Parsing AST".to_string());
+        progress.set_file_counts(total_files, total_files);
+        progress.set_progress(1.0); // Scanning completed
+
+        progress.set_step(crate::game::models::loading_steps::StepType::Extracting);
 
         // Process files in parallel with better progress tracking
         // Split files into smaller chunks for better progress visibility
@@ -68,7 +106,7 @@ impl CodeExtractor {
 
         for chunk in files_to_process.chunks(chunk_size) {
             // Process this chunk in parallel
-            let chunk_results: Result<Vec<Vec<CodeChunk>>> = chunk
+            let chunk_results: Vec<Result<Vec<CodeChunk>>> = chunk
                 .par_iter()
                 .map(|(path, language)| Self::extract_from_file_static(path, *language, &_options))
                 .collect();
@@ -77,19 +115,29 @@ impl CodeExtractor {
             processed_files += chunk.len();
             progress.set_file_counts(processed_files, total_files);
 
-            // Update spinner for each chunk to show progress
-            progress.update_spinner();
+            // Progress updates are now cheap - LoadingScreen controls rendering
 
-            // Collect results
-            let chunk_results = chunk_results?;
-            for file_chunks in chunk_results {
-                all_chunks.extend(file_chunks);
+            // Collect results, skip failed files but continue processing
+            for (i, result) in chunk_results.into_iter().enumerate() {
+                match result {
+                    Ok(file_chunks) => {
+                        all_chunks.extend(file_chunks);
+                    }
+                    Err(e) => {
+                        let file_path = &chunk[i].0;
+                        eprintln!(
+                            "Warning: Failed to extract from file {:?}: {}",
+                            file_path, e
+                        );
+                        // Continue processing other files instead of crashing
+                    }
+                }
             }
         }
 
         progress.set_file_counts(total_files, total_files);
         progress.set_current_file(None);
-        progress.set_phase("Finalizing".to_string());
+        progress.set_step(crate::game::models::loading_steps::StepType::Finalizing);
 
         Ok(all_chunks)
     }

src/extractor/parser.rs

@@ -36,7 +36,7 @@ impl RepositoryLoader {
         )
     }
 
-    pub fn load_challenges_from_repository_with_progress<P: ProgressReporter + ?Sized>(
+    pub fn load_challenges_from_repository_with_progress<P: ProgressReporter>(
         &mut self,
         repo_path: &Path,
         options: Option<ExtractionOptions>,
@@ -58,15 +58,17 @@ impl RepositoryLoader {
             return Err(GitTypeError::NoSupportedFiles);
         }
 
-        progress.set_phase("Generating challenges".to_string());
+        progress.set_step(crate::game::models::loading_steps::StepType::Generating);
         // Expand chunks into multiple challenges across difficulties
-        let challenges = self.converter.convert_chunks_to_challenges(chunks);
+        let challenges = self
+            .converter
+            .convert_chunks_to_challenges_with_progress(chunks, progress);
 
-        progress.set_phase("Finalizing".to_string());
+        progress.set_step(crate::game::models::loading_steps::StepType::Finalizing);
         Ok(challenges)
     }
 
-    pub fn load_challenges_with_difficulty<P: ProgressReporter + ?Sized>(
+    pub fn load_challenges_with_difficulty<P: ProgressReporter>(
         &mut self,
         repo_path: &Path,
         options: Option<ExtractionOptions>,
@@ -81,10 +83,10 @@ impl RepositoryLoader {
 
         match difficulty {
             DifficultyLevel::Zen => {
-                progress.set_phase("Loading whole files".to_string());
+                progress.set_step(crate::game::models::loading_steps::StepType::Generating);
                 let file_paths = self.collect_source_files(repo_path)?;
                 let challenges = self.converter.convert_whole_files_to_challenges(file_paths);
-                progress.set_phase("Finalizing".to_string());
+                progress.set_step(crate::game::models::loading_steps::StepType::Finalizing);
                 Ok(challenges)
             }
             _ => {
@@ -99,12 +101,12 @@ impl RepositoryLoader {
                     return Err(GitTypeError::NoSupportedFiles);
                 }
 
-                progress.set_phase("Generating challenges".to_string());
+                progress.set_step(crate::game::models::loading_steps::StepType::Generating);
                 let challenges = self
                     .converter
                     .convert_with_difficulty_split(chunks, difficulty);
 
-                progress.set_phase("Finalizing".to_string());
+                progress.set_step(crate::game::models::loading_steps::StepType::Finalizing);
                 Ok(challenges)
             }
         }