Merge pull request #76 from voxpupuli/git-converter

Converts puppet forge module entries into puppetfile git entries

Author: logicminds

README.md

@@ -68,6 +68,15 @@ Ignoring specific modules:
 Under specific conditions you may not wish to report on specific modules being out of date,
 to ignore a module create `.r10kignore` file in the same directory as your Puppetfile.
 
+### r10k::print_git_conversion
+This rake task will go through the puppetfile and convert forge based modules into git based modules using
+the modules't source repository and version tag.
+
+This feature is useful when you want to bring all the forge modules into git source control.  This assumes every module
+tags the release or provides a valid repo url.  We recommend to manually review
+the output provided by this task before replacing the forge based content in your puppetfile as not every module author 
+tagged a release or provided a working url.
+
 ### r10k:solve_dependencies
 
 Reads the Puppetfile in the current directory and uses the ruby 'solve' library to find

lib/ra10ke.rb

@@ -41,6 +41,7 @@ def initialize(*args)
         define_task_duplicates(*args)
         define_task_install(*args)
         define_task_validate(*args)
+        define_task_print_git_conversion(*args)
       end
     end
 

lib/ra10ke/dependencies.rb

@@ -138,6 +138,49 @@ def print_table(mods)
     end
   end
 
+  def define_task_print_git_conversion(*_args)
+    desc "Convert and print forge modules to git format"
+    task :print_git_conversion do
+      require 'ra10ke/git_repo'
+      require 'r10k/puppetfile'
+      require 'puppet_forge'
+
+      PuppetForge.user_agent = "ra10ke/#{Ra10ke::VERSION}"
+      puppetfile = get_puppetfile
+      puppetfile.load!
+      PuppetForge.host = puppetfile.forge if puppetfile.forge =~ /^http/
+
+      # ignore file allows for "don't tell me about this"
+      ignore_modules = []
+      if File.exist?('.r10kignore')
+        ignore_modules = File.readlines('.r10kignore').each {|l| l.chomp!}
+      end
+      forge_mods = puppetfile.modules.find_all { |mod| mod.instance_of?(R10K::Module::Forge) && mod.v3_module.homepage_url? }
+     
+      threads = forge_mods.map do |mod|
+       Thread.new do 
+          source_url = mod.v3_module.attributes.dig(:current_release, :metadata, :source) || mod.v3_module.homepage_url 
+          # git:// does not work with ls-remote command, convert to https
+          source_url = source_url.gsub('git://', 'https://')
+          source_url = source_url.gsub(/\Agit\@(.*)\:(.*)/) do
+            "https://#{$1}/#{$2}"
+          end
+          repo = ::Ra10ke::GitRepo.new(source_url)
+          ref = repo.get_ref_like(mod.expected_version) 
+          ref_name = ref ? ref[:name] : "bad url or tag #{mod.expected_version} is missing"
+          <<~EOF
+          mod '#{mod.name}',
+            :git => '#{source_url}',
+            :ref => '#{ref_name}'
+
+          EOF
+        end
+      end
+      output = threads.map { |th| th.join.value }
+      puts output
+    end
+  end
+
   def define_task_dependencies(*_args)
     desc "Print outdated forge modules"
     task :dependencies do

lib/ra10ke/git_repo.rb

@@ -85,8 +85,8 @@ def all_refs
       @all_refs ||= begin
         remote_refs.each_with_object([]) do |line, refs|
           sha, ref = line.split("\t")
+          next refs if line.include?('redirecting')
           next refs if sha.eql?('ref: refs/heads/master')
-
           _, type, name, subtype = ref.chomp.split('/')
           next refs unless name
 
@@ -99,6 +99,18 @@ def all_refs
       end
     end
 
+    # @summary searches all the refs for a ref similar to the name provided
+    #          This is useful when looking for tags if a v or not a v
+    # @param ref_name [String]
+    # @return [String] the matching ref_name or nil
+    def get_ref_like(ref_name)
+      return nil unless valid_url?
+      ref = all_refs.find do |ref|
+        ref[:name].include?(ref_name)
+      end
+      ref
+    end
+
     # useful for mocking easily
     # @param cmd [String]
     # @param silent [Boolean] set to true if you wish to send output to /dev/null, false by default

spec/fixtures/Puppetfile_git_conversion

@@ -0,0 +1,28 @@
+mod 'choria',
+  :git => 'https://github.com/choria-io/puppet-choria',
+  :ref => '0.26.2'
+
+mod 'inifile',
+  :git => 'https://github.com/puppetlabs/puppetlabs-inifile',
+  :ref => '2.2.0'
+
+mod 'ruby',
+  :git => 'https://github.com/puppetlabs/puppetlabs-ruby.git',
+  :ref => 'v1.0.1'
+
+mod 'stdlib',
+  :git => 'https://github.com/puppetlabs/puppetlabs-stdlib',
+  :ref => '4.24.0'
+
+mod 'concat',
+  :git => 'https://github.com/puppetlabs/puppetlabs-concat',
+  :ref => '4.0.0'
+
+mod 'ntp',
+  :git => 'https://github.com/puppetlabs/puppetlabs-ntp',
+  :ref => '6.4.1'
+
+mod 'archive',
+  :git => 'https://github.com/voxpupuli/puppet-archive',
+  :ref => 'v3.1.1'
+

spec/ra10ke/dependencies_spec.rb

@@ -2,6 +2,8 @@
 require 'r10k/puppetfile'
 require 'spec_helper'
 require 'ra10ke/dependencies'
+require 'ra10ke'
+
 RSpec::Mocks.configuration.allow_message_expectations_on_nil = true
 
 RSpec.describe 'Ra10ke::Dependencies::Verification' do
@@ -89,6 +91,18 @@
           'tags' => test_tags,
         })).to eq(latest_tag)
       end
-     end
+    end
+
+    context 'convert to ref' do
+       
+      it 'run rake task' do
+        output_conversion = File.read(File.join(fixtures_dir, 'Puppetfile_git_conversion'))
+        require 'ra10ke'
+        Ra10ke::RakeTask.new do |t|
+          t.basedir = fixtures_dir
+        end
+        expect{Rake::Task['r10k:print_git_conversion'].invoke}.to output(output_conversion).to_stdout
+      end
+    end
   end
 end

spec/ra10ke/git_repo_spec.rb

@@ -38,6 +38,10 @@
       expect(instance.remote_refs.first).to eq("1b3322d525e96bf7d0565b08703e2a44c90e7b4a\tHEAD\n")
     end
 
+    it '#get_ref_like' do
+      expect(instance.get_ref_like('8.0.0'))
+    end
+
     it '#valid_ref?' do
       expect(instance.valid_ref?('master')).to be true
     end