Converts puppet forge module entries into puppetfile git entries

  * This feature is useful when you want to bring all the forge modules
    in house on the local git server.  This will auto lookup
    the module's homepage url and use that as the git source.

    Sometimes the author doesn't provide the homepage url so this
    will fail and the code isn't always tagged to the version so
    some manual intervention will be required before final approval
    of conversion output.

Author: logicminds

README.md

@@ -68,6 +68,15 @@ Ignoring specific modules:
 Under specific conditions you may not wish to report on specific modules being out of date,
 to ignore a module create `.r10kignore` file in the same directory as your Puppetfile.
 
+### r10k::print_git_conversion
+This rake task will go through the puppetfile and convert forge based modules into git based modules using
+the modules't source repository and version tag.
+
+This feature is useful when you want to bring all the forge modules into git source control.  This assumes every module
+tags the release or provides a valid repo url.  We recommend to manually review
+the output provided by this task before replacing the forge based content in your puppetfile as not every module author 
+tagged a release or provided a working url.
+
 ### r10k:solve_dependencies
 
 Reads the Puppetfile in the current directory and uses the ruby 'solve' library to find

lib/ra10ke.rb

@@ -41,6 +41,7 @@ def initialize(*args)
         define_task_duplicates(*args)
         define_task_install(*args)
         define_task_validate(*args)
+        define_task_print_git_conversion(*args)
       end
     end
 

lib/ra10ke/dependencies.rb

@@ -138,6 +138,49 @@ def print_table(mods)
     end
   end
 
+  def define_task_print_git_conversion(*_args)
+    desc "Convert and print forge modules to git format"
+    task :print_git_conversion do
+      require 'ra10ke/git_repo'
+      require 'r10k/puppetfile'
+      require 'puppet_forge'
+
+      PuppetForge.user_agent = "ra10ke/#{Ra10ke::VERSION}"
+      puppetfile = get_puppetfile
+      puppetfile.load!
+      PuppetForge.host = puppetfile.forge if puppetfile.forge =~ /^http/
+
+      # ignore file allows for "don't tell me about this"
+      ignore_modules = []
+      if File.exist?('.r10kignore')
+        ignore_modules = File.readlines('.r10kignore').each {|l| l.chomp!}
+      end
+      forge_mods = puppetfile.modules.find_all { |mod| mod.instance_of?(R10K::Module::Forge) && mod.v3_module.homepage_url? }
+     
+      threads = forge_mods.map do |mod|
+       Thread.new do 
+          source_url = mod.v3_module.attributes.dig(:current_release, :metadata, :source) || mod.v3_module.homepage_url 
+          # git:// does not work with ls-remote command, convert to https
+          source_url = source_url.gsub('git://', 'https://')
+          source_url = source_url.gsub(/\Agit\@(.*)\:(.*)/) do
+            "https://#{$1}/#{$2}"
+          end
+          repo = ::Ra10ke::GitRepo.new(source_url)
+          ref = repo.get_ref_like(mod.expected_version) 
+          ref_name = ref ? ref[:name] : "bad url or tag #{mod.expected_version} is missing"
+          <<~EOF
+          mod '#{mod.name}',
+            :git => '#{source_url}',
+            :ref => '#{ref_name}'
+
+          EOF
+        end
+      end
+      output = threads.map { |th| th.join.value }
+      puts output
+    end
+  end
+
   def define_task_dependencies(*_args)
     desc "Print outdated forge modules"
     task :dependencies do

lib/ra10ke/git_repo.rb

@@ -85,8 +85,8 @@ def all_refs
       @all_refs ||= begin
         remote_refs.each_with_object([]) do |line, refs|
           sha, ref = line.split("\t")
+          next refs if line.include?('redirecting')
           next refs if sha.eql?('ref: refs/heads/master')
-
           _, type, name, subtype = ref.chomp.split('/')
           next refs unless name
 
@@ -99,6 +99,18 @@ def all_refs
       end
     end
 
+    # @summary searches all the refs for a ref similar to the name provided
+    #          This is useful when looking for tags if a v or not a v
+    # @param ref_name [String]
+    # @return [String] the matching ref_name or nil
+    def get_ref_like(ref_name)
+      return nil unless valid_url?
+      ref = all_refs.find do |ref|
+        ref[:name].include?(ref_name)
+      end
+      ref
+    end
+
     # useful for mocking easily
     # @param cmd [String]
     # @param silent [Boolean] set to true if you wish to send output to /dev/null, false by default

spec/fixtures/Puppetfile_git_conversion

@@ -0,0 +1,28 @@
+mod 'choria',
+  :git => 'https://github.com/choria-io/puppet-choria',
+  :ref => '0.26.2'
+
+mod 'inifile',
+  :git => 'https://github.com/puppetlabs/puppetlabs-inifile',
+  :ref => '2.2.0'
+
+mod 'ruby',
+  :git => 'https://github.com/puppetlabs/puppetlabs-ruby.git',
+  :ref => 'v1.0.1'
+
+mod 'stdlib',
+  :git => 'https://github.com/puppetlabs/puppetlabs-stdlib',
+  :ref => '4.24.0'
+
+mod 'concat',
+  :git => 'https://github.com/puppetlabs/puppetlabs-concat',
+  :ref => '4.0.0'
+
+mod 'ntp',
+  :git => 'https://github.com/puppetlabs/puppetlabs-ntp',
+  :ref => '6.4.1'
+
+mod 'archive',
+  :git => 'https://github.com/voxpupuli/puppet-archive',
+  :ref => 'v3.1.1'
+

spec/ra10ke/dependencies_spec.rb

@@ -2,6 +2,8 @@
 require 'r10k/puppetfile'
 require 'spec_helper'
 require 'ra10ke/dependencies'
+require 'ra10ke'
+
 RSpec::Mocks.configuration.allow_message_expectations_on_nil = true
 
 RSpec.describe 'Ra10ke::Dependencies::Verification' do
@@ -89,6 +91,18 @@
           'tags' => test_tags,
         })).to eq(latest_tag)
       end
-     end
+    end
+
+    context 'convert to ref' do
+       
+      it 'run rake task' do
+        output_conversion = File.read(File.join(fixtures_dir, 'Puppetfile_git_conversion'))
+        require 'ra10ke'
+        Ra10ke::RakeTask.new do |t|
+          t.basedir = fixtures_dir
+        end
+        expect{Rake::Task['r10k:print_git_conversion'].invoke}.to output(output_conversion).to_stdout
+      end
+    end
   end
 end

spec/ra10ke/git_repo_spec.rb

@@ -38,6 +38,10 @@
       expect(instance.remote_refs.first).to eq("1b3322d525e96bf7d0565b08703e2a44c90e7b4a\tHEAD\n")
     end
 
+    it '#get_ref_like' do
+      expect(instance.get_ref_like('8.0.0'))
+    end
+
     it '#valid_ref?' do
       expect(instance.valid_ref?('master')).to be true
     end