diff --git a/bolt.gemspec b/bolt.gemspec index d476f3dcc5..dff76291b2 100644 --- a/bolt.gemspec +++ b/bolt.gemspec @@ -57,7 +57,7 @@ Gem::Specification.new do |spec| spec.add_dependency "net-ssh-krb", "~> 0.5" spec.add_dependency "orchestrator_client", "~> 0.5" spec.add_dependency "puppet", ">= 6.18.0" - spec.add_dependency "puppetfile-resolver", "~> 0.5" + spec.add_dependency "puppetfile-resolver", ">= 0.6.2", "< 1.0" spec.add_dependency "puppet-resource_api", ">= 1.8.1" spec.add_dependency "puppet-strings", "~> 2.3" spec.add_dependency "r10k", "~> 3.10" diff --git a/documentation/bolt_installing_modules.md b/documentation/bolt_installing_modules.md index 32b9db5d72..0df925b871 100644 --- a/documentation/bolt_installing_modules.md +++ b/documentation/bolt_installing_modules.md @@ -10,9 +10,9 @@ You can use the command line to: downloaded a project from source control and want to install its dependencies. - Update your project's modules. -If you need to install a module from a GitHub repository or an alternate Forge, -or you need to use a Forge proxy, you can manually configure your Bolt modules -in your Bolt project configuration file (`bolt-project.yaml`). +If you need to install a module from a git repository or an alternate Forge, or +you need to use a Forge proxy, you can manually configure your Bolt modules in +your Bolt project configuration file (`bolt-project.yaml`). ## Create a Bolt project with pre-installed modules @@ -216,7 +216,7 @@ To specify a git module, use the following keys in the specification: | Key | Description | Required | | --- | --- | :-: | -| `git` | The URI to the GitHub repository. URI must begin with either `https://github.com`or `git@github.com`. | ✓ | +| `git` | The URI to the git repository. URI must begin with either `git@`, `http://`, or `https://`. | ✓ | | `name` | The name of the module. Bolt uses this name for the module in the Puppetfile, the directory that the module's contents are downloaded to, and as a namespace for the module's content. To avoid errors, make sure this name matches the name specified in the module's `metadata.json`. **Required if `resolve` is `false`.** | | | `ref` | The git reference to checkout. Can be either a branch, commit, or tag. | ✓ | | `resolve` | Boolean. Whether to resolve the module's dependencies when installing modules. | | @@ -233,8 +233,6 @@ modules: ref: '7.0.0' ``` -Bolt only supports installing git modules from GitHub. - ## Pin a module version If you need to pin a module in your Bolt project to a specific version, you can @@ -364,8 +362,6 @@ resolve dependencies for. You might want to skip dependency resolution for a module if: - The module has outdated or incorrect metadata. -- The module is a git module hosted in a repository other than a public GitHub - repository. For example, a private GitHub repository. - Bolt can't cleanly resolve the module's dependencies. You can configure Bolt to skip dependency resolution for a module by setting the @@ -378,8 +374,8 @@ dependencies, Bolt generates a Puppetfile with the resolved modules and dependencies, as well as the modules it did not resolve dependencies for. For example, if your project includes the `puppetlabs/ntp` Forge module and a -git module named `private_module` hosted in a private GitHub repository, you can -configure Bolt to skip dependency resolution for `private_module`: +git module named `private_module` that has incorrect metadata, you can configure +Bolt to skip dependency resolution for `private_module`: ```yaml # bolt-project.yaml @@ -453,11 +449,6 @@ without resolving dependencies. The process for manually managing your modules uses the new `module` subcommand, and replaces the now deprecated `puppetfile` subcommand. -The most common scenario where Bolt can't resolve module dependencies is when -a project includes git modules that are in a repository other than a public -GitHub repository. If your project includes this type of module, you must -manually manage your project's Puppetfile. - To manually manage a project's Puppetfile and install modules without resolving dependencies, follow these steps: @@ -466,10 +457,9 @@ dependencies, follow these steps: the Puppetfile. For example: ```ruby - # Modules from a private git repository - mod 'private-module', git: 'https://github.com/bolt-user/private-module.git', ref: 'main' - - # Modules from an alternate Forge + mod 'private-module', + git: 'https://github.com/bolt-user/private-module.git', + ref: 'main' mod 'puppetlabs/apache', '5.7.0' mod 'puppetlabs/stdlib', '6.5.0' mod 'puppetlabs/concat', '6.3.0' @@ -505,9 +495,7 @@ project](projects.md#migrate-a-bolt-project). In some cases, Bolt is unable to resolve module dependencies and manage your project's modules for you. If Bolt can't resolve your module dependencies, you can manage your project's Puppetfile manually and use Bolt to install the -modules listed in the Puppetfile without resolving dependencies. The most common -scenario where Bolt can't resolve module dependencies is when a project includes -git modules that are in a repository other than a public GitHub repository. +modules listed in the Puppetfile without resolving dependencies. The module management feature makes changes to configuration files and changes the directory where modules are installed. To migrate your project, do the diff --git a/lib/bolt/module_installer.rb b/lib/bolt/module_installer.rb index 6950a2cff7..da549273f2 100644 --- a/lib/bolt/module_installer.rb +++ b/lib/bolt/module_installer.rb @@ -18,7 +18,7 @@ def initialize(outputter, pal) # Adds a single module to the project. # def add(name, specs, puppetfile_path, moduledir, project_file, config) - project_specs = Specs.new(specs) + project_specs = Specs.new(specs, config) # Exit early if project config already includes a spec with this name. if project_specs.include?(name) @@ -151,7 +151,7 @@ def install(specs, path, moduledir, config = {}, force: false, resolve: true) @outputter.print_message("Installing project modules\n\n") if resolve != false - specs = Specs.new(specs) + specs = Specs.new(specs, config) # If forcibly installing or if there is no Puppetfile, resolve # and write a Puppetfile. diff --git a/lib/bolt/module_installer/resolver.rb b/lib/bolt/module_installer/resolver.rb index 99c305225d..7985b8abb1 100644 --- a/lib/bolt/module_installer/resolver.rb +++ b/lib/bolt/module_installer/resolver.rb @@ -49,7 +49,7 @@ def resolve(specs, config = {}) spec_searcher_configuration: spec_searcher_config(config) ) rescue StandardError => e - raise Bolt::Error.new(e.message, 'bolt/module-resolver-error') + raise Bolt::Error.new("Unable to resolve modules: #{e.message}", 'bolt/module-resolver-error') end # Create the Puppetfile object. diff --git a/lib/bolt/module_installer/specs.rb b/lib/bolt/module_installer/specs.rb index ff030757c8..14c5dba22d 100644 --- a/lib/bolt/module_installer/specs.rb +++ b/lib/bolt/module_installer/specs.rb @@ -7,8 +7,10 @@ module Bolt class ModuleInstaller class Specs - def initialize(specs = []) - @specs = [] + def initialize(specs = [], config = {}) + @specs = [] + @config = config + add_specs(specs) assert_unique_names end @@ -49,7 +51,7 @@ def add_specs(*specs) # private def spec_from_hash(hash) return ForgeSpec.new(hash) if ForgeSpec.implements?(hash) - return GitSpec.new(hash) if GitSpec.implements?(hash) + return GitSpec.new(hash, @config) if GitSpec.implements?(hash) raise Bolt::ValidationError, <<~MESSAGE.chomp Invalid module specification: diff --git a/lib/bolt/module_installer/specs/git_spec.rb b/lib/bolt/module_installer/specs/git_spec.rb index 07428974ae..72c395d132 100644 --- a/lib/bolt/module_installer/specs/git_spec.rb +++ b/lib/bolt/module_installer/specs/git_spec.rb @@ -1,9 +1,15 @@ # frozen_string_literal: true require 'json' +require 'net/http' +require 'open3' require 'set' require_relative '../../../bolt/error' +require_relative '../../../bolt/logger' +require_relative '../../../bolt/module_installer/specs/id/gitclone' +require_relative '../../../bolt/module_installer/specs/id/github' +require_relative '../../../bolt/module_installer/specs/id/gitlab' # This class represents a Git module specification. # @@ -17,12 +23,19 @@ class GitSpec attr_reader :git, :ref, :resolve, :type - def initialize(init_hash) - @resolve = init_hash.key?('resolve') ? init_hash['resolve'] : true - @name = parse_name(init_hash['name']) - @git, @repo = parse_git(init_hash['git']) - @ref = init_hash['ref'] - @type = :git + def initialize(init_hash, config = {}) + @logger = Bolt::Logger.logger(self) + @resolve = init_hash.key?('resolve') ? init_hash['resolve'] : true + @git = init_hash['git'] + @ref = init_hash['ref'] + @name = parse_name(init_hash['name']) + @proxy = config.dig('proxy') + @type = :git + + unless @resolve == true || @resolve == false + raise Bolt::ValidationError, + "Option 'resolve' for module spec #{@git} must be a Boolean" + end if @name.nil? && @resolve == false raise Bolt::ValidationError, @@ -30,9 +43,9 @@ def initialize(init_hash) "must include a 'name' key when 'resolve' is false." end - unless @resolve == true || @resolve == false + unless valid_url?(@git) raise Bolt::ValidationError, - "Option 'resolve' for module spec #{@git} must be a Boolean" + "Invalid URI #{@git}. Valid URIs must begin with 'git@', 'http://', or 'https://'." end end @@ -57,22 +70,6 @@ def self.implements?(hash) match[:name] end - # Gets the repo from the git URL. - # - private def parse_git(git) - return [git, nil] unless @resolve - - repo = if git.start_with?('git@github.com:') - git.split('git@github.com:').last.split('.git').first - elsif git.start_with?('https://github.com') - git.split('https://github.com/').last.split('.git').first - else - raise Bolt::ValidationError, invalid_git_msg(git) - end - - [git, repo] - end - # Returns true if the specification is satisfied by the module. # def satisfied_by?(mod) @@ -88,14 +85,6 @@ def to_hash } end - # Returns an error message that the provided repo is not a git repo or - # is private. - # - private def invalid_git_msg(repo_name) - "#{repo_name} is not a public GitHub repository. See https://pup.pt/no-resolve "\ - "for information on how to install this module." - end - # Returns a PuppetfileResolver::Model::GitModule object for resolving. # def to_resolver_module @@ -107,90 +96,53 @@ def to_resolver_module end end - # Resolves the module's title from the module metadata. This is lazily - # resolved since Bolt does not always need to know a Git module's name. + # Returns the module's name. # def name - @name ||= begin - url = "https://raw.githubusercontent.com/#{@repo}/#{sha}/metadata.json" - response = make_request(:Get, url) - - case response - when Net::HTTPOK - body = JSON.parse(response.body) - - unless body.key?('name') - raise Bolt::Error.new( - "Missing name in metadata.json at #{git}. This is not a valid module.", - "bolt/missing-module-name-error" - ) - end - - parse_name(body['name']) - else - raise Bolt::Error.new( - "Missing metadata.json at #{git}. This is not a valid module.", - "bolt/missing-module-metadata-error" - ) - end - end + @name ||= parse_name(id.name) end - # Resolves the SHA for the specified ref. This is lazily resolved since - # Bolt does not always need to know a Git module's SHA. + # Returns the SHA for the module's ref. # def sha - @sha ||= begin - url = "https://api.github.com/repos/#{@repo}/commits/#{ref}" - headers = ENV['GITHUB_TOKEN'] ? { "Authorization" => "token #{ENV['GITHUB_TOKEN']}" } : {} - response = make_request(:Get, url, headers) - - case response - when Net::HTTPOK - body = JSON.parse(response.body) - body['sha'] - when Net::HTTPUnauthorized - raise Bolt::Error.new( - "Invalid token at GITHUB_TOKEN, unable to resolve git modules.", - "bolt/invalid-git-token-error" - ) - when Net::HTTPForbidden - message = "GitHub API rate limit exceeded, unable to resolve git modules. " - - unless ENV['GITHUB_TOKEN'] - message += "To increase your rate limit, set the GITHUB_TOKEN environment "\ - "variable with a GitHub personal access token." - end - - raise Bolt::Error.new(message, 'bolt/github-api-rate-limit-error') - when Net::HTTPNotFound - raise Bolt::Error.new(invalid_git_msg(git), "bolt/missing-git-repository-error") - else + id.sha + end + + # Gets the ID for the module based on the specified ref and git URL. + # This is lazily resolved since Bolt does not always need this information, + # and requesting it is expensive. + # + private def id + @id ||= begin + # The request methods here return an ID object if the module name and SHA + # were found and nil otherwise. This lets Bolt try multiple methods for + # finding the module name and SHA, and short circuiting as soon as it does. + module_id = Bolt::ModuleInstaller::Specs::ID::GitHub.request(@git, @ref, @proxy) || + Bolt::ModuleInstaller::Specs::ID::GitLab.request(@git, @ref, @proxy) || + Bolt::ModuleInstaller::Specs::ID::GitClone.request(@git, @ref, @proxy) + + unless module_id raise Bolt::Error.new( - "Ref #{ref} at #{git} is not a commit, tag, or branch.", - "bolt/invalid-git-ref-error" + "Unable to locate metadata and calculate SHA for ref #{@ref} at #{@git}. This may "\ + "not be a valid module. For more information about how Bolt attempted to locate "\ + "this information, check the debugging logs.", + 'bolt/missing-module-metadata-error' ) end + + module_id end end - # Makes a generic HTTP request. + # Returns true if the URL is valid. # - private def make_request(verb, url, headers = {}) - require 'net/http' - - uri = URI.parse(url) - opts = { use_ssl: uri.scheme == 'https' } + private def valid_url?(url) + return true if url.start_with?('git@') - Net::HTTP.start(uri.host, uri.port, opts) do |client| - request = Net::HTTP.const_get(verb).new(uri, headers) - client.request(request) - end - rescue StandardError => e - raise Bolt::Error.new( - "Failed to connect to #{uri}: #{e.message}", - "bolt/http-connect-error" - ) + uri = URI.parse(url) + uri.is_a?(URI::HTTP) && uri.host + rescue URI::InvalidURIError + false end end end diff --git a/lib/bolt/module_installer/specs/id/base.rb b/lib/bolt/module_installer/specs/id/base.rb new file mode 100644 index 0000000000..e0ab08891e --- /dev/null +++ b/lib/bolt/module_installer/specs/id/base.rb @@ -0,0 +1,116 @@ +# frozen_string_literal: true + +require 'json' +require 'net/http' + +require_relative '../../../../bolt/error' +require_relative '../../../../bolt/logger' + +module Bolt + class ModuleInstaller + class Specs + class ID + class Base + attr_reader :name, :sha + + # @param name [String] The module's name. + # @param sha [String] The ref's SHA1. + # + def initialize(name, sha) + @name = name + @sha = sha + end + + # Request the name and SHA for a module and ref. + # This method must return either an ID object or nil. The GitSpec + # class relies on this class to return an ID object to indicate + # the module was found, or nil to indicate that it should try to + # find it another way (such as cloning the repo). + # + # @param git [String] The URL to the git repo. + # @param ref [String] The ref to checkout. + # @param proxy [String] A proxy to use when making requests. + # + def self.request(git, ref, proxy) + name, sha = name_and_sha(git, ref, proxy) + name && sha ? new(name, sha) : nil + end + + # Stub method for retrieving the module's name and SHA. Must + # be implemented by all sub classes. + # + private_class_method def self.name_and_sha(_git, _ref, _proxy) + raise NotImplementedError, 'Class does not implemented #name_and_sha' + end + + # Makes a HTTP request. + # + # @param url [String] The URL to make the request to. + # @param proxy [String] A proxy to use when making the request. + # @param headers [Hash] Headers to send with the request. + # + private_class_method def self.make_request(url, proxy, headers = {}) + uri = URI.parse(url) + opts = { use_ssl: uri.scheme == 'https' } + args = [uri.host, uri.port] + + if proxy + proxy = URI.parse(proxy) + args += [proxy.host, proxy.port, proxy.user, proxy.password] + end + + Bolt::Logger.debug("Making request to #{loc(url, proxy)}") + + Net::HTTP.start(*args, opts) do |client| + client.request(Net::HTTP::Get.new(uri, headers)) + end + rescue StandardError => e + raise Bolt::Error.new( + "Failed to connect to #{loc(uri, proxy)}: #{e.message}", + "bolt/http-connect-error" + ) + end + + # Returns a formatted string describing the URL and proxy used when making + # a request. + # + # @param url [String, URI::HTTP] The URL used. + # @param proxy [String, URI::HTTP] The proxy used. + # + private_class_method def self.loc(url, proxy) + proxy ? "#{url} with proxy #{proxy}" : url.to_s + end + + # Parses the metadata and validates that it is a Hash. + # + # @param metadata [String] The JSON data to parse. + # + private_class_method def self.parse_name_from_metadata(metadata) + metadata = JSON.parse(metadata) + + unless metadata.is_a?(Hash) + raise Bolt::Error.new( + "Invalid metadata. Expected a Hash, got a #{metadata.class}: #{metadata}", + "bolt/invalid-module-metadata-error" + ) + end + + unless metadata.key?('name') + raise Bolt::Error.new( + "Invalid metadata. Metadata must include a 'name' key.", + "bolt/missing-module-name-error" + ) + end + + metadata['name'] + rescue JSON::ParserError => e + raise Bolt::Error.new( + "Unable to parse metadata as JSON: #{e.message}", + "bolt/metadata-parse-error" + ) + end + end + end + end + end +end diff --git a/lib/bolt/module_installer/specs/id/gitclone.rb b/lib/bolt/module_installer/specs/id/gitclone.rb new file mode 100644 index 0000000000..d761a4ac78 --- /dev/null +++ b/lib/bolt/module_installer/specs/id/gitclone.rb @@ -0,0 +1,120 @@ +# frozen_string_literal: true + +require_relative '../../../../bolt/module_installer/specs/id/base' + +module Bolt + class ModuleInstaller + class Specs + class ID + class GitClone < Base + # Returns the name and SHA for the module at the given ref. + # + # @param git [String] The URL to the git repo. + # @param ref [String] The ref to checkout. + # @param proxy [String] The proxy to use when cloning. + # + private_class_method def self.name_and_sha(git, ref, proxy) + require 'open3' + + unless git? + Bolt::Logger.debug("'git' executable not found, unable to use git clone resolution.") + return nil + end + + # Clone the repo into a temp directory that will be automatically cleaned up. + Dir.mktmpdir do |dir| + return nil unless clone_repo(git, ref, dir, proxy) + + # Extract the name from the metadata file and calculate the SHA. + Dir.chdir(dir) do + [request_name(git, ref), request_sha(git, ref)] + end + end + end + + # Requests a module's metadata and returns the name from it. + # + # @param git [String] The URL to the git repo. + # @param ref [String] The ref to checkout. + # + private_class_method def self.request_name(git, ref) + command = %W[git show #{ref}:metadata.json] + Bolt::Logger.debug("Executing command '#{command.join(' ')}'") + + out, err, status = Open3.capture3(*command) + + unless status.success? + raise Bolt::Error.new( + "Unable to find metadata file at #{git}: #{err}", + "bolt/missing-metadata-file-error" + ) + end + + Bolt::Logger.debug("Found metadata file at #{git}") + parse_name_from_metadata(out) + end + + # Requests the SHA for the specified ref. + # + # @param git [String] The URL to the git repo. + # @param ref [String] The ref to checkout. + # + private_class_method def self.request_sha(git, ref) + command = %W[git rev-parse #{ref}^{commit}] + Bolt::Logger.debug("Executing command '#{command.join(' ')}'") + + out, err, status = Open3.capture3(*command) + + if status.success? + out.strip + else + raise Bolt::Error.new( + "Unable to calculate SHA for ref #{ref} at #{git}: #{err}", + "bolt/invalid-ref-error" + ) + end + end + + # Clones the repository. First attempts to clone a bare repository + # and falls back to cloning the full repository if that fails. Cloning + # a bare repository is significantly faster for large modules, but + # cloning a bare repository using a commit is not supported. + # + # @param git [String] The URL to the git repo. + # @param ref [String] The ref to checkout. + # @param dir [String] The directory to clone the repo to. + # @param proxy [String] The proxy to use when cloning. + # + private_class_method def self.clone_repo(git, ref, dir, proxy) + clone = %W[git clone #{git} #{dir}] + clone += %W[--config "http.proxy=#{proxy}" --config "https.proxy=#{proxy}"] if proxy + + bare_clone = clone + %w[--bare --depth=1] + bare_clone.push("--branch=#{ref}") unless ref == 'HEAD' + + # Attempt to clone a bare repository + Bolt::Logger.debug("Executing command '#{bare_clone.join(' ')}'") + _out, err, status = Open3.capture3(*bare_clone) + return true if status.success? + Bolt::Logger.debug("Unable to clone bare repository at #{loc(git, proxy)}: #{err}") + + # Fall back to cloning the full repository + Bolt::Logger.debug("Executing command '#{clone.join(' ')}'") + _out, err, status = Open3.capture3(*clone) + Bolt::Logger.debug("Unable to clone repository at #{loc(git, proxy)}: #{err}") unless status.success? + status.success? + end + + # Returns true if the 'git' executable is available. + # + private_class_method def self.git? + Open3.capture3('git', '--version') + true + rescue Errno::ENOENT + false + end + end + end + end + end +end diff --git a/lib/bolt/module_installer/specs/id/github.rb b/lib/bolt/module_installer/specs/id/github.rb new file mode 100644 index 0000000000..57afffd833 --- /dev/null +++ b/lib/bolt/module_installer/specs/id/github.rb @@ -0,0 +1,90 @@ +# frozen_string_literal: true + +require_relative '../../../../bolt/module_installer/specs/id/base' + +module Bolt + class ModuleInstaller + class Specs + class ID + class GitHub < Base + # Returns the name and SHA for the module at the given ref. + # + # @param git [String] The URL to the git repo. + # @param ref [String] The ref to use. + # @param proxy [String] The proxy to use when making requests. + # + private_class_method def self.name_and_sha(git, ref, proxy) + repo = parse_repo(git) + return nil unless repo + [request_name(repo, ref, proxy), request_sha(repo, ref, proxy)] + end + + # Parses the repo path out of the URL. + # + # @param git [String] The URL to the git repo. + # + private_class_method def self.parse_repo(git) + if git.start_with?('git@github.com:') + git.split('git@github.com:').last.split('.git').first + elsif git.start_with?('https://github.com') + git.split('https://github.com/').last.split('.git').first + end + end + + # Requests a module's metadata and returns the name from it. + # + # @param repo [String] The repo ID, i.e. 'owner/repo' + # @param ref [String] The ref to use. + # @param proxy [String] The proxy to use when making requests. + # + private_class_method def self.request_name(repo, ref, proxy) + metadata_url = "https://raw.githubusercontent.com/#{repo}/#{ref}/metadata.json" + response = make_request(metadata_url, proxy) + + case response + when Net::HTTPOK + Bolt::Logger.debug("Found metadata file at #{loc(metadata_url, proxy)}") + parse_name_from_metadata(response.body) + else + Bolt::Logger.debug("Unable to find metadata file at #{loc(metadata_url, proxy)}") + nil + end + end + + # Requests the SHA for the specified ref. + # + # @param repo [String] The repo ID, i.e. 'owner/repo' + # @param ref [String] The ref to resolve. + # @param proxy [String] The proxy to use when making requests. + # + private_class_method def self.request_sha(repo, ref, proxy) + url = "https://api.github.com/repos/#{repo}/commits/#{ref}" + headers = ENV['GITHUB_TOKEN'] ? { "Authorization" => "token #{ENV['GITHUB_TOKEN']}" } : {} + response = make_request(url, proxy, headers) + + case response + when Net::HTTPOK + JSON.parse(response.body).fetch('sha', nil) + when Net::HTTPUnauthorized + Bolt::Logger.debug("Invalid token at GITHUB_TOKEN, unable to calculate SHA.") + nil + when Net::HTTPForbidden + message = "GitHub API rate limit exceeded, unable to calculate SHA." + + unless ENV['GITHUB_TOKEN'] + message += " To increase your rate limit, set the GITHUB_TOKEN environment "\ + "variable with a GitHub personal access token." + end + + Bolt::Logger.debug(message) + nil + else + Bolt::Logger.debug("Unable to calculate SHA for ref #{ref}") + nil + end + end + end + end + end + end +end diff --git a/lib/bolt/module_installer/specs/id/gitlab.rb b/lib/bolt/module_installer/specs/id/gitlab.rb new file mode 100644 index 0000000000..870dce4eef --- /dev/null +++ b/lib/bolt/module_installer/specs/id/gitlab.rb @@ -0,0 +1,92 @@ +# frozen_string_literal: true + +require_relative '../../../../bolt/module_installer/specs/id/base' + +module Bolt + class ModuleInstaller + class Specs + class ID + class GitLab < Base + # Returns the name and SHA for the module at the given ref. + # + # @param git [String] The URL to the git repo. + # @param ref [String] The ref to use. + # @param proxy [String] The proxy to use when making requests. + # + private_class_method def self.name_and_sha(git, ref, proxy) + repo = parse_repo(git) + return nil unless repo + [request_name(repo, ref, proxy), request_sha(repo, ref, proxy)] + end + + # Parses the repo path out of the URL. + # + # @param git [String] The URL to the git repo. + # + private_class_method def self.parse_repo(git) + if git.start_with?('git@gitlab.com:') + git.split('git@gitlab.com:').last.split('.git').first + elsif git.start_with?('https://gitlab.com') + git.split('https://gitlab.com/').last.split('.git').first + end + end + + # Requests a module's metadata and returns the name from it. + # + # @param repo [String] The repo ID, i.e. 'owner/repo' + # @param ref [String] The ref to use. + # @param proxy [String] The proxy to use when making requests. + # + private_class_method def self.request_name(repo, ref, proxy) + metadata_url = "https://gitlab.com/#{repo}/-/raw/#{ref}/metadata.json" + response = make_request(metadata_url, proxy) + + case response + when Net::HTTPOK + Bolt::Logger.debug("Found metadata file at #{loc(metadata_url, proxy)}") + parse_name_from_metadata(response.body) + else + Bolt::Logger.debug("Unable to find metadata file at #{loc(metadata_url, proxy)}") + nil + end + end + + # Requests the SHA for the specified ref. + # + # @param repo [String] The repo ID, i.e. 'owner/repo' + # @param ref [String] The ref to resolve. + # @param proxy [String] The proxy to use when making requests. + # + private_class_method def self.request_sha(repo, ref, proxy) + require 'cgi' + + url = "https://gitlab.com/api/v4/projects/#{CGI.escape(repo)}/repository/commits/#{ref}" + headers = ENV['GITLAB_TOKEN'] ? { "PRIVATE-TOKEN" => ENV['GITLAB_TOKEN'] } : {} + response = make_request(url, proxy, headers) + + case response + when Net::HTTPOK + JSON.parse(response.body).fetch('id', nil) + when Net::HTTPUnauthorized + Bolt::Logger.debug("Invalid token at GITLAB_TOKEN, unable to calculate SHA.") + nil + when Net::HTTPForbidden + message = "GitLab API rate limit exceeded, unable to calculate SHA." + + unless ENV['GITLAB_TOKEN'] + message += " To increase your rate limit, set the GITLAB_TOKEN environment "\ + "variable with a GitLab personal access token." + end + + Bolt::Logger.debug(message) + nil + else + Bolt::Logger.debug("Unable to calculate SHA for ref #{ref}") + nil + end + end + end + end + end + end +end diff --git a/spec/integration/module_installer/module_installer_spec.rb b/spec/integration/module_installer/module_installer_spec.rb index 9d4fbddd11..abf1ebbbd0 100644 --- a/spec/integration/module_installer/module_installer_spec.rb +++ b/spec/integration/module_installer/module_installer_spec.rb @@ -102,6 +102,102 @@ end end + context 'resolving git modules' do + let(:project_config) { { 'modules' => [mod] } } + + context 'public GitHub source' do + let(:mod) do + { + 'git' => 'https://github.com/puppetlabs/puppetlabs-yaml.git', + 'ref' => '0.1.0' + } + end + + it 'installs the module' do + result = run_cli_json(command, project: project) + + expect(result).to eq( + 'success' => true, + 'puppetfile' => project.puppetfile.to_s, + 'moduledir' => project.managed_moduledir.to_s + ) + expect(project.puppetfile.exist?).to be(true) + expect(project.managed_moduledir.exist?).to be(true) + + puppetfile_content = File.read(project.puppetfile) + + expect(puppetfile_content.lines).to include( + /mod 'yaml'/, + %r{git: 'https://github.com/puppetlabs/puppetlabs-yaml.git'}, + /ref: 'bdaa6b531fde16baab5752916a49423925493f2f'/ + ) + end + end + + context 'public GitLab source' do + let(:mod) do + { + 'git' => 'https://gitlab.com/simp/pupmod-simp-crypto_policy.git', + 'ref' => '0.1.0' + } + end + + it 'installs the module' do + result = run_cli_json(command, project: project) + + expect(result).to eq( + 'success' => true, + 'puppetfile' => project.puppetfile.to_s, + 'moduledir' => project.managed_moduledir.to_s + ) + expect(project.puppetfile.exist?).to be(true) + expect(project.managed_moduledir.exist?).to be(true) + + puppetfile_content = File.read(project.puppetfile) + + expect(puppetfile_content.lines).to include( + /mod 'crypto_policy'/, + %r{git: 'https://gitlab.com/simp/pupmod-simp-crypto_policy.git'}, + /ref: '04fe0b7fd0d4a8fa9db4a32845a11171f1007b3a'/ + ) + end + end + + context 'git clone source' do + let(:mod) do + { + 'git' => 'https://github.com/puppetlabs/puppetlabs-yaml.git', + 'ref' => '0.1.0' + } + end + + it 'installs the module' do + # Make Bolt believe this module needs to be cloned since private repos + # require authorization that the test runner doesn't have. + allow(Bolt::ModuleInstaller::Specs::ID::GitHub).to receive(:request).and_return(nil) + allow(Bolt::ModuleInstaller::Specs::ID::GitLab).to receive(:request).and_return(nil) + + result = run_cli_json(command, project: project) + + expect(result).to eq( + 'success' => true, + 'puppetfile' => project.puppetfile.to_s, + 'moduledir' => project.managed_moduledir.to_s + ) + expect(project.puppetfile.exist?).to be(true) + expect(project.managed_moduledir.exist?).to be(true) + + puppetfile_content = File.read(project.puppetfile) + + expect(puppetfile_content.lines).to include( + /mod 'yaml'/, + %r{git: 'https://github.com/puppetlabs/puppetlabs-yaml.git'}, + /ref: 'bdaa6b531fde16baab5752916a49423925493f2f'/ + ) + end + end + end + context 'with forge and git modules' do let(:project_config) do { @@ -208,9 +304,12 @@ end it 'errors' do + # Skip GitClone resolution, which requires authentication that the runner does not have + allow(Bolt::ModuleInstaller::Specs::ID::GitClone).to receive(:request).and_return(nil) + expect { run_cli(command, project: project) }.to raise_error( Bolt::Error, - %r{https://github.com/puppetlabs/puppetlabs-foobarbaz is not a public GitHub repository.} + /Unable to locate metadata.*This may not be a valid module/ ) end end diff --git a/spec/unit/module_installer/specs/git_spec_spec.rb b/spec/unit/module_installer/specs/git_spec_spec.rb index 111483f201..41674aca09 100644 --- a/spec/unit/module_installer/specs/git_spec_spec.rb +++ b/spec/unit/module_installer/specs/git_spec_spec.rb @@ -38,10 +38,10 @@ end it 'errors with invalid git source' do - init_hash['git'] = 'https://gitlab.com/puppetlabs/puppetlabs-yaml' + init_hash['git'] = 'gitlab.com/puppetlabs/puppetlabs-yaml' expect { spec }.to raise_error( Bolt::ValidationError, - /^.*is not a public GitHub repository/ + /Invalid URI #{init_hash['git']}/ ) end @@ -105,10 +105,7 @@ context '#to_resolver_module' do it 'returns a puppetfile-resolver module object' do - allow(spec).to receive(:sha).and_return(ref) - mod = spec.to_resolver_module - - expect(mod).to be_a(PuppetfileResolver::Puppetfile::GitModule) + expect(spec.to_resolver_module).to be_a(PuppetfileResolver::Puppetfile::GitModule) end end @@ -118,76 +115,5 @@ it 'resolves and returns the module name' do expect(spec.name).to eq(name) end - - context 'with missing metadata.json' do - let(:git) { 'https://github.com/puppetlabs/bolt' } - - it 'errors' do - expect { spec.name }.to raise_error( - Bolt::Error, - /Missing metadata\.json/ - ) - end - end - end - - context '#resolve_sha' do - context 'with a valid commit' do - let(:ref) { '79f98ffd3faf8d3badb1084a676e5fc1cbac464e' } - - it 'resolves and returns a SHA' do - expect(spec.sha).to eq('79f98ffd3faf8d3badb1084a676e5fc1cbac464e') - end - end - - context 'with a valid tag' do - let(:ref) { '0.2.0' } - - it 'resolves and returns a SHA' do - expect(spec.sha).to eq('79f98ffd3faf8d3badb1084a676e5fc1cbac464e') - end - end - - context 'with a valid branch' do - let(:ref) { 'main' } - - it 'resolves and returns a SHA' do - expect(spec.sha).to be_a(String) - end - end - - context 'with an invalid ref' do - let(:ref) { 'foobar' } - - it 'errors' do - expect { spec.sha }.to raise_error( - Bolt::Error, - /not a commit, tag, or branch/ - ) - end - end - - context 'with an invalid repository' do - let(:git) { 'https://github.com/puppetlabs/foobarbaz' } - - it 'errors' do - expect { spec.sha }.to raise_error( - Bolt::Error, - /is not a public GitHub repository/ - ) - end - end - - it 'errors with an invalid GitHub token' do - original = ENV['GITHUB_TOKEN'] - ENV['GITHUB_TOKEN'] = 'foo' - - expect { spec.sha }.to raise_error( - Bolt::Error, - /Invalid token at GITHUB_TOKEN/ - ) - ensure - ENV['GITHUB_TOKEN'] = original - end end end diff --git a/spec/unit/module_installer/specs/id/gitclone_spec.rb b/spec/unit/module_installer/specs/id/gitclone_spec.rb new file mode 100644 index 0000000000..36d3950ca9 --- /dev/null +++ b/spec/unit/module_installer/specs/id/gitclone_spec.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +require 'spec_helper' + +require 'bolt/module_installer/specs/id/gitclone' + +describe Bolt::ModuleInstaller::Specs::ID::GitClone do + let(:id) { described_class.request(git, ref, nil) } + let(:git) { "https://github.com/puppetlabs/#{name}" } + let(:ref) { '0.1.0' } + let(:sha) { 'bdaa6b531fde16baab5752916a49423925493f2f' } + let(:name) { 'puppetlabs-yaml' } + let(:metadata) { "{\"name\":\"#{name}\"}" } + + let(:err) { 'something went wrong' } + let(:status) { double('status', success?: true) } + let(:err_status) { double('err_status', success?: false) } + + before(:each) do + allow(described_class).to receive(:git?).and_return(true) + allow(Open3).to receive(:capture3).with('git', 'rev-parse', any_args).and_return([sha, '', status]) + allow(Open3).to receive(:capture3).with('git', 'show', any_args).and_return([metadata, '', status]) + allow(Open3).to receive(:capture3).with('git', 'clone', any_args).and_return(['', '', status]) + end + + it 'returns module name and SHA' do + expect(id).to be_instance_of(described_class) + expect(id.name).to eq(name) + expect(id.sha).to eq(sha) + end + + it 'returns nil if git executable is not available' do + allow(described_class).to receive(:git?).and_return(false) + expect(id).to eq(nil) + expect(@log_output.readlines).to include(/'git' executable not found/) + end + + it 'returns nil if unable to clone repo' do + allow(Open3).to receive(:capture3).and_return(['', err, err_status]) + expect(id).to eq(nil) + expect(@log_output.readlines).to include(/Unable to clone bare repo.*#{err}/, /Unable to clone repo.*#{err}/) + end + + it 'errors if unable to find metadata' do + expect(Open3).to receive(:capture3).with('git', 'show', any_args).and_return(['', err, err_status]) + expect { id }.to raise_error(/Unable to find metadata file.*#{err}/) + end + + it 'errors if unable to parse metadata' do + expect(Open3).to receive(:capture3).with('git', 'show', any_args).and_return(['foo', '', status]) + expect { id }.to raise_error(/Unable to parse metadata as JSON/) + end + + it 'errors if metadata is not a Hash' do + expect(Open3).to receive(:capture3).with('git', 'show', any_args).and_return(['"foo"', '', status]) + expect { id }.to raise_error(/Invalid metadata. Expected a Hash, got a String/) + end + + it 'errors if metadata is missing a name' do + expect(Open3).to receive(:capture3).with('git', 'show', any_args).and_return(['{}', '', status]) + expect { id }.to raise_error(/Invalid metadata. Metadata must include a 'name' key./) + end + + it 'errors if unable to calculate SHA' do + expect(Open3).to receive(:capture3).with('git', 'rev-parse', any_args).and_return(['', err, err_status]) + expect { id }.to raise_error(/Unable to calculate SHA for ref #{ref}.*#{err}/) + end +end diff --git a/spec/unit/module_installer/specs/id/github_spec.rb b/spec/unit/module_installer/specs/id/github_spec.rb new file mode 100644 index 0000000000..acf107415f --- /dev/null +++ b/spec/unit/module_installer/specs/id/github_spec.rb @@ -0,0 +1,67 @@ +# frozen_string_literal: true + +require 'spec_helper' + +require 'bolt/module_installer/specs/id/github' + +describe Bolt::ModuleInstaller::Specs::ID::GitHub do + let(:id) { described_class.request(git, ref, nil) } + let(:git) { "https://github.com/puppetlabs/#{name}" } + let(:ref) { '0.1.0' } + let(:sha) { 'bdaa6b531fde16baab5752916a49423925493f2f' } + let(:name) { 'puppetlabs-yaml' } + let(:metadata) { "{\"name\":\"#{name}\"}" } + let(:sha_json) { "{\"sha\":\"#{sha}\"}" } + + let(:metadata_response) { double('metadata_response', body: metadata) } + let(:sha_response) { double('sha_response', body: sha_json) } + + before(:each) do + allow(described_class).to receive(:make_request) + .with(/raw.githubusercontent.com/, any_args) + .and_return(metadata_response) + allow(described_class).to receive(:make_request) + .with(/api.github.com/, any_args) + .and_return(sha_response) + allow(Net::HTTPOK).to receive(:===).with(metadata_response).and_return(true) + allow(Net::HTTPOK).to receive(:===).with(sha_response).and_return(true) + end + + it 'returns module name and sha' do + expect(id).to be_instance_of(described_class) + expect(id.name).to eq(name) + expect(id.sha).to eq(sha) + end + + it 'returns nil if not a GitHub repo' do + id = described_class.request('https://gitlab.com/puppetlabs/puppetlabs-yaml', ref, nil) + expect(id).to eq(nil) + end + + it 'returns nil if unable to find metadata file' do + allow(Net::HTTPOK).to receive(:===).with(metadata_response).and_return(false) + expect(id).to eq(nil) + expect(@log_output.readlines).to include(/Unable to find metadata file/) + end + + it 'errors if unable to parse metadata' do + allow(metadata_response).to receive(:body).and_return('foo') + expect { id }.to raise_error(/Unable to parse metadata as JSON/) + end + + it 'errors if metadata is not a Hash' do + allow(metadata_response).to receive(:body).and_return('"foo"') + expect { id }.to raise_error(/Invalid metadata. Expected a Hash, got a String/) + end + + it 'errors if metadata is missing a name' do + allow(metadata_response).to receive(:body).and_return('{}') + expect { id }.to raise_error(/Invalid metadata. Metadata must include a 'name' key./) + end + + it 'errors if unable to calculate SHA' do + expect(Net::HTTPOK).to receive(:===).with(sha_response).and_return(false) + expect(id).to eq(nil) + expect(@log_output.readlines).to include(/Unable to calculate SHA for ref #{ref}/) + end +end diff --git a/spec/unit/module_installer/specs/id/gitlab_spec.rb b/spec/unit/module_installer/specs/id/gitlab_spec.rb new file mode 100644 index 0000000000..6540f53db8 --- /dev/null +++ b/spec/unit/module_installer/specs/id/gitlab_spec.rb @@ -0,0 +1,67 @@ +# frozen_string_literal: true + +require 'spec_helper' + +require 'bolt/module_installer/specs/id/gitlab' + +describe Bolt::ModuleInstaller::Specs::ID::GitLab do + let(:id) { described_class.request(git, ref, nil) } + let(:git) { "https://gitlab.com/puppetlabs/#{name}" } + let(:ref) { '0.1.0' } + let(:sha) { 'bdaa6b531fde16baab5752916a49423925493f2f' } + let(:name) { 'puppetlabs-yaml' } + let(:metadata) { "{\"name\":\"#{name}\"}" } + let(:sha_json) { "{\"id\":\"#{sha}\"}" } + + let(:metadata_response) { double('metadata_response', body: metadata) } + let(:sha_response) { double('sha_response', body: sha_json) } + + before(:each) do + allow(described_class).to receive(:make_request) + .with(%r{gitlab.com/puppetlabs}, any_args) + .and_return(metadata_response) + allow(described_class).to receive(:make_request) + .with(%r{gitlab.com/api}, any_args) + .and_return(sha_response) + allow(Net::HTTPOK).to receive(:===).with(metadata_response).and_return(true) + allow(Net::HTTPOK).to receive(:===).with(sha_response).and_return(true) + end + + it 'returns module name and sha' do + expect(id).to be_instance_of(described_class) + expect(id.name).to eq(name) + expect(id.sha).to eq(sha) + end + + it 'returns nil if not a GitLab repo' do + id = described_class.request('https://github.com/puppetlabs/puppetlabs-yaml', ref, nil) + expect(id).to eq(nil) + end + + it 'returns nil if unable to find metadata file' do + allow(Net::HTTPOK).to receive(:===).with(metadata_response).and_return(false) + expect(id).to eq(nil) + expect(@log_output.readlines).to include(/Unable to find metadata file/) + end + + it 'errors if unable to parse metadata' do + allow(metadata_response).to receive(:body).and_return('foo') + expect { id }.to raise_error(/Unable to parse metadata as JSON/) + end + + it 'errors if metadata is not a Hash' do + allow(metadata_response).to receive(:body).and_return('"foo"') + expect { id }.to raise_error(/Invalid metadata. Expected a Hash, got a String/) + end + + it 'errors if metadata is missing a name' do + allow(metadata_response).to receive(:body).and_return('{}') + expect { id }.to raise_error(/Invalid metadata. Metadata must include a 'name' key./) + end + + it 'errors if unable to calculate SHA' do + expect(Net::HTTPOK).to receive(:===).with(sha_response).and_return(false) + expect(id).to eq(nil) + expect(@log_output.readlines).to include(/Unable to calculate SHA for ref #{ref}/) + end +end