From 1e95757d2cb027cbd7c2cc584462a8edf0bd170a Mon Sep 17 00:00:00 2001 From: "Brian J. Cardiff" Date: Thu, 15 May 2025 23:39:19 -0300 Subject: [PATCH 01/12] Add parts_from_cli helper method to parse cli arguments --- spec/unit/dependency_spec.cr | 43 ++++++++++++++++++++ src/dependency.cr | 76 ++++++++++++++++++++++++++++++++++++ src/resolvers/resolver.cr | 4 ++ 3 files changed, 123 insertions(+) diff --git a/spec/unit/dependency_spec.cr b/spec/unit/dependency_spec.cr index 90bb47b4..663cbfb7 100644 --- a/spec/unit/dependency_spec.cr +++ b/spec/unit/dependency_spec.cr @@ -71,6 +71,49 @@ module Shards parse_dependency({foo: {git: "", tag: "rc-1.0"}}).to_s.should eq("foo (tag rc-1.0)") parse_dependency({foo: {git: "", commit: "4478d8afe8c728f44b47d3582a270423cd7fc07d"}}).to_s.should eq("foo (commit 4478d8a)") end + + it ".parts_from_cli" do + # GitHub short syntax + Dependency.parts_from_cli("github:foo/bar").should eq({resolver_key: "github", source: "foo/bar", requirement: Any}) + Dependency.parts_from_cli("github:Foo/Bar@1.2.3").should eq({resolver_key: "github", source: "Foo/Bar", requirement: VersionReq.new("~> 1.2.3")}) + + # GitHub urls + Dependency.parts_from_cli("https://github.com/foo/bar").should eq({resolver_key: "github", source: "foo/bar", requirement: Any}) + Dependency.parts_from_cli("https://github.com/Foo/Bar/commit/000000").should eq({resolver_key: "github", source: "Foo/Bar", requirement: GitCommitRef.new("000000")}) + Dependency.parts_from_cli("https://github.com/Foo/Bar/tree/v1.2.3").should eq({resolver_key: "github", source: "Foo/Bar", requirement: GitTagRef.new("v1.2.3")}) + Dependency.parts_from_cli("https://github.com/Foo/Bar/tree/some/branch").should eq({resolver_key: "github", source: "Foo/Bar", requirement: GitBranchRef.new("some/branch")}) + + # GitLab short syntax + Dependency.parts_from_cli("gitlab:foo/bar").should eq({resolver_key: "gitlab", source: "foo/bar", requirement: Any}) + + # GitLab urls + Dependency.parts_from_cli("https://gitlab.com/foo/bar").should eq({resolver_key: "gitlab", source: "foo/bar", requirement: Any}) + + # Bitbucket short syntax + Dependency.parts_from_cli("bitbucket:foo/bar").should eq({resolver_key: "bitbucket", source: "foo/bar", requirement: Any}) + + # bitbucket urls + Dependency.parts_from_cli("https://bitbucket.com/foo/bar").should eq({resolver_key: "bitbucket", source: "foo/bar", requirement: Any}) + + # Git convenient syntax since resolver matches scheme + Dependency.parts_from_cli("git://git.example.org/crystal-library.git").should eq({resolver_key: "git", source: "git://git.example.org/crystal-library.git", requirement: Any}) + + # Local paths + local_absolute = File.join(tmp_path, "local") + local_relative = File.join("spec", ".repositories", "local") # rel_path is relative to integration spec + Dir.mkdir_p(local_absolute) + + # Path short syntax + Dependency.parts_from_cli(local_absolute).should eq({resolver_key: "path", source: local_absolute, requirement: Any}) + Dependency.parts_from_cli(local_relative).should eq({resolver_key: "path", source: local_relative, requirement: Any}) + + # Path resolver syntax + Dependency.parts_from_cli("path:#{local_absolute}").should eq({resolver_key: "path", source: local_absolute, requirement: Any}) + Dependency.parts_from_cli("path:#{local_relative}").should eq({resolver_key: "path", source: local_relative, requirement: Any}) + + # Other resolvers short + Dependency.parts_from_cli("git:git://git.example.org/crystal-library.git").should eq({resolver_key: "git", source: "git://git.example.org/crystal-library.git", requirement: Any}) + end end end diff --git a/src/dependency.cr b/src/dependency.cr index c4d51b26..f645df3c 100644 --- a/src/dependency.cr +++ b/src/dependency.cr @@ -11,6 +11,82 @@ module Shards def initialize(@name : String, @resolver : Resolver, @requirement : Requirement = Any) end + # :nodoc: + # + # Parse the dependency from a CLI argument + # and return the parts needed to create the proper dependency. + # + # Split to allow better unit testing. + def self.parts_from_cli(value : String) : {resolver_key: String, source: String, requirement: Requirement} + resolver_key = nil + source = "" + requirement = Any + + if File.directory?(value) + resolver_key = "path" + source = value + end + + if value.starts_with?("https://github.com") + resolver_key = "github" + uri = URI.parse(value) + source = uri.path[1..-1] # drop first "/"" + + components = source.split("/") + case components[2]? + when "commit" + source = "#{components[0]}/#{components[1]}" + requirement = GitCommitRef.new(components[3]) + when "tree" + source = "#{components[0]}/#{components[1]}" + requirement = if components[3].starts_with?("v") + GitTagRef.new(components[3]) + else + GitBranchRef.new(components[3..-1].join("/")) + end + end + end + + if value.starts_with?("https://gitlab.com") + resolver_key = "gitlab" + uri = URI.parse(value) + source = uri.path[1..-1] # drop first "/"" + end + + if value.starts_with?("https://bitbucket.com") + resolver_key = "bitbucket" + uri = URI.parse(value) + source = uri.path[1..-1] # drop first "/"" + end + + if value.starts_with?("git://") + resolver_key = "git" + source = value + end + + unless resolver_key + Resolver.resolver_keys.each do |key| + key_schema = "#{key}:" + if value.starts_with?(key_schema) + resolver_key = key + source = value.sub(key_schema, "") + + # narrow down requirement + if source.includes?("@") + source, version = source.split("@") + requirement = VersionReq.new("~> #{version}") + end + + break + end + end + end + + raise Shards::Error.new("Invalid dependency format: #{value}") unless resolver_key + + {resolver_key: resolver_key, source: source, requirement: requirement} + end + def self.from_yaml(pull : YAML::PullParser) mapping_start = pull.location name = pull.read_scalar diff --git a/src/resolvers/resolver.cr b/src/resolvers/resolver.cr index 9adfe7cd..b1069f5c 100644 --- a/src/resolvers/resolver.cr +++ b/src/resolvers/resolver.cr @@ -103,6 +103,10 @@ module Shards private RESOLVER_CLASSES = {} of String => Resolver.class private RESOLVER_CACHE = {} of ResolverCacheKey => Resolver + def self.resolver_keys + RESOLVER_CLASSES.keys + end + def self.register_resolver(key, resolver) RESOLVER_CLASSES[key] = resolver end From aae1479410a963c99fc796714be4eafe7d120af2 Mon Sep 17 00:00:00 2001 From: "Brian J. Cardiff" Date: Thu, 15 May 2025 23:49:13 -0300 Subject: [PATCH 02/12] Allow Dependency to preserve resolver key and source --- src/dependency.cr | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/src/dependency.cr b/src/dependency.cr index f645df3c..4a43b39d 100644 --- a/src/dependency.cr +++ b/src/dependency.cr @@ -7,8 +7,29 @@ module Shards property name : String property resolver : Resolver property requirement : Requirement + # resolver's key and source are normalized. We preserve the key and source to be used + # in the shard.yml file in these field. This is used to generate the shard.yml file + # in a more human-readable way. + # A Dependency can still be created without them, but it will not be possible to + # generate the shard.yml file. + property! resolver_key : String + property! source : String + + def initialize(@name : String, @resolver : Resolver, @requirement : Requirement = Any, @resolver_key : String? = nil, @source : String? = nil) + end + + # Parse a dependency from a CLI argument + def self.from_cli(value : String) : Dependency + parts = parts_from_cli(value) + + # We need to check the actual shard name to create a dependency. + # This requires getting the actual spec file from some matching version. + resolver = Resolver.find_resolver(parts[:resolver_key], "unknown", parts[:source]) + version = resolver.versions_for(parts[:requirement]).first || raise Shards::Error.new("No versions found for dependency: #{value}") + spec = resolver.spec(version) + name = spec.name || raise Shards::Error.new("No name found for dependency: #{value}") - def initialize(@name : String, @resolver : Resolver, @requirement : Requirement = Any) + Dependency.new(name, resolver, parts[:requirement], parts[:resolver_key], parts[:source]) end # :nodoc: @@ -120,6 +141,7 @@ module Shards end end + # Used to generate the shard.lock file. def to_yaml(yaml : YAML::Builder) yaml.scalar name yaml.mapping do @@ -129,6 +151,16 @@ module Shards end end + # Used to generate the shard.yml file. + def to_shard_yaml(yaml : YAML::Builder) + yaml.scalar name + yaml.mapping do + yaml.scalar resolver_key + yaml.scalar source + requirement.to_yaml(yaml) + end + end + def as_package? version = case req = @requirement From eafca308e9ffa4078d03f8a1bb9ab2406ff296f0 Mon Sep 17 00:00:00 2001 From: "Brian J. Cardiff" Date: Fri, 16 May 2025 10:21:02 -0300 Subject: [PATCH 03/12] Extract logic from Dependency, update supported syntax --- spec/unit/dependency_definition_spec.cr | 61 +++++++++++++ spec/unit/dependency_spec.cr | 43 --------- src/dependency.cr | 109 +--------------------- src/dependency_definition.cr | 114 ++++++++++++++++++++++++ 4 files changed, 176 insertions(+), 151 deletions(-) create mode 100644 spec/unit/dependency_definition_spec.cr create mode 100644 src/dependency_definition.cr diff --git a/spec/unit/dependency_definition_spec.cr b/spec/unit/dependency_definition_spec.cr new file mode 100644 index 00000000..795d6f59 --- /dev/null +++ b/spec/unit/dependency_definition_spec.cr @@ -0,0 +1,61 @@ +require "./spec_helper" +require "../../src/dependency_definition" + +private def expect_parses(value, resolver_key : String, source : String, requirement : Shards::Requirement) + Shards::DependencyDefinition.parts_from_cli(value).should eq(Shards::DependencyDefinition::Parts.new(resolver_key: resolver_key, source: source, requirement: requirement)) +end + +module Shards + describe DependencyDefinition do + it ".parts_from_cli" do + # GitHub short syntax + expect_parses("github:foo/bar", "github", "foo/bar", Any) + expect_parses("github:Foo/Bar@1.2.3", "github", "Foo/Bar", VersionReq.new("~> 1.2.3")) + + # GitHub urls + expect_parses("https://github.com/foo/bar", "github", "foo/bar", Any) + + # GitHub urls from clone popup + expect_parses("https://github.com/foo/bar.git", "github", "foo/bar", Any) + expect_parses("git@github.com:foo/bar.git", "git", "git@github.com:foo/bar.git", Any) + + # GitLab short syntax + expect_parses("gitlab:foo/bar", "gitlab", "foo/bar", Any) + + # GitLab urls + expect_parses("https://gitlab.com/foo/bar", "gitlab", "foo/bar", Any) + + # GitLab urls from clone popup + expect_parses("https://gitlab.com/foo/bar.git", "gitlab", "foo/bar", Any) + expect_parses("git@gitlab.com:foo/bar.git", "git", "git@gitlab.com:foo/bar.git", requirement: Any) + + # Bitbucket short syntax + expect_parses("bitbucket:foo/bar", "bitbucket", "foo/bar", Any) + + # bitbucket urls + expect_parses("https://bitbucket.com/foo/bar", "bitbucket", "foo/bar", Any) + + # Git convenient syntax since resolver matches scheme + expect_parses("git://git.example.org/crystal-library.git", "git", "git://git.example.org/crystal-library.git", Any) + expect_parses("git@example.org:foo/bar.git", "git", "git@example.org:foo/bar.git", Any) + + # Local paths + local_absolute = "/an/absolute/path" + local_relative = "an/relative/path" + + # Path short syntax + expect_parses("./#{local_relative}", "path", "./#{local_relative}", Any) + expect_parses("../#{local_relative}", "path", "../#{local_relative}", Any) + expect_parses(".\\relative\\windows", "path", ".\\relative\\windows", Any) + expect_parses("..\\relative\\windows", "path", "..\\relative\\windows", Any) + # Path file schema + expect_parses("file://#{local_relative}", "path", local_relative, Any) + expect_parses("file://#{local_absolute}", "path", local_absolute, Any) + # Path resolver syntax + expect_parses("path:#{local_absolute}", "path", local_absolute, Any) + expect_parses("path:#{local_relative}", "path", local_relative, Any) + # Other resolvers short + expect_parses("git:git://git.example.org/crystal-library.git", "git", "git://git.example.org/crystal-library.git", Any) + end + end +end diff --git a/spec/unit/dependency_spec.cr b/spec/unit/dependency_spec.cr index 663cbfb7..90bb47b4 100644 --- a/spec/unit/dependency_spec.cr +++ b/spec/unit/dependency_spec.cr @@ -71,49 +71,6 @@ module Shards parse_dependency({foo: {git: "", tag: "rc-1.0"}}).to_s.should eq("foo (tag rc-1.0)") parse_dependency({foo: {git: "", commit: "4478d8afe8c728f44b47d3582a270423cd7fc07d"}}).to_s.should eq("foo (commit 4478d8a)") end - - it ".parts_from_cli" do - # GitHub short syntax - Dependency.parts_from_cli("github:foo/bar").should eq({resolver_key: "github", source: "foo/bar", requirement: Any}) - Dependency.parts_from_cli("github:Foo/Bar@1.2.3").should eq({resolver_key: "github", source: "Foo/Bar", requirement: VersionReq.new("~> 1.2.3")}) - - # GitHub urls - Dependency.parts_from_cli("https://github.com/foo/bar").should eq({resolver_key: "github", source: "foo/bar", requirement: Any}) - Dependency.parts_from_cli("https://github.com/Foo/Bar/commit/000000").should eq({resolver_key: "github", source: "Foo/Bar", requirement: GitCommitRef.new("000000")}) - Dependency.parts_from_cli("https://github.com/Foo/Bar/tree/v1.2.3").should eq({resolver_key: "github", source: "Foo/Bar", requirement: GitTagRef.new("v1.2.3")}) - Dependency.parts_from_cli("https://github.com/Foo/Bar/tree/some/branch").should eq({resolver_key: "github", source: "Foo/Bar", requirement: GitBranchRef.new("some/branch")}) - - # GitLab short syntax - Dependency.parts_from_cli("gitlab:foo/bar").should eq({resolver_key: "gitlab", source: "foo/bar", requirement: Any}) - - # GitLab urls - Dependency.parts_from_cli("https://gitlab.com/foo/bar").should eq({resolver_key: "gitlab", source: "foo/bar", requirement: Any}) - - # Bitbucket short syntax - Dependency.parts_from_cli("bitbucket:foo/bar").should eq({resolver_key: "bitbucket", source: "foo/bar", requirement: Any}) - - # bitbucket urls - Dependency.parts_from_cli("https://bitbucket.com/foo/bar").should eq({resolver_key: "bitbucket", source: "foo/bar", requirement: Any}) - - # Git convenient syntax since resolver matches scheme - Dependency.parts_from_cli("git://git.example.org/crystal-library.git").should eq({resolver_key: "git", source: "git://git.example.org/crystal-library.git", requirement: Any}) - - # Local paths - local_absolute = File.join(tmp_path, "local") - local_relative = File.join("spec", ".repositories", "local") # rel_path is relative to integration spec - Dir.mkdir_p(local_absolute) - - # Path short syntax - Dependency.parts_from_cli(local_absolute).should eq({resolver_key: "path", source: local_absolute, requirement: Any}) - Dependency.parts_from_cli(local_relative).should eq({resolver_key: "path", source: local_relative, requirement: Any}) - - # Path resolver syntax - Dependency.parts_from_cli("path:#{local_absolute}").should eq({resolver_key: "path", source: local_absolute, requirement: Any}) - Dependency.parts_from_cli("path:#{local_relative}").should eq({resolver_key: "path", source: local_relative, requirement: Any}) - - # Other resolvers short - Dependency.parts_from_cli("git:git://git.example.org/crystal-library.git").should eq({resolver_key: "git", source: "git://git.example.org/crystal-library.git", requirement: Any}) - end end end diff --git a/src/dependency.cr b/src/dependency.cr index 4a43b39d..25263d25 100644 --- a/src/dependency.cr +++ b/src/dependency.cr @@ -7,105 +7,8 @@ module Shards property name : String property resolver : Resolver property requirement : Requirement - # resolver's key and source are normalized. We preserve the key and source to be used - # in the shard.yml file in these field. This is used to generate the shard.yml file - # in a more human-readable way. - # A Dependency can still be created without them, but it will not be possible to - # generate the shard.yml file. - property! resolver_key : String - property! source : String - - def initialize(@name : String, @resolver : Resolver, @requirement : Requirement = Any, @resolver_key : String? = nil, @source : String? = nil) - end - - # Parse a dependency from a CLI argument - def self.from_cli(value : String) : Dependency - parts = parts_from_cli(value) - # We need to check the actual shard name to create a dependency. - # This requires getting the actual spec file from some matching version. - resolver = Resolver.find_resolver(parts[:resolver_key], "unknown", parts[:source]) - version = resolver.versions_for(parts[:requirement]).first || raise Shards::Error.new("No versions found for dependency: #{value}") - spec = resolver.spec(version) - name = spec.name || raise Shards::Error.new("No name found for dependency: #{value}") - - Dependency.new(name, resolver, parts[:requirement], parts[:resolver_key], parts[:source]) - end - - # :nodoc: - # - # Parse the dependency from a CLI argument - # and return the parts needed to create the proper dependency. - # - # Split to allow better unit testing. - def self.parts_from_cli(value : String) : {resolver_key: String, source: String, requirement: Requirement} - resolver_key = nil - source = "" - requirement = Any - - if File.directory?(value) - resolver_key = "path" - source = value - end - - if value.starts_with?("https://github.com") - resolver_key = "github" - uri = URI.parse(value) - source = uri.path[1..-1] # drop first "/"" - - components = source.split("/") - case components[2]? - when "commit" - source = "#{components[0]}/#{components[1]}" - requirement = GitCommitRef.new(components[3]) - when "tree" - source = "#{components[0]}/#{components[1]}" - requirement = if components[3].starts_with?("v") - GitTagRef.new(components[3]) - else - GitBranchRef.new(components[3..-1].join("/")) - end - end - end - - if value.starts_with?("https://gitlab.com") - resolver_key = "gitlab" - uri = URI.parse(value) - source = uri.path[1..-1] # drop first "/"" - end - - if value.starts_with?("https://bitbucket.com") - resolver_key = "bitbucket" - uri = URI.parse(value) - source = uri.path[1..-1] # drop first "/"" - end - - if value.starts_with?("git://") - resolver_key = "git" - source = value - end - - unless resolver_key - Resolver.resolver_keys.each do |key| - key_schema = "#{key}:" - if value.starts_with?(key_schema) - resolver_key = key - source = value.sub(key_schema, "") - - # narrow down requirement - if source.includes?("@") - source, version = source.split("@") - requirement = VersionReq.new("~> #{version}") - end - - break - end - end - end - - raise Shards::Error.new("Invalid dependency format: #{value}") unless resolver_key - - {resolver_key: resolver_key, source: source, requirement: requirement} + def initialize(@name : String, @resolver : Resolver, @requirement : Requirement = Any) end def self.from_yaml(pull : YAML::PullParser) @@ -151,16 +54,6 @@ module Shards end end - # Used to generate the shard.yml file. - def to_shard_yaml(yaml : YAML::Builder) - yaml.scalar name - yaml.mapping do - yaml.scalar resolver_key - yaml.scalar source - requirement.to_yaml(yaml) - end - end - def as_package? version = case req = @requirement diff --git a/src/dependency_definition.cr b/src/dependency_definition.cr new file mode 100644 index 00000000..0027f140 --- /dev/null +++ b/src/dependency_definition.cr @@ -0,0 +1,114 @@ +require "./dependency" + +module Shards + class DependencyDefinition + record Parts, resolver_key : String, source : String, requirement : Requirement + + property dependency : Dependency + # resolver's key and source are normalized. We preserve the key and source to be used + # in the shard.yml file in these field. This is used to generate the shard.yml file + # in a more human-readable way. + property resolver_key : String + property source : String + + def initialize(@dependency : Dependency, @resolver_key : String, @source : String) + end + + # Used to generate the shard.yml file. + def to_yaml(yaml : YAML::Builder) + yaml.scalar dependency.name + yaml.mapping do + yaml.scalar resolver_key + yaml.scalar source + dependency.requirement.to_yaml(yaml) + end + end + + # Parse a dependency from a CLI argument + def self.from_cli(value : String) : DependencyDefinition + parts = parts_from_cli(value) + + # We need to check the actual shard name to create a dependency. + # This requires getting the actual spec file from some matching version. + resolver = Resolver.find_resolver(parts.resolver_key, "unknown", parts.source) + version = resolver.versions_for(parts.requirement).first || raise Shards::Error.new("No versions found for dependency: #{value}") + spec = resolver.spec(version) + name = spec.name || raise Shards::Error.new("No name found for dependency: #{value}") + + DependencyDefinition.new(Dependency.new(name, resolver, parts.requirement), parts.resolver_key, parts.source) + end + + # :nodoc: + # + # Parse the dependency from a CLI argument + # and return the parts needed to create the proper dependency. + # + # Split to allow better unit testing. + def self.parts_from_cli(value : String) : Parts + resolver_key = nil + source = "" + requirement = Any + + if value.starts_with?("file://") + resolver_key = "path" + source = value[7..-1] # drop "file://" + end + + # relative paths + if value.starts_with?("./") || value.starts_with?("../") || value.starts_with?(".\\") || value.starts_with?("..\\") + resolver_key = "path" + source = value + end + + if value.starts_with?("https://github.com") + resolver_key = "github" + uri = URI.parse(value) + source = uri.path[1..-1].rchop(".git") # drop first "/"" + end + + if value.starts_with?("https://gitlab.com") + resolver_key = "gitlab" + uri = URI.parse(value) + source = uri.path[1..-1].rchop(".git") # drop first "/"" + end + + if value.starts_with?("https://bitbucket.com") + resolver_key = "bitbucket" + uri = URI.parse(value) + source = uri.path[1..-1] # drop first "/"" + end + + if value.starts_with?("git://") + resolver_key = "git" + source = value + end + + if value.starts_with?("git@") + resolver_key = "git" + source = value + end + + unless resolver_key + Resolver.resolver_keys.each do |key| + key_schema = "#{key}:" + if value.starts_with?(key_schema) + resolver_key = key + source = value.sub(key_schema, "") + + # narrow down requirement + if source.includes?("@") + source, version = source.split("@") + requirement = VersionReq.new("~> #{version}") + end + + break + end + end + end + + raise Shards::Error.new("Invalid dependency format: #{value}") unless resolver_key + + Parts.new(resolver_key: resolver_key, source: source, requirement: requirement) + end + end +end From 040cbdd489cff6a8afa80a37687abf3e3164dac0 Mon Sep 17 00:00:00 2001 From: "Brian J. Cardiff" Date: Fri, 16 May 2025 17:59:59 -0300 Subject: [PATCH 04/12] single logic for known git providers --- src/dependency_definition.cr | 18 +++--------------- src/resolvers/git.cr | 20 ++++++++++---------- 2 files changed, 13 insertions(+), 25 deletions(-) diff --git a/src/dependency_definition.cr b/src/dependency_definition.cr index 0027f140..821e087c 100644 --- a/src/dependency_definition.cr +++ b/src/dependency_definition.cr @@ -60,24 +60,12 @@ module Shards source = value end - if value.starts_with?("https://github.com") - resolver_key = "github" - uri = URI.parse(value) + uri = URI.parse(value) + if uri.scheme != "file" && uri.host && + (resolver_key = GitResolver::KNOWN_PROVIDERS[uri.host]?) source = uri.path[1..-1].rchop(".git") # drop first "/"" end - if value.starts_with?("https://gitlab.com") - resolver_key = "gitlab" - uri = URI.parse(value) - source = uri.path[1..-1].rchop(".git") # drop first "/"" - end - - if value.starts_with?("https://bitbucket.com") - resolver_key = "bitbucket" - uri = URI.parse(value) - source = uri.path[1..-1] # drop first "/"" - end - if value.starts_with?("git://") resolver_key = "git" source = value diff --git a/src/resolvers/git.cr b/src/resolvers/git.cr index 955c8d4f..be9ca0bb 100644 --- a/src/resolvers/git.cr +++ b/src/resolvers/git.cr @@ -100,15 +100,15 @@ module Shards "git" end - private KNOWN_PROVIDERS = { - "www.github.com", - "github.com", - "www.bitbucket.com", - "bitbucket.com", - "www.gitlab.com", - "gitlab.com", - "www.codeberg.org", - "codeberg.org", + KNOWN_PROVIDERS = { + "www.github.com" => "github", + "github.com" => "github", + "www.bitbucket.com" => "bitbucket", + "bitbucket.com" => "bitbucket", + "www.gitlab.com" => "gitlab", + "gitlab.com" => "gitlab", + "www.codeberg.org" => "codeberg", + "codeberg.org" => "codeberg", } def self.normalize_key_source(key : String, source : String) : {String, String} @@ -117,7 +117,7 @@ module Shards uri = URI.parse(source) downcased_host = uri.host.try &.downcase scheme = uri.scheme.try &.downcase - if scheme.in?("git", "http", "https") && downcased_host && downcased_host.in?(KNOWN_PROVIDERS) + if scheme.in?("git", "http", "https") && downcased_host && downcased_host.in?(KNOWN_PROVIDERS.keys) # browsers are requested to enforce HTTP Strict Transport Security uri.scheme = "https" downcased_path = uri.path.downcase From 53353dc6198f34e1438c9d669b63fbb50e5dce68 Mon Sep 17 00:00:00 2001 From: "Brian J. Cardiff" Date: Fri, 16 May 2025 18:05:05 -0300 Subject: [PATCH 05/12] normalize paths from windows to posix --- spec/unit/dependency_definition_spec.cr | 6 ++++-- src/dependency_definition.cr | 5 +++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/spec/unit/dependency_definition_spec.cr b/spec/unit/dependency_definition_spec.cr index 795d6f59..7e38574e 100644 --- a/spec/unit/dependency_definition_spec.cr +++ b/spec/unit/dependency_definition_spec.cr @@ -46,8 +46,10 @@ module Shards # Path short syntax expect_parses("./#{local_relative}", "path", "./#{local_relative}", Any) expect_parses("../#{local_relative}", "path", "../#{local_relative}", Any) - expect_parses(".\\relative\\windows", "path", ".\\relative\\windows", Any) - expect_parses("..\\relative\\windows", "path", "..\\relative\\windows", Any) + {% if flag?(:windows) %} + expect_parses(".\\relative\\windows", "path", "./relative/windows", Any) + expect_parses("..\\relative\\windows", "path", "../relative/windows", Any) + {% end %} # Path file schema expect_parses("file://#{local_relative}", "path", local_relative, Any) expect_parses("file://#{local_absolute}", "path", local_absolute, Any) diff --git a/src/dependency_definition.cr b/src/dependency_definition.cr index 821e087c..55b20250 100644 --- a/src/dependency_definition.cr +++ b/src/dependency_definition.cr @@ -55,9 +55,10 @@ module Shards end # relative paths - if value.starts_with?("./") || value.starts_with?("../") || value.starts_with?(".\\") || value.starts_with?("..\\") + path = Path[value].to_posix.to_s + if path.starts_with?("./") || path.starts_with?("../") resolver_key = "path" - source = value + source = path end uri = URI.parse(value) From 91c3656831cd21b1252c2a35c9372cc02b322b0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Tue, 20 May 2025 09:59:11 +0200 Subject: [PATCH 06/12] URI-based dependency parsing --- spec/unit/dependency_definition_spec.cr | 13 +++- src/dependency_definition.cr | 90 +++++++++++-------------- src/resolvers/resolver.cr | 6 +- 3 files changed, 54 insertions(+), 55 deletions(-) diff --git a/spec/unit/dependency_definition_spec.cr b/spec/unit/dependency_definition_spec.cr index 7e38574e..679c7458 100644 --- a/spec/unit/dependency_definition_spec.cr +++ b/spec/unit/dependency_definition_spec.cr @@ -35,6 +35,11 @@ module Shards # bitbucket urls expect_parses("https://bitbucket.com/foo/bar", "bitbucket", "foo/bar", Any) + # unknown https urls + expect_raises Shards::Error, "Cannot determine resolver for HTTPS URI" do + Shards::DependencyDefinition.parts_from_cli("https://example.com/foo/bar") + end + # Git convenient syntax since resolver matches scheme expect_parses("git://git.example.org/crystal-library.git", "git", "git://git.example.org/crystal-library.git", Any) expect_parses("git@example.org:foo/bar.git", "git", "git@example.org:foo/bar.git", Any) @@ -51,13 +56,19 @@ module Shards expect_parses("..\\relative\\windows", "path", "../relative/windows", Any) {% end %} # Path file schema - expect_parses("file://#{local_relative}", "path", local_relative, Any) + expect_raises Shards::Error, "Invalid file URI" do + Shards::DependencyDefinition.parts_from_cli("file://#{local_relative}") + end + expect_parses("file:#{local_relative}", "path", local_relative, Any) + expect_parses("file:#{local_absolute}", "path", local_absolute, Any) expect_parses("file://#{local_absolute}", "path", local_absolute, Any) # Path resolver syntax expect_parses("path:#{local_absolute}", "path", local_absolute, Any) expect_parses("path:#{local_relative}", "path", local_relative, Any) # Other resolvers short expect_parses("git:git://git.example.org/crystal-library.git", "git", "git://git.example.org/crystal-library.git", Any) + expect_parses("git+https://example.org/foo/bar", "git", "https://example.org/foo/bar", Any) + expect_parses("git:https://example.org/foo/bar", "git", "https://example.org/foo/bar", Any) end end end diff --git a/src/dependency_definition.cr b/src/dependency_definition.cr index 55b20250..bb207045 100644 --- a/src/dependency_definition.cr +++ b/src/dependency_definition.cr @@ -2,7 +2,7 @@ require "./dependency" module Shards class DependencyDefinition - record Parts, resolver_key : String, source : String, requirement : Requirement + record Parts, resolver_key : String, source : String, requirement : Requirement = Any property dependency : Dependency # resolver's key and source are normalized. We preserve the key and source to be used @@ -45,59 +45,51 @@ module Shards # # Split to allow better unit testing. def self.parts_from_cli(value : String) : Parts - resolver_key = nil - source = "" - requirement = Any - - if value.starts_with?("file://") - resolver_key = "path" - source = value[7..-1] # drop "file://" - end - - # relative paths - path = Path[value].to_posix.to_s - if path.starts_with?("./") || path.starts_with?("../") - resolver_key = "path" - source = path - end - uri = URI.parse(value) - if uri.scheme != "file" && uri.host && - (resolver_key = GitResolver::KNOWN_PROVIDERS[uri.host]?) - source = uri.path[1..-1].rchop(".git") # drop first "/"" - end - - if value.starts_with?("git://") - resolver_key = "git" - source = value - end - - if value.starts_with?("git@") - resolver_key = "git" - source = value - end - - unless resolver_key - Resolver.resolver_keys.each do |key| - key_schema = "#{key}:" - if value.starts_with?(key_schema) - resolver_key = key - source = value.sub(key_schema, "") - - # narrow down requirement - if source.includes?("@") - source, version = source.split("@") - requirement = VersionReq.new("~> #{version}") - end - break + case scheme = uri.scheme + when Nil + case value + when .starts_with?("./"), .starts_with?("../") + Parts.new("path", Path[value].to_posix.to_s) + when .starts_with?("git@") + Parts.new("git", value) + else + raise Shards::Error.new("Invalid dependency format: #{value}") + end + when "file" + raise Shards::Error.new("Invalid file URI: #{uri}") if !uri.host.in?(nil, "", "localhost") || uri.port || uri.user + Parts.new("path", uri.path) + when "https" + if resolver_key = GitResolver::KNOWN_PROVIDERS[uri.host]? + Parts.new(resolver_key, uri.path[1..-1].rchop(".git")) # drop first "/"" + else + raise Shards::Error.new("Cannot determine resolver for HTTPS URI: #{value}") + end + when "git" + if uri.host + Parts.new("git", uri.to_s) + else + Parts.new("git", uri.path) + end + when "git+https" + uri.scheme = "https" + Parts.new("git", uri.to_s) + else + if resolver_class = Resolver::RESOLVER_CLASSES[scheme]? + uri.scheme = nil + source = uri.to_s + # narrow down requirement + requirement = Any + if source.includes?("@") + source, version = source.split("@") + requirement = VersionReq.new("~> #{version}") end + + return Parts.new(scheme, source, requirement) end + raise Shards::Error.new("Invalid dependency format: #{value}") end - - raise Shards::Error.new("Invalid dependency format: #{value}") unless resolver_key - - Parts.new(resolver_key: resolver_key, source: source, requirement: requirement) end end end diff --git a/src/resolvers/resolver.cr b/src/resolvers/resolver.cr index b1069f5c..7e9fd462 100644 --- a/src/resolvers/resolver.cr +++ b/src/resolvers/resolver.cr @@ -100,13 +100,9 @@ module Shards end private record ResolverCacheKey, key : String, name : String, source : String - private RESOLVER_CLASSES = {} of String => Resolver.class + RESOLVER_CLASSES = {} of String => Resolver.class private RESOLVER_CACHE = {} of ResolverCacheKey => Resolver - def self.resolver_keys - RESOLVER_CLASSES.keys - end - def self.register_resolver(key, resolver) RESOLVER_CLASSES[key] = resolver end From 47ec20925f3003ab207e6f67d77e296163f62145 Mon Sep 17 00:00:00 2001 From: "Brian J. Cardiff" Date: Wed, 21 May 2025 16:58:34 -0300 Subject: [PATCH 07/12] Fix windows CI --- spec/unit/dependency_definition_spec.cr | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/spec/unit/dependency_definition_spec.cr b/spec/unit/dependency_definition_spec.cr index 679c7458..3276f806 100644 --- a/spec/unit/dependency_definition_spec.cr +++ b/spec/unit/dependency_definition_spec.cr @@ -49,11 +49,12 @@ module Shards local_relative = "an/relative/path" # Path short syntax - expect_parses("./#{local_relative}", "path", "./#{local_relative}", Any) expect_parses("../#{local_relative}", "path", "../#{local_relative}", Any) {% if flag?(:windows) %} expect_parses(".\\relative\\windows", "path", "./relative/windows", Any) expect_parses("..\\relative\\windows", "path", "../relative/windows", Any) + {% else %} + expect_parses("./#{local_relative}", "path", "./#{local_relative}", Any) {% end %} # Path file schema expect_raises Shards::Error, "Invalid file URI" do From ac81d3cca6a01826934b42308aac3dbe6bcd22a6 Mon Sep 17 00:00:00 2001 From: "Brian J. Cardiff" Date: Mon, 26 May 2025 10:06:23 -0300 Subject: [PATCH 08/12] Fix windows CI (take 2) --- src/dependency_definition.cr | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/dependency_definition.cr b/src/dependency_definition.cr index bb207045..90fa2a88 100644 --- a/src/dependency_definition.cr +++ b/src/dependency_definition.cr @@ -52,6 +52,12 @@ module Shards case value when .starts_with?("./"), .starts_with?("../") Parts.new("path", Path[value].to_posix.to_s) + when .starts_with?(".\\"), .starts_with?("..\\") + {% if flag?(:windows) %} + Parts.new("path", Path[value].to_posix.to_s) + {% else %} + raise Shards::Error.new("Invalid dependency format: #{value}") + {% end %} when .starts_with?("git@") Parts.new("git", value) else From d03ae53afeaa7e0ddd5f0d5dbffb214d4c4e16c4 Mon Sep 17 00:00:00 2001 From: "Brian J. Cardiff" Date: Mon, 26 May 2025 11:40:15 -0300 Subject: [PATCH 09/12] Update dependency_definition.cr MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Johannes Müller --- src/dependency_definition.cr | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/src/dependency_definition.cr b/src/dependency_definition.cr index 90fa2a88..e5697129 100644 --- a/src/dependency_definition.cr +++ b/src/dependency_definition.cr @@ -72,18 +72,13 @@ module Shards else raise Shards::Error.new("Cannot determine resolver for HTTPS URI: #{value}") end - when "git" - if uri.host - Parts.new("git", uri.to_s) - else - Parts.new("git", uri.path) - end - when "git+https" - uri.scheme = "https" - Parts.new("git", uri.to_s) else - if resolver_class = Resolver::RESOLVER_CLASSES[scheme]? - uri.scheme = nil + scheme, _, subscheme = scheme.partition('+') + subscheme = subscheme.presence + if Resolver.find_class(scheme) + if uri.host.nil? || subscheme + uri.scheme = subscheme + end source = uri.to_s # narrow down requirement requirement = Any From 05f3d9760032c8bc7cd3b8ae63e1f03eedcce885 Mon Sep 17 00:00:00 2001 From: "Brian J. Cardiff" Date: Sun, 8 Jun 2025 00:43:42 -0300 Subject: [PATCH 10/12] Use # instead of @ --- spec/unit/dependency_definition_spec.cr | 2 +- src/dependency_definition.cr | 8 +++----- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/spec/unit/dependency_definition_spec.cr b/spec/unit/dependency_definition_spec.cr index 3276f806..33ce6abf 100644 --- a/spec/unit/dependency_definition_spec.cr +++ b/spec/unit/dependency_definition_spec.cr @@ -10,7 +10,7 @@ module Shards it ".parts_from_cli" do # GitHub short syntax expect_parses("github:foo/bar", "github", "foo/bar", Any) - expect_parses("github:Foo/Bar@1.2.3", "github", "Foo/Bar", VersionReq.new("~> 1.2.3")) + expect_parses("github:Foo/Bar#1.2.3", "github", "Foo/Bar", VersionReq.new("~> 1.2.3")) # GitHub urls expect_parses("https://github.com/foo/bar", "github", "foo/bar", Any) diff --git a/src/dependency_definition.cr b/src/dependency_definition.cr index e5697129..6c1fe32a 100644 --- a/src/dependency_definition.cr +++ b/src/dependency_definition.cr @@ -79,15 +79,13 @@ module Shards if uri.host.nil? || subscheme uri.scheme = subscheme end - source = uri.to_s # narrow down requirement requirement = Any - if source.includes?("@") - source, version = source.split("@") + if version = uri.fragment + uri.fragment = nil requirement = VersionReq.new("~> #{version}") end - - return Parts.new(scheme, source, requirement) + return Parts.new(scheme, uri.to_s, requirement) end raise Shards::Error.new("Invalid dependency format: #{value}") end From 05483f28098c01e0d37cd3ecf574e59ff345104c Mon Sep 17 00:00:00 2001 From: "Brian J. Cardiff" Date: Sun, 8 Jun 2025 00:53:59 -0300 Subject: [PATCH 11/12] Add # syntax to other source schemes --- spec/unit/dependency_definition_spec.cr | 4 ++++ src/dependency_definition.cr | 26 +++++++++++++------------ 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/spec/unit/dependency_definition_spec.cr b/spec/unit/dependency_definition_spec.cr index 33ce6abf..da4f0bb9 100644 --- a/spec/unit/dependency_definition_spec.cr +++ b/spec/unit/dependency_definition_spec.cr @@ -14,13 +14,17 @@ module Shards # GitHub urls expect_parses("https://github.com/foo/bar", "github", "foo/bar", Any) + expect_parses("https://github.com/foo/bar#1.2.3", "github", "foo/bar", VersionReq.new("~> 1.2.3")) # GitHub urls from clone popup expect_parses("https://github.com/foo/bar.git", "github", "foo/bar", Any) + expect_parses("https://github.com/foo/bar.git#1.2.3", "github", "foo/bar", VersionReq.new("~> 1.2.3")) expect_parses("git@github.com:foo/bar.git", "git", "git@github.com:foo/bar.git", Any) + expect_parses("git@github.com:foo/bar.git#1.2.3", "git", "git@github.com:foo/bar.git", VersionReq.new("~> 1.2.3")) # GitLab short syntax expect_parses("gitlab:foo/bar", "gitlab", "foo/bar", Any) + expect_parses("gitlab:foo/bar#1.2.3", "gitlab", "foo/bar", VersionReq.new("~> 1.2.3")) # GitLab urls expect_parses("https://gitlab.com/foo/bar", "gitlab", "foo/bar", Any) diff --git a/src/dependency_definition.cr b/src/dependency_definition.cr index 6c1fe32a..0eae25c0 100644 --- a/src/dependency_definition.cr +++ b/src/dependency_definition.cr @@ -2,7 +2,7 @@ require "./dependency" module Shards class DependencyDefinition - record Parts, resolver_key : String, source : String, requirement : Requirement = Any + record Parts, resolver_key : String, source : String, requirement : Requirement property dependency : Dependency # resolver's key and source are normalized. We preserve the key and source to be used @@ -47,28 +47,36 @@ module Shards def self.parts_from_cli(value : String) : Parts uri = URI.parse(value) + # fragment parsing for version requirement + requirement = Any + if fragment = uri.fragment + uri.fragment = nil + value = value.rchop("##{fragment}") + requirement = VersionReq.new("~> #{fragment}") + end + case scheme = uri.scheme when Nil case value when .starts_with?("./"), .starts_with?("../") - Parts.new("path", Path[value].to_posix.to_s) + Parts.new("path", Path[value].to_posix.to_s, Any) when .starts_with?(".\\"), .starts_with?("..\\") {% if flag?(:windows) %} - Parts.new("path", Path[value].to_posix.to_s) + Parts.new("path", Path[value].to_posix.to_s, Any) {% else %} raise Shards::Error.new("Invalid dependency format: #{value}") {% end %} when .starts_with?("git@") - Parts.new("git", value) + Parts.new("git", value, requirement) else raise Shards::Error.new("Invalid dependency format: #{value}") end when "file" raise Shards::Error.new("Invalid file URI: #{uri}") if !uri.host.in?(nil, "", "localhost") || uri.port || uri.user - Parts.new("path", uri.path) + Parts.new("path", uri.path, Any) when "https" if resolver_key = GitResolver::KNOWN_PROVIDERS[uri.host]? - Parts.new(resolver_key, uri.path[1..-1].rchop(".git")) # drop first "/"" + Parts.new(resolver_key, uri.path[1..-1].rchop(".git"), requirement) # drop first "/"" else raise Shards::Error.new("Cannot determine resolver for HTTPS URI: #{value}") end @@ -79,12 +87,6 @@ module Shards if uri.host.nil? || subscheme uri.scheme = subscheme end - # narrow down requirement - requirement = Any - if version = uri.fragment - uri.fragment = nil - requirement = VersionReq.new("~> #{version}") - end return Parts.new(scheme, uri.to_s, requirement) end raise Shards::Error.new("Invalid dependency format: #{value}") From 47831a1648bff46f0263967b3fd226a12cd184d9 Mon Sep 17 00:00:00 2001 From: "Brian J. Cardiff" Date: Wed, 11 Jun 2025 11:24:45 -0300 Subject: [PATCH 12/12] Make # be exact version --- spec/unit/dependency_definition_spec.cr | 10 +++++----- src/dependency_definition.cr | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/spec/unit/dependency_definition_spec.cr b/spec/unit/dependency_definition_spec.cr index da4f0bb9..9e43facc 100644 --- a/spec/unit/dependency_definition_spec.cr +++ b/spec/unit/dependency_definition_spec.cr @@ -10,21 +10,21 @@ module Shards it ".parts_from_cli" do # GitHub short syntax expect_parses("github:foo/bar", "github", "foo/bar", Any) - expect_parses("github:Foo/Bar#1.2.3", "github", "Foo/Bar", VersionReq.new("~> 1.2.3")) + expect_parses("github:Foo/Bar#1.2.3", "github", "Foo/Bar", Version.new("1.2.3")) # GitHub urls expect_parses("https://github.com/foo/bar", "github", "foo/bar", Any) - expect_parses("https://github.com/foo/bar#1.2.3", "github", "foo/bar", VersionReq.new("~> 1.2.3")) + expect_parses("https://github.com/foo/bar#1.2.3", "github", "foo/bar", Version.new("1.2.3")) # GitHub urls from clone popup expect_parses("https://github.com/foo/bar.git", "github", "foo/bar", Any) - expect_parses("https://github.com/foo/bar.git#1.2.3", "github", "foo/bar", VersionReq.new("~> 1.2.3")) + expect_parses("https://github.com/foo/bar.git#1.2.3", "github", "foo/bar", Version.new("1.2.3")) expect_parses("git@github.com:foo/bar.git", "git", "git@github.com:foo/bar.git", Any) - expect_parses("git@github.com:foo/bar.git#1.2.3", "git", "git@github.com:foo/bar.git", VersionReq.new("~> 1.2.3")) + expect_parses("git@github.com:foo/bar.git#1.2.3", "git", "git@github.com:foo/bar.git", Version.new("1.2.3")) # GitLab short syntax expect_parses("gitlab:foo/bar", "gitlab", "foo/bar", Any) - expect_parses("gitlab:foo/bar#1.2.3", "gitlab", "foo/bar", VersionReq.new("~> 1.2.3")) + expect_parses("gitlab:foo/bar#1.2.3", "gitlab", "foo/bar", Version.new("1.2.3")) # GitLab urls expect_parses("https://gitlab.com/foo/bar", "gitlab", "foo/bar", Any) diff --git a/src/dependency_definition.cr b/src/dependency_definition.cr index 0eae25c0..f97fa5af 100644 --- a/src/dependency_definition.cr +++ b/src/dependency_definition.cr @@ -52,7 +52,7 @@ module Shards if fragment = uri.fragment uri.fragment = nil value = value.rchop("##{fragment}") - requirement = VersionReq.new("~> #{fragment}") + requirement = Version.new(fragment) end case scheme = uri.scheme