From 7172207468bf1c9ab417c350e428e02b877f25a2 Mon Sep 17 00:00:00 2001 From: James Tucker Date: Mon, 31 May 2021 22:24:39 -0700 Subject: [PATCH 01/10] gems: update gemfile for recent bundler --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index f2557c91..ed6f6645 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -82,4 +82,4 @@ DEPENDENCIES simplecov (~> 0.16.1) BUNDLED WITH - 1.17.3 + 2.2.19 From d025965e225e34884955a3448af7d8b55f1ce397 Mon Sep 17 00:00:00 2001 From: James Tucker Date: Mon, 31 May 2021 22:27:45 -0700 Subject: [PATCH 02/10] secp256k1: fix repo path and update upstream hash --- Rakefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Rakefile b/Rakefile index 3f3dfbea..c6e59b7f 100644 --- a/Rakefile +++ b/Rakefile @@ -2,7 +2,7 @@ require 'bundler/gem_tasks' require 'rspec/core/rake_task' # libsecp256k1 repository URL -LIBSECP256K1_REPO = 'https://github.com/bitcoin-core/secp256k1/'.freeze +LIBSECP256K1_REPO = 'https://github.com/bitcoin-core/secp256k1.git'.freeze # Folder into which libsecp256k1 repository is cloned LIBSECP256K1_PATH = 'secp256k1'.freeze @@ -34,8 +34,8 @@ end desc 'Compiles the libsecp256k1 library' task :build_libsecp256k1, [:force] do |_, args| - # Commit hash for libsecp256k1 from May 31, 2018. - COMMIT_HASH = '1e6f1f5ad5e7f1e3ef79313ec02023902bf8175c'.freeze + # Commit hash for libsecp256k1 from March 31, 2021. + COMMIT_HASH = '50f33677122fed79dedb05e8046b2fea93496201'.freeze force = args[:force] From 9dd02dbbcfc39239eb23cf5e0e3b3b717bbdae79 Mon Sep 17 00:00:00 2001 From: James Tucker Date: Mon, 31 May 2021 22:40:39 -0700 Subject: [PATCH 03/10] gemfile: declare dependency on scanf This is often bundled with ruby, but not always. --- Gemfile | 1 + Gemfile.lock | 2 ++ 2 files changed, 3 insertions(+) diff --git a/Gemfile b/Gemfile index 8f40ccc7..04492327 100644 --- a/Gemfile +++ b/Gemfile @@ -9,6 +9,7 @@ group :test do gem 'rubocop', '~> 0.58.2' gem 'simplecov', '~> 0.16.1' gem 'minitest', '~> 5.11.3' + gem 'scanf' end group :development do diff --git a/Gemfile.lock b/Gemfile.lock index ed6f6645..ab8670ad 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -58,6 +58,7 @@ GEM ruby-progressbar (~> 1.7) unicode-display_width (~> 1.0, >= 1.0.1) ruby-progressbar (1.9.0) + scanf (1.0.0) scrypt (3.0.5) ffi-compiler (>= 1.0, < 2.0) simplecov (0.16.1) @@ -79,6 +80,7 @@ DEPENDENCIES rake (~> 12.3.1) rspec (~> 3.7.0) rubocop (~> 0.58.2) + scanf simplecov (~> 0.16.1) BUNDLED WITH From f04ad572b103f8dcc61c94dc0aa460d538794d9f Mon Sep 17 00:00:00 2001 From: James Tucker Date: Mon, 31 May 2021 22:41:53 -0700 Subject: [PATCH 04/10] gemfile: update dependencies No more warnings, yay. --- Gemfile | 16 ++++---- Gemfile.lock | 107 ++++++++++++++++++++++++++------------------------- 2 files changed, 63 insertions(+), 60 deletions(-) diff --git a/Gemfile b/Gemfile index 04492327..b3e03b7b 100644 --- a/Gemfile +++ b/Gemfile @@ -3,16 +3,16 @@ source "https://rubygems.org" gemspec group :test do - gem 'rake', '~> 12.3.1' - gem 'bacon', '~> 1.2.0' - gem 'rspec', '~> 3.7.0' - gem 'rubocop', '~> 0.58.2' - gem 'simplecov', '~> 0.16.1' - gem 'minitest', '~> 5.11.3' + gem 'rake' + gem 'bacon' + gem 'rspec' + gem 'rubocop' + gem 'simplecov' + gem 'minitest' gem 'scanf' end group :development do - gem 'pry', '~> 0.11.3' - gem 'pry-byebug', '~> 3.6.0' + gem 'pry' + gem 'pry-byebug' end diff --git a/Gemfile.lock b/Gemfile.lock index ab8670ad..7fe38586 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -9,79 +9,82 @@ PATH GEM remote: https://rubygems.org/ specs: - ast (2.4.0) + ast (2.4.2) bacon (1.2.0) - byebug (10.0.2) - coderay (1.1.2) - diff-lcs (1.3) - docile (1.3.1) + byebug (11.1.3) + coderay (1.1.3) + diff-lcs (1.4.4) + docile (1.4.0) eventmachine (1.2.7) - ffi (1.9.25) + ffi (1.15.1) ffi-compiler (1.0.1) ffi (>= 1.0.0) rake - jaro_winkler (1.5.1) - json (2.1.0) - method_source (0.9.0) - minitest (5.11.3) - parallel (1.12.1) - parser (2.5.1.2) - ast (~> 2.4.0) - powerpack (0.1.2) - pry (0.11.3) - coderay (~> 1.1.0) - method_source (~> 0.9.0) - pry-byebug (3.6.0) - byebug (~> 10.0) - pry (~> 0.10) + method_source (1.0.0) + minitest (5.14.4) + parallel (1.20.1) + parser (3.0.1.1) + ast (~> 2.4.1) + pry (0.13.1) + coderay (~> 1.1) + method_source (~> 1.0) + pry-byebug (3.9.0) + byebug (~> 11.0) + pry (~> 0.13.0) rainbow (3.0.0) - rake (12.3.1) - rspec (3.7.0) - rspec-core (~> 3.7.0) - rspec-expectations (~> 3.7.0) - rspec-mocks (~> 3.7.0) - rspec-core (3.7.1) - rspec-support (~> 3.7.0) - rspec-expectations (3.7.0) + rake (13.0.3) + regexp_parser (2.1.1) + rexml (3.2.5) + rspec (3.10.0) + rspec-core (~> 3.10.0) + rspec-expectations (~> 3.10.0) + rspec-mocks (~> 3.10.0) + rspec-core (3.10.1) + rspec-support (~> 3.10.0) + rspec-expectations (3.10.1) diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.7.0) - rspec-mocks (3.7.0) + rspec-support (~> 3.10.0) + rspec-mocks (3.10.2) diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.7.0) - rspec-support (3.7.1) - rubocop (0.58.2) - jaro_winkler (~> 1.5.1) + rspec-support (~> 3.10.0) + rspec-support (3.10.2) + rubocop (1.15.0) parallel (~> 1.10) - parser (>= 2.5, != 2.5.1.1) - powerpack (~> 0.1) + parser (>= 3.0.0.0) rainbow (>= 2.2.2, < 4.0) + regexp_parser (>= 1.8, < 3.0) + rexml + rubocop-ast (>= 1.5.0, < 2.0) ruby-progressbar (~> 1.7) - unicode-display_width (~> 1.0, >= 1.0.1) - ruby-progressbar (1.9.0) + unicode-display_width (>= 1.4.0, < 3.0) + rubocop-ast (1.7.0) + parser (>= 3.0.1.1) + ruby-progressbar (1.11.0) scanf (1.0.0) - scrypt (3.0.5) + scrypt (3.0.7) ffi-compiler (>= 1.0, < 2.0) - simplecov (0.16.1) + simplecov (0.21.2) docile (~> 1.1) - json (>= 1.8, < 3) - simplecov-html (~> 0.10.0) - simplecov-html (0.10.2) - unicode-display_width (1.4.0) + simplecov-html (~> 0.11) + simplecov_json_formatter (~> 0.1) + simplecov-html (0.12.3) + simplecov_json_formatter (0.1.3) + unicode-display_width (2.0.0) PLATFORMS ruby DEPENDENCIES - bacon (~> 1.2.0) + bacon bitcoin-ruby! - minitest (~> 5.11.3) - pry (~> 0.11.3) - pry-byebug (~> 3.6.0) - rake (~> 12.3.1) - rspec (~> 3.7.0) - rubocop (~> 0.58.2) + minitest + pry + pry-byebug + rake + rspec + rubocop scanf - simplecov (~> 0.16.1) + simplecov BUNDLED WITH 2.2.19 From b680d3fbffcd3158ec3381afbd1a183479ae2ca7 Mon Sep 17 00:00:00 2001 From: James Tucker Date: Mon, 31 May 2021 22:38:29 -0700 Subject: [PATCH 05/10] scrypt: update to new function name --- lib/bitcoin.rb | 2 +- lib/bitcoin/key.rb | 4 ++-- lib/bitcoin/litecoin.rb | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/bitcoin.rb b/lib/bitcoin.rb index 8848a130..53e0a0bc 100644 --- a/lib/bitcoin.rb +++ b/lib/bitcoin.rb @@ -325,7 +325,7 @@ def litecoin_hash(hex) bytes = [hex].pack("H*").reverse begin require "scrypt" unless defined?(::SCrypt) - hash = SCrypt::Engine.__sc_crypt(bytes, bytes, 1024, 1, 1, 32) + hash = SCrypt::Engine.scrypt(bytes, bytes, 1024, 1, 1, 32) rescue LoadError hash = Litecoin::Scrypt.scrypt_1024_1_1_256_sp(bytes) end diff --git a/lib/bitcoin/key.rb b/lib/bitcoin/key.rb index f8c6a855..1a46f9c7 100644 --- a/lib/bitcoin/key.rb +++ b/lib/bitcoin/key.rb @@ -183,7 +183,7 @@ def to_bip38(passphrase) addresshash = Digest::SHA256.digest( Digest::SHA256.digest( self.addr ) )[0...4] require 'scrypt' unless defined?(::SCrypt::Engine) - buf = SCrypt::Engine.__sc_crypt(passphrase, addresshash, 16384, 8, 8, 64) + buf = SCrypt::Engine.scrypt(passphrase, addresshash, 16384, 8, 8, 64) derivedhalf1, derivedhalf2 = buf[0...32], buf[32..-1] aes = proc{|k,a,b| @@ -212,7 +212,7 @@ def self.from_bip38(encrypted_privkey, passphrase) raise "Invalid checksum" unless Digest::SHA256.digest(Digest::SHA256.digest(version + flagbyte + addresshash + encryptedhalf1 + encryptedhalf2))[0...4] == checksum require 'scrypt' unless defined?(::SCrypt::Engine) - buf = SCrypt::Engine.__sc_crypt(passphrase, addresshash, 16384, 8, 8, 64) + buf = SCrypt::Engine.scrypt(passphrase, addresshash, 16384, 8, 8, 64) derivedhalf1, derivedhalf2 = buf[0...32], buf[32..-1] aes = proc{|k,a| diff --git a/lib/bitcoin/litecoin.rb b/lib/bitcoin/litecoin.rb index cfa88b79..55b685ae 100644 --- a/lib/bitcoin/litecoin.rb +++ b/lib/bitcoin/litecoin.rb @@ -68,7 +68,7 @@ def xor_salsa8(a, b, a_offset, b_offset) begin require "scrypt" - hash = SCrypt::Engine.__sc_crypt(secret_bytes, secret_bytes, 1024, 1, 1, 32) + hash = SCrypt::Engine.scrypt(secret_bytes, secret_bytes, 1024, 1, 1, 32) p hash.reverse.unpack("H*")[0] == "00000000002bef4107f882f6115e0b01f348d21195dacd3582aa2dabd7985806" rescue LoadError puts "scrypt gem not found, using native scrypt" @@ -77,7 +77,7 @@ def xor_salsa8(a, b, a_offset, b_offset) require 'benchmark' Benchmark.bmbm{|x| - x.report("v1"){ SCrypt::Engine.__sc_crypt(secret_bytes, secret_bytes, 1024, 1, 1, 32).reverse.unpack("H*") rescue nil } + x.report("v1"){ SCrypt::Engine.scrypt(secret_bytes, secret_bytes, 1024, 1, 1, 32).reverse.unpack("H*") rescue nil } x.report("v2"){ Litecoin::Scrypt.scrypt_1024_1_1_256_sp(secret_bytes).reverse.unpack("H*")[0] } } end From 0b3164ceafd1deb01f0ac12c75ab1d9495d04b81 Mon Sep 17 00:00:00 2001 From: James Tucker Date: Mon, 31 May 2021 22:43:31 -0700 Subject: [PATCH 06/10] git: add secp256k1.so to ignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 71814124..a52fec4f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +secp256k1.so *.gem .bundle pkg/* From 9f391d68877a6434688c02c5b372721c44aded52 Mon Sep 17 00:00:00 2001 From: James Tucker Date: Mon, 31 May 2021 22:53:52 -0700 Subject: [PATCH 07/10] ci: add github action over ubuntu macos is disabled for now, as it's crashy, as per patch trigger. --- .github/workflows/ruby.yml | 42 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 .github/workflows/ruby.yml diff --git a/.github/workflows/ruby.yml b/.github/workflows/ruby.yml new file mode 100644 index 00000000..04d308f5 --- /dev/null +++ b/.github/workflows/ruby.yml @@ -0,0 +1,42 @@ +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. +# This workflow will download a prebuilt Ruby version, install dependencies and run tests with Rake +# For more information see: https://github.com/marketplace/actions/setup-ruby-jruby-and-truffleruby + +name: Ruby + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + test: + + runs-on: ${{ matrix.os }} + strategy: + matrix: + # os: [ubuntu-latest, macos-latest] + os: [ubuntu-latest] + ruby-version: ['2.6', '2.7', '3.0'] + + steps: + - uses: actions/checkout@v2 + - name: Brew automake + run: brew install automake + if: matrix.os == 'macos-latest' + - name: Set up Ruby + # To automatically get bug fixes and new Ruby versions for ruby/setup-ruby, + # change this to (see https://github.com/ruby/setup-ruby#versioning): + # uses: ruby/setup-ruby@v1 + uses: ruby/setup-ruby@473e4d8fe5dd94ee328fdfca9f8c9c7afc9dae5e + with: + ruby-version: ${{ matrix.ruby-version }} + bundler-cache: true # runs 'bundle install' and caches installed gems automatically + - name: Build secp256k1 + run: bundle exec rake build_libsecp256k1 + - name: Run tests + run: bundle exec rake From cafb20b93041fa4680e72f7fdfd5c31bed0d0cb8 Mon Sep 17 00:00:00 2001 From: James Tucker Date: Tue, 1 Jun 2021 22:12:40 -0700 Subject: [PATCH 08/10] openssl: make compatible with libressl LibreSSL uses a mostly OpenSSL 1.0.1 compatible API, with the alarmingly notable exception of the version API. When the library is loaded on macOS with LibreSSL the verison check introduced in #42 the gem was incompatible with LibreSSL. This change may not be ideally forward-compatible if APIs change and may need a new strategy eventually, but is not noticeably weaker than the version based strategy. It is notably functional on both macOS with LibreSSL 2.8.3, as well as remaining compatible with OpenSSL 1.1.1f on Ubuntu. --- .github/workflows/ruby.yml | 3 +-- lib/bitcoin/ffi/openssl.rb | 22 ++++------------------ 2 files changed, 5 insertions(+), 20 deletions(-) diff --git a/.github/workflows/ruby.yml b/.github/workflows/ruby.yml index 04d308f5..01e8c59e 100644 --- a/.github/workflows/ruby.yml +++ b/.github/workflows/ruby.yml @@ -19,8 +19,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - # os: [ubuntu-latest, macos-latest] - os: [ubuntu-latest] + os: [ubuntu-latest, macos-latest] ruby-version: ['2.6', '2.7', '3.0'] steps: diff --git a/lib/bitcoin/ffi/openssl.rb b/lib/bitcoin/ffi/openssl.rb index d3abb316..685cc124 100644 --- a/lib/bitcoin/ffi/openssl.rb +++ b/lib/bitcoin/ffi/openssl.rb @@ -11,6 +11,7 @@ module OpenSSL_EC # rubocop:disable Naming/ClassAndModuleCamelCase ffi_lib 'libeay32', 'ssleay32' else ffi_lib [ + FFI::CURRENT_PROCESS, 'libssl.so.1.1.0', 'libssl.so.1.1', 'libssl.so.1.0.0', 'libssl.so.10', 'ssl' @@ -21,10 +22,6 @@ module OpenSSL_EC # rubocop:disable Naming/ClassAndModuleCamelCase POINT_CONVERSION_COMPRESSED = 2 POINT_CONVERSION_UNCOMPRESSED = 4 - # OpenSSL 1.1.0 version as a numerical version value as defined in: - # https://www.openssl.org/docs/man1.1.0/man3/OpenSSL_version.html - VERSION_1_1_0_NUM = 0x10100000 - # OpenSSL 1.1.0 engine constants, taken from: # https://github.com/openssl/openssl/blob/2be8c56a39b0ec2ec5af6ceaf729df154d784a43/include/openssl/crypto.h OPENSSL_INIT_ENGINE_RDRAND = 0x00000200 @@ -52,21 +49,10 @@ module OpenSSL_EC # rubocop:disable Naming/ClassAndModuleCamelCase attach_function :SSLeay, [], :long end - # Returns the version of SSL present. - # - # @return [Integer] version number as an integer. - def self.version - if self.respond_to?(:OpenSSL_version_num) - OpenSSL_version_num() - else - SSLeay() - end - end - - if version >= VERSION_1_1_0_NUM + begin # Initialization procedure for the library was changed in OpenSSL 1.1.0 attach_function :OPENSSL_init_ssl, [:uint64, :pointer], :int - else + rescue FFI::NotFoundError attach_function :SSL_library_init, [], :int attach_function :ERR_load_crypto_strings, [], :void attach_function :SSL_load_error_strings, [], :void @@ -391,7 +377,7 @@ def self.init_ffi_ssl @ssl_loaded ||= false return if @ssl_loaded - if version >= VERSION_1_1_0_NUM + if self.method_defined?(:OPENSSL_init_ssl) OPENSSL_init_ssl( OPENSSL_INIT_LOAD_SSL_STRINGS | OPENSSL_INIT_ENGINE_ALL_BUILTIN, nil From 953269325d3b6a1fe87fd10569cb7bc78cd5f0a2 Mon Sep 17 00:00:00 2001 From: James Tucker Date: Tue, 1 Jun 2021 22:20:44 -0700 Subject: [PATCH 09/10] ci: add previous OS version dimensions --- .github/workflows/ruby.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ruby.yml b/.github/workflows/ruby.yml index 01e8c59e..b4236a97 100644 --- a/.github/workflows/ruby.yml +++ b/.github/workflows/ruby.yml @@ -15,18 +15,16 @@ on: jobs: test: - runs-on: ${{ matrix.os }} strategy: + fail-fast: false + matrix: - os: [ubuntu-latest, macos-latest] + os: [ubuntu-latest, macos-latest, ubuntu-18.04, macos-10.15] ruby-version: ['2.6', '2.7', '3.0'] steps: - uses: actions/checkout@v2 - - name: Brew automake - run: brew install automake - if: matrix.os == 'macos-latest' - name: Set up Ruby # To automatically get bug fixes and new Ruby versions for ruby/setup-ruby, # change this to (see https://github.com/ruby/setup-ruby#versioning): @@ -35,6 +33,9 @@ jobs: with: ruby-version: ${{ matrix.ruby-version }} bundler-cache: true # runs 'bundle install' and caches installed gems automatically + - name: Brew automake + run: brew install automake + if: startsWith(matrix.os, 'macOS') - name: Build secp256k1 run: bundle exec rake build_libsecp256k1 - name: Run tests From ee22140522b1ee75a363ff7aaa56d1925d5bde9d Mon Sep 17 00:00:00 2001 From: James Tucker Date: Tue, 1 Jun 2021 23:09:11 -0700 Subject: [PATCH 10/10] ffi: don't dlopen extra libraries On macOS the library load is handled in a specific and incompatible way, which is handled by the extension. As the gem uses the extension elsewhere, taking a direct dependency on that, and looking up symbols from in-process should be safe and sufficient. --- lib/bitcoin/ffi/openssl.rb | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/lib/bitcoin/ffi/openssl.rb b/lib/bitcoin/ffi/openssl.rb index 685cc124..30e187b4 100644 --- a/lib/bitcoin/ffi/openssl.rb +++ b/lib/bitcoin/ffi/openssl.rb @@ -1,5 +1,6 @@ # encoding: ascii-8bit +require 'openssl' require 'ffi' module Bitcoin @@ -7,16 +8,9 @@ module Bitcoin # ported from: https://github.com/sipa/bitcoin/blob/2d40fe4da9ea82af4b652b691a4185431d6e47a8/key.h module OpenSSL_EC # rubocop:disable Naming/ClassAndModuleCamelCase extend FFI::Library - if FFI::Platform.windows? - ffi_lib 'libeay32', 'ssleay32' - else - ffi_lib [ - FFI::CURRENT_PROCESS, - 'libssl.so.1.1.0', 'libssl.so.1.1', - 'libssl.so.1.0.0', 'libssl.so.10', - 'ssl' - ] - end + + # Use the library loaded by the extension require above. + ffi_lib FFI::CURRENT_PROCESS NID_secp256k1 = 714 # rubocop:disable Naming/ConstantName POINT_CONVERSION_COMPRESSED = 2