From 36db569c145dbe208b6b0b0f38775cd8787dfb76 Mon Sep 17 00:00:00 2001 From: Jason Lunn Date: Thu, 18 Apr 2024 13:22:13 -0400 Subject: [PATCH 1/7] Evaluate a fix for FFI --- .github/workflows/test_ruby.yml | 12 ++++-------- ruby/Gemfile | 2 ++ 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test_ruby.yml b/.github/workflows/test_ruby.yml index 38cb5ef5b236..9a57cdb0ec26 100644 --- a/.github/workflows/test_ruby.yml +++ b/.github/workflows/test_ruby.yml @@ -20,13 +20,11 @@ jobs: # Test both FFI and Native implementations on the highest and lowest # Ruby versions for CRuby and JRuby, but only on Bazel 5.x. - { name: Ruby 3.0, ruby: ruby-3.0.2, ffi: NATIVE } - # TODO Re-enable these once flakes are fixed - #- { name: Ruby 3.0, ruby: ruby-3.0.2, ffi: FFI } + - { name: Ruby 3.0, ruby: ruby-3.0.2, ffi: FFI } - { name: Ruby 3.1, ruby: ruby-3.1.0 } - { name: Ruby 3.2, ruby: ruby-3.2.0 } - { name: Ruby 3.3, ruby: ruby-3.3.0, ffi: NATIVE } - # TODO Re-enable these once flakes are fixed - #- { name: Ruby 3.3, ruby: ruby-3.3.0, ffi: FFI } + - { name: Ruby 3.3, ruby: ruby-3.3.0, ffi: FFI } - { name: JRuby 9.4, ruby: jruby-9.4.6.0, ffi: NATIVE } - { name: JRuby 9.4, ruby: jruby-9.4.6.0, ffi: FFI } @@ -116,13 +114,11 @@ jobs: # Ruby versions for CRuby, but only on Bazel 5.x. # Quote versions numbers otherwise 3.0 will render as 3 - { version: "3.0", ffi: NATIVE } - # TODO Re-enable these once flakes are fixed - #- { version: "3.0", ffi: FFI } + - { version: "3.0", ffi: FFI } - { version: "3.1" } - { version: "3.2" } - { version: "3.3", ffi: NATIVE } - # TODO Re-enable these once flakes are fixed - #- { version: "3.3", ffi: FFI } + - { version: "3.3", ffi: FFI } name: MacOS Ruby ${{ matrix.version }}${{ matrix.ffi == 'FFI' && ' FFI' || '' }} runs-on: macos-12 diff --git a/ruby/Gemfile b/ruby/Gemfile index fa75df156323..1817490ee5e4 100644 --- a/ruby/Gemfile +++ b/ruby/Gemfile @@ -1,3 +1,5 @@ source 'https://rubygems.org' +gem 'ffi', github: 'KJTsanaktsidis/ffi', branch: 'missing_wb_struct_layout' + gemspec From 46c440502e12214916a2f92321580e379f5490a6 Mon Sep 17 00:00:00 2001 From: Jason Lunn Date: Thu, 18 Apr 2024 13:22:13 -0400 Subject: [PATCH 2/7] Evaluate a fix for FFI --- ruby/Gemfile | 2 -- 1 file changed, 2 deletions(-) diff --git a/ruby/Gemfile b/ruby/Gemfile index 1817490ee5e4..fa75df156323 100644 --- a/ruby/Gemfile +++ b/ruby/Gemfile @@ -1,5 +1,3 @@ source 'https://rubygems.org' -gem 'ffi', github: 'KJTsanaktsidis/ffi', branch: 'missing_wb_struct_layout' - gemspec From fbae2a572c2a02ec562b9c222dfd52c1cd6356e7 Mon Sep 17 00:00:00 2001 From: Jason Lunn Date: Mon, 22 Apr 2024 13:28:30 -0400 Subject: [PATCH 3/7] Evaluate a prerelease fix for FFI that has been merged to FFI source. --- .github/workflows/test_ruby.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test_ruby.yml b/.github/workflows/test_ruby.yml index 9a57cdb0ec26..6beba2ba99de 100644 --- a/.github/workflows/test_ruby.yml +++ b/.github/workflows/test_ruby.yml @@ -38,7 +38,7 @@ jobs: - name: Run tests uses: protocolbuffers/protobuf-ci/bazel-docker@v3 with: - image: ${{ matrix.image || format('us-docker.pkg.dev/protobuf-build/containers/test/linux/ruby:{0}-6.3.0-a6940b1421a71325ef4c7828ec72d404f56bbf30', matrix.ruby) }} + image: ${{ matrix.image || format('us-docker.pkg.dev/protobuf-build/containers/test/linux/ruby:{0}-6.3.0-f2b50cf85430d5d37f3f24352d6d8ad86d57da78', matrix.ruby) }} credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} bazel-cache: ruby_linux/${{ matrix.ruby }}_${{ matrix.bazel }} bazel: test //ruby/... //ruby/tests:ruby_version --test_env=KOKORO_RUBY_VERSION --test_env=BAZEL=true ${{ matrix.ffi == 'FFI' && '--//ruby:ffi=enabled --test_env=PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION=FFI' || '' }} @@ -56,7 +56,7 @@ jobs: id: cross-compile uses: protocolbuffers/protobuf-ci/cross-compile-protoc@v3 with: - image: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:6.3.0-91a0ac83e968068672bc6001a4d474cfd9a50f1d + image: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:6.3.0-97f82260fd504923d8af642d567afb2d83a1959d credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} architecture: linux-i386 @@ -87,7 +87,7 @@ jobs: id: cross-compile uses: protocolbuffers/protobuf-ci/cross-compile-protoc@v3 with: - image: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:6.3.0-91a0ac83e968068672bc6001a4d474cfd9a50f1d + image: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:6.3.0-97f82260fd504923d8af642d567afb2d83a1959d credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} architecture: linux-aarch64 @@ -168,7 +168,7 @@ jobs: - name: Run tests uses: protocolbuffers/protobuf-ci/bazel-docker@v3 with: - image: us-docker.pkg.dev/protobuf-build/containers/test/linux/ruby:${{ matrix.ruby }}-6.3.0-a6940b1421a71325ef4c7828ec72d404f56bbf30 + image: us-docker.pkg.dev/protobuf-build/containers/test/linux/ruby:${{ matrix.ruby }}-6.3.0-f2b50cf85430d5d37f3f24352d6d8ad86d57da78 credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} bazel-cache: ruby_install/${{ matrix.ruby }}_${{ matrix.bazel }} bash: > From e7715ce4e478e33fd82f48e813f0aeeefeba4de6 Mon Sep 17 00:00:00 2001 From: Jason Lunn Date: Mon, 22 Apr 2024 13:50:34 -0400 Subject: [PATCH 4/7] Evaluate a prerelease fix for FFI that has been merged to FFI source. Bump `rules_ruby` to latest. --- WORKSPACE | 122 ++++++++++++++++++++++++------------------------------ 1 file changed, 54 insertions(+), 68 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index 63785c3f4b98..c527f941b81e 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -1,38 +1,69 @@ workspace(name = "com_google_protobuf") +# For `cc_proto_blacklist_test` and `build_test`. +load("@bazel_skylib//:workspace.bzl", "bazel_skylib_workspace") +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +load("@build_bazel_apple_support//lib:repositories.bzl", "apple_support_dependencies") +load("@build_bazel_rules_apple//apple:repositories.bzl", "apple_rules_dependencies") +load("@com_google_googletest//:googletest_deps.bzl", "googletest_deps") +load("@crate_index//:defs.bzl", "crate_repositories") +load("@fuzzing_py_deps//:requirements.bzl", fuzzing_py_deps_install_deps = "install_deps") +load("@io_bazel_rules_kotlin//kotlin:core.bzl", "kt_register_toolchains") + +# For `kt_jvm_library` +load("@io_bazel_rules_kotlin//kotlin:repositories.bzl", "kotlin_repositories") +load("@maven//:defs.bzl", "pinned_maven_install") +load("@pip_deps//:requirements.bzl", "install_deps") +load("@rules_cc//cc:repositories.bzl", "rules_cc_dependencies") +load("@rules_fuzzing//fuzzing:init.bzl", "rules_fuzzing_init") +load("@rules_fuzzing//fuzzing:repositories.bzl", "rules_fuzzing_dependencies") +load("@rules_jvm_external//:defs.bzl", "maven_install") +load("@rules_jvm_external//:repositories.bzl", "rules_jvm_external_deps") +load("@rules_jvm_external//:setup.bzl", "rules_jvm_external_setup") +load("@rules_pkg//:deps.bzl", "rules_pkg_dependencies") +load("@rules_python//python:repositories.bzl", "py_repositories") +load("@rules_ruby//ruby:defs.bzl", "ruby_runtime") +load("@rules_rust//crate_universe:defs.bzl", "crate", "crates_repository") +load("@rules_rust//rust:repositories.bzl", "rules_rust_dependencies", "rust_register_toolchains") +load("@system_python//:pip.bzl", "pip_parse") +load("@system_ruby//:bundle.bzl", "ruby_bundle") + +# Load common dependencies first to ensure we use the correct version +load("//:protobuf_deps.bzl", "PROTOBUF_MAVEN_ARTIFACTS", "protobuf_deps") +load("//python/dist:system_python.bzl", "system_python") + # An explicit self-reference to work around changes in Bazel 7.0 # See https://github.com/bazelbuild/bazel/issues/19973#issuecomment-1787814450 # buildifier: disable=duplicated-name -local_repository(name = "com_google_protobuf", path = ".") +local_repository( + name = "com_google_protobuf", + path = ".", +) # Second self-reference that makes it possible to load proto rules from @protobuf. # buildifier: disable=duplicated-name -local_repository(name = "protobuf", path = ".") - -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +local_repository( + name = "protobuf", + path = ".", +) local_repository( name = "com_google_protobuf_examples", path = "examples", ) -# Load common dependencies first to ensure we use the correct version -load("//:protobuf_deps.bzl", "PROTOBUF_MAVEN_ARTIFACTS", "protobuf_deps") - protobuf_deps() -load("@rules_python//python:repositories.bzl", "py_repositories") - py_repositories() # Bazel platform rules. http_archive( name = "platforms", + sha256 = "3a561c99e7bdbe9173aa653fd579fe849f1d8d67395780ab4770b1f381431d51", urls = [ "https://mirror.bazel.build/github.com/bazelbuild/platforms/releases/download/0.0.7/platforms-0.0.7.tar.gz", "https://github.com/bazelbuild/platforms/releases/download/0.0.7/platforms-0.0.7.tar.gz", ], - sha256 = "3a561c99e7bdbe9173aa653fd579fe849f1d8d67395780ab4770b1f381431d51", ) http_archive( @@ -40,24 +71,16 @@ http_archive( sha256 = "730215d76eace9dd49bf74ce044e8daa065d175f1ac891cc1d6bb184ef94e565", strip_prefix = "googletest-f53219cdcb7b084ef57414efea92ee5b71989558", urls = [ - "https://github.com/google/googletest/archive/f53219cdcb7b084ef57414efea92ee5b71989558.tar.gz" # 2023-03-16 + "https://github.com/google/googletest/archive/f53219cdcb7b084ef57414efea92ee5b71989558.tar.gz", # 2023-03-16 ], ) -load("@com_google_googletest//:googletest_deps.bzl", "googletest_deps") - googletest_deps() -load("@rules_jvm_external//:repositories.bzl", "rules_jvm_external_deps") - rules_jvm_external_deps() -load("@rules_jvm_external//:setup.bzl", "rules_jvm_external_setup") - rules_jvm_external_setup() -load("@rules_jvm_external//:defs.bzl", "maven_install") - maven_install( artifacts = PROTOBUF_MAVEN_ARTIFACTS, # For updating instructions, see: @@ -69,51 +92,31 @@ maven_install( ], ) -load("@maven//:defs.bzl", "pinned_maven_install") - pinned_maven_install() -# For `cc_proto_blacklist_test` and `build_test`. -load("@bazel_skylib//:workspace.bzl", "bazel_skylib_workspace") - bazel_skylib_workspace() -load("@rules_pkg//:deps.bzl", "rules_pkg_dependencies") - rules_pkg_dependencies() -load("@build_bazel_rules_apple//apple:repositories.bzl", "apple_rules_dependencies") - apple_rules_dependencies() -load("@build_bazel_apple_support//lib:repositories.bzl", "apple_support_dependencies") - apple_support_dependencies() -load("@rules_cc//cc:repositories.bzl", "rules_cc_dependencies") - rules_cc_dependencies() -# For `kt_jvm_library` -load("@io_bazel_rules_kotlin//kotlin:repositories.bzl", "kotlin_repositories") - kotlin_repositories() -load("@io_bazel_rules_kotlin//kotlin:core.bzl", "kt_register_toolchains") - kt_register_toolchains() http_archive( name = "rules_ruby", + sha256 = "ecf2bbdd7fdaccefe5eef68e800f529503f942da3acb2e8a32d96fb2f8e176f2", + strip_prefix = "rules_ruby-4f05aac363689551acb6d13fa015e51352af3960", urls = [ - "https://github.com/protocolbuffers/rules_ruby/archive/b7f3e9756f3c45527be27bc38840d5a1ba690436.zip" + "https://github.com/protocolbuffers/rules_ruby/archive/4f05aac363689551acb6d13fa015e51352af3960.zip", ], - strip_prefix = "rules_ruby-b7f3e9756f3c45527be27bc38840d5a1ba690436", - sha256 = "347927fd8de6132099fcdc58e8f7eab7bde4eb2fd424546b9cd4f1c6f8f8bad8", ) -load("@rules_ruby//ruby:defs.bzl", "ruby_runtime") - ruby_runtime("system_ruby") register_toolchains("@system_ruby//:toolchain") @@ -129,8 +132,6 @@ register_toolchains("@system_ruby//:toolchain") # #register_toolchains("@jruby-9.3//:toolchain") -load("@system_ruby//:bundle.bzl", "ruby_bundle") - ruby_bundle( name = "protobuf_bundle", srcs = ["//ruby:google-protobuf.gemspec"], @@ -150,57 +151,45 @@ http_archive( http_archive( name = "com_github_google_benchmark", - urls = ["https://github.com/google/benchmark/archive/0baacde3618ca617da95375e0af13ce1baadea47.zip"], - strip_prefix = "benchmark-0baacde3618ca617da95375e0af13ce1baadea47", sha256 = "62e2f2e6d8a744d67e4bbc212fcfd06647080de4253c97ad5c6749e09faf2cb0", + strip_prefix = "benchmark-0baacde3618ca617da95375e0af13ce1baadea47", + urls = ["https://github.com/google/benchmark/archive/0baacde3618ca617da95375e0af13ce1baadea47.zip"], ) http_archive( name = "com_google_googleapis", - urls = ["https://github.com/googleapis/googleapis/archive/d81d0b9e6993d6ab425dff4d7c3d05fb2e59fa57.zip"], - strip_prefix = "googleapis-d81d0b9e6993d6ab425dff4d7c3d05fb2e59fa57", - sha256 = "d986023c3d8d2e1b161e9361366669cac9fb97c2a07e656c2548aca389248bb4", build_file = "//benchmarks:BUILD.googleapis", patch_cmds = ["find google -type f -name BUILD.bazel -delete"], + sha256 = "d986023c3d8d2e1b161e9361366669cac9fb97c2a07e656c2548aca389248bb4", + strip_prefix = "googleapis-d81d0b9e6993d6ab425dff4d7c3d05fb2e59fa57", + urls = ["https://github.com/googleapis/googleapis/archive/d81d0b9e6993d6ab425dff4d7c3d05fb2e59fa57.zip"], ) -load("//python/dist:system_python.bzl", "system_python") - system_python( name = "system_python", minimum_python_version = "3.7", ) -load("@system_python//:pip.bzl", "pip_parse") - pip_parse( name = "pip_deps", requirements = "//python:requirements.txt", ) -load("@pip_deps//:requirements.bzl", "install_deps") - install_deps() http_archive( name = "rules_fuzzing", + patch_args = ["-p1"], + patches = ["//third_party:rules_fuzzing.patch"], sha256 = "ff52ef4845ab00e95d29c02a9e32e9eff4e0a4c9c8a6bcf8407a2f19eb3f9190", strip_prefix = "rules_fuzzing-0.4.1", urls = ["https://github.com/bazelbuild/rules_fuzzing/releases/download/v0.4.1/rules_fuzzing-0.4.1.zip"], - patches = ["//third_party:rules_fuzzing.patch"], - patch_args = ["-p1"], ) -load("@rules_fuzzing//fuzzing:repositories.bzl", "rules_fuzzing_dependencies") - rules_fuzzing_dependencies() -load("@rules_fuzzing//fuzzing:init.bzl", "rules_fuzzing_init") - rules_fuzzing_init() -load("@fuzzing_py_deps//:requirements.bzl", fuzzing_py_deps_install_deps = "install_deps") - fuzzing_py_deps_install_deps() http_archive( @@ -209,12 +198,10 @@ http_archive( urls = ["https://github.com/bazelbuild/rules_rust/releases/download/0.29.1/rules_rust-v0.29.1.tar.gz"], ) -load("@rules_rust//rust:repositories.bzl", "rules_rust_dependencies", "rust_register_toolchains") - rules_rust_dependencies() rust_register_toolchains(edition = "2021") -load("@rules_rust//crate_universe:defs.bzl", "crate", "crates_repository") + # to repin, invoke `CARGO_BAZEL_REPIN=1 bazel sync --only=crate_index` crates_repository( name = "crate_index", @@ -225,10 +212,9 @@ crates_repository( version = ">0.0.0", ), "paste": crate.spec( - version = ">=1", + version = ">=1", ), }, ) -load("@crate_index//:defs.bzl", "crate_repositories") crate_repositories() From 02ec605121b75c34733661ee4d781c02591dfbce Mon Sep 17 00:00:00 2001 From: Jason Lunn Date: Mon, 22 Apr 2024 14:58:48 -0400 Subject: [PATCH 5/7] Evaluate a prerelease fix for FFI that has been merged to FFI source. Bump `rules_ruby` to latest. Don't require `bigdecimal` unless needed. --- ruby/google-protobuf.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ruby/google-protobuf.gemspec b/ruby/google-protobuf.gemspec index 3a0b6211ff6f..324abd5d6bb8 100644 --- a/ruby/google-protobuf.gemspec +++ b/ruby/google-protobuf.gemspec @@ -33,7 +33,7 @@ Gem::Specification.new do |s| end s.required_ruby_version = '>= 3.0' # bigdecimal must be used as a non-built in gem as of ruby-3.4 - s.add_dependency "bigdecimal" + s.add_dependency "bigdecimal" if RUBY_VERSION >= '3.4.0' # TODO: evaluate removing Rakefile and moving logic to extconf.rb, so that we # can remove this runtime dependency on rake. See the discussion here for # more details: From c023c2bbb2976f6074c5cd1219bd84557162b0fa Mon Sep 17 00:00:00 2001 From: Jason Lunn Date: Mon, 22 Apr 2024 15:24:17 -0400 Subject: [PATCH 6/7] Use FFI 1.17.RC2. Rollback `rules_ruby` to latest. --- WORKSPACE | 6 +++--- ruby/google-protobuf.gemspec | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index c527f941b81e..240c6da093b5 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -110,10 +110,10 @@ kt_register_toolchains() http_archive( name = "rules_ruby", - sha256 = "ecf2bbdd7fdaccefe5eef68e800f529503f942da3acb2e8a32d96fb2f8e176f2", - strip_prefix = "rules_ruby-4f05aac363689551acb6d13fa015e51352af3960", + sha256 = "347927fd8de6132099fcdc58e8f7eab7bde4eb2fd424546b9cd4f1c6f8f8bad8", + strip_prefix = "rules_ruby-b7f3e9756f3c45527be27bc38840d5a1ba690436", urls = [ - "https://github.com/protocolbuffers/rules_ruby/archive/4f05aac363689551acb6d13fa015e51352af3960.zip", + "https://github.com/protocolbuffers/rules_ruby/archive/b7f3e9756f3c45527be27bc38840d5a1ba690436.zip", ], ) diff --git a/ruby/google-protobuf.gemspec b/ruby/google-protobuf.gemspec index 324abd5d6bb8..bf6fb988ad42 100644 --- a/ruby/google-protobuf.gemspec +++ b/ruby/google-protobuf.gemspec @@ -19,7 +19,7 @@ Gem::Specification.new do |s| BUILD\.bazel|extconf\.rb|wrap_memcpy\.c)$/x end s.extensions = ["ext/google/protobuf_c/Rakefile"] - s.add_dependency "ffi", "~>1" + s.add_dependency "ffi", "~>1.17.0.rc2" s.add_dependency "ffi-compiler", "~>1" else s.files += Dir.glob('ext/**/*').reject do |file| From 44d19d40edd49599f8dee1ff085fef5c3e3f13c6 Mon Sep 17 00:00:00 2001 From: Jason Lunn Date: Mon, 22 Apr 2024 15:34:20 -0400 Subject: [PATCH 7/7] Merge from main. Use FFI 1.17.RC2. --- ruby/google-protobuf.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ruby/google-protobuf.gemspec b/ruby/google-protobuf.gemspec index bf6fb988ad42..e60bb84d5fc5 100644 --- a/ruby/google-protobuf.gemspec +++ b/ruby/google-protobuf.gemspec @@ -39,7 +39,7 @@ Gem::Specification.new do |s| # more details: # https://github.com/protocolbuffers/protobuf/pull/15203 s.add_dependency "rake", ">= 13" - s.add_development_dependency "ffi", "~>1" + s.add_development_dependency "ffi", "~>1.17.0.rc2" s.add_development_dependency "ffi-compiler", "~>1" s.add_development_dependency "rake-compiler", "~> 1.1.0" s.add_development_dependency "test-unit", '~> 3.0', '>= 3.0.9'