Merge pull request #310 from ruby/katei/new-build-system

Re-organize build system output

.github/workflows/build.yml

@@ -121,7 +121,7 @@ jobs:
         run: rake ${{ matrix.entry.prerelease }}[${{ inputs.prerel_name }}]
         if: ${{ inputs.prerel_name != '' && matrix.entry.prerelease != '' }}
       - name: rake ${{ matrix.entry.task }}
-        run: docker run -v "$GITHUB_WORKSPACE:/home/me/build" -w /home/me/build -e "RUBYWASM_UID=$(id -u)" -e "RUBYWASM_GID=$(id -g)" -e "GNUMAKEFLAGS=-j$((1 + $(nproc --all)))" ${{ steps.builder-image.outputs.imageid }} rake ${{ matrix.entry.task }}
+        run: docker run -v "$GITHUB_WORKSPACE:/home/me/build" -w /home/me/build -e "ENABLE_GITHUB_ACTIONS_MARKUP=true" -e "RUBYWASM_UID=$(id -u)" -e "RUBYWASM_GID=$(id -g)" -e "GNUMAKEFLAGS=-j$((1 + $(nproc --all)))" ${{ steps.builder-image.outputs.imageid }} rake --verbose ${{ matrix.entry.task }}
       - name: rake ${{ matrix.entry.test }}
         run: rake ${{ matrix.entry.test }}
         if: ${{ matrix.entry.test != '' }}

Steepfile

@@ -12,6 +12,7 @@ target :lib do
   library "fileutils"
   library "open-uri"
   library "uri"
+  library "shellwords"
 
   configure_code_diagnostics(D::Ruby.default)
 end

ext/extinit.c.erb

@@ -1,7 +1,20 @@
 require "erb"
+require "optparse"
+
+opts = OptionParser.new
+opts.on("--cc CC") {|cc| @cc = cc }
+opts.on("--output FILE") {|o| @o = o }
+
+opts.parse!(ARGV)
+if @cc.nil? || @o.nil?
+  puts opts.help
+  exit 1
+end
 
 exts = ARGV
-puts ERB.new(DATA.read).run
+
+c_src = ERB.new(DATA.read).result
+IO.popen("#{@cc} -c -xc - -o #{@o}", "w") {|f| f << c_src }
 
 __END__
 #define init(func, name) { \

lib/ruby_wasm/build.rb

@@ -5,8 +5,63 @@
 module RubyWasm
   # Build executor to run the actual build commands.
   class BuildExecutor
-    def system(*args, **kwargs)
-      Kernel.system(*args, **kwargs)
+    def initialize(verbose: false)
+      @verbose = verbose
+      @github_actions_markup = ENV["ENABLE_GITHUB_ACTIONS_MARKUP"] != nil
+    end
+
+    def system(*args, chdir: nil, out: nil, env: nil)
+      _print_command(args, env)
+
+      if @verbose
+        out ||= $stdout
+      else
+        # Capture stdout by default
+        out_pipe = IO.pipe
+        out = out_pipe[1]
+      end
+      # @type var kwargs: Hash[Symbol, untyped]
+      kwargs = { exception: true, out: out }
+      kwargs[:chdir] = chdir if chdir
+      begin
+        if env
+          Kernel.system(env, *args.to_a.map(&:to_s), **kwargs)
+        else
+          Kernel.system(*args.to_a.map(&:to_s), **kwargs)
+        end
+      ensure
+        out.close if out_pipe
+      end
+    rescue => e
+      if out_pipe
+        # Print the output of the failed command
+        puts out_pipe[0].read
+      end
+      $stdout.flush
+      raise e
+    end
+
+    def begin_section(klass, name, note)
+      message = "\e[1;36m==>\e[0m \e[1m#{klass}(#{name}) -- #{note}\e[0m"
+      if @github_actions_markup
+        puts "::group::#{message}"
+      else
+        puts message
+      end
+
+      # Record the start time
+      @start_times ||= Hash.new
+      @start_times[[klass, name]] = Time.now
+
+      $stdout.flush
+    end
+
+    def end_section(klass, name)
+      took = Time.now - @start_times[[klass, name]]
+      if @github_actions_markup
+        puts "::endgroup::"
+      end
+      puts "\e[1;36m==>\e[0m \e[1m#{klass}(#{name}) -- done in #{took.round(2)}s\e[0m"
     end
 
     def rm_rf(list)
@@ -32,5 +87,15 @@ def mkdir_p(list)
     def write(path, data)
       File.write(path, data)
     end
+
+    private
+
+    def _print_command(args, env)
+      require "shellwords"
+      # Bold cyan
+      print "\e[1;36m  ==>\e[0m "
+      print "env " + env.map { |k, v| "#{k}=#{v}" }.join(" ") + " " if env
+      print args.map { |arg| Shellwords.escape(arg.to_s) }.join(" ") + "\n"
+    end
   end
 end

lib/ruby_wasm/build/product/baseruby.rb

@@ -24,10 +24,11 @@ def build(executor)
       executor.mkdir_p product_build_dir
       @source.build(executor)
       return if Dir.exist?(install_dir)
-      Dir.chdir(product_build_dir) do
-        executor.system "#{@source.configure_file} --prefix=#{install_dir} --disable-install-doc"
-        executor.system "make install"
-      end
+      executor.system @source.configure_file,
+                      "--prefix=#{install_dir}",
+                      "--disable-install-doc",
+                      chdir: product_build_dir
+      executor.system "make", "install", chdir: product_build_dir
     end
   end
 end

lib/ruby_wasm/build/product/crossruby.rb

@@ -32,7 +32,11 @@ def build(executor, crossruby)
       objdir = product_build_dir crossruby
       executor.mkdir_p objdir
       do_extconf executor, crossruby
-      executor.system %Q(make -C "#{objdir}" #{make_args(crossruby).join(" ")} #{lib}.a)
+      executor.system "make",
+                      "-C",
+                      "#{objdir}",
+                      *make_args(crossruby),
+                      "#{lib}.a"
       # A ext can provide link args by link.filelist. It contains only built archive file by default.
       unless File.exist?(linklist(crossruby))
         executor.write(
@@ -49,31 +53,34 @@ def do_extconf(executor, crossruby)
         "--disable=gems",
         # HACK: top_srcdir is required to find ruby headers
         "-e",
-        %Q('$top_srcdir="#{source.src_dir}"'),
+        %Q($top_srcdir="#{source.src_dir}"),
         # HACK: extout is required to find config.h
         "-e",
-        %Q('$extout="#{crossruby.build_dir}/.ext"'),
+        %Q($extout="#{crossruby.build_dir}/.ext"),
         # HACK: force static ext build by imitating extmk
         "-e",
-        "'$static = true; trace_var(:$static) {|v| $static = true }'",
+        "$static = true; trace_var(:$static) {|v| $static = true }",
         # HACK: $0 should be extconf.rb path due to mkmf source file detection
         # and we want to insert some hacks before it. But -e and $0 cannot be
         # used together, so we rewrite $0 in -e.
         "-e",
-        %Q('$0="#{@srcdir}/extconf.rb"'),
+        %Q($0="#{@srcdir}/extconf.rb"),
         "-e",
-        %Q('require_relative "#{@srcdir}/extconf.rb"'),
+        %Q(require_relative "#{@srcdir}/extconf.rb"),
         "-I#{crossruby.build_dir}"
       ]
       # Clear RUBYOPT to avoid loading unrelated bundle setup
-      executor.system ({ "RUBYOPT" => "" }),
-                      "#{crossruby.baseruby_path} #{extconf_args.join(" ")}",
-                      chdir: objdir
+      executor.system crossruby.baseruby_path,
+                      *extconf_args,
+                      chdir: objdir,
+                      env: {
+                        "RUBYOPT" => ""
+                      }
     end
 
     def do_install_rb(executor, crossruby)
       objdir = product_build_dir crossruby
-      executor.system %Q(make -C "#{objdir}" #{make_args(crossruby).join(" ")} install-rb)
+      executor.system "make", "-C", objdir, *make_args(crossruby), "install-rb"
     end
 
     def cache_key(digest)
@@ -129,38 +136,58 @@ def initialize(
     def configure(executor, reconfigure: false)
       if !File.exist?("#{build_dir}/Makefile") || reconfigure
         args = configure_args(RbConfig::CONFIG["host"], toolchain)
-        executor.system "#{source.configure_file} #{args.join(" ")}",
-                        chdir: build_dir
+        executor.system source.configure_file, *args, chdir: build_dir
       end
       # NOTE: we need rbconfig.rb at configuration time to build user given extensions with mkmf
-      executor.system "make rbconfig.rb", chdir: build_dir
+      executor.system "make", "rbconfig.rb", chdir: build_dir
     end
 
     def build_exts(executor)
-      @user_exts.each { |prod| prod.build(executor, self) }
-      executor.mkdir_p File.dirname(extinit_obj)
-      executor.system %Q(ruby #{extinit_c_erb} #{@user_exts.map(&:name).join(" ")} | #{toolchain.cc} -c -x c - -o #{extinit_obj})
+      @user_exts.each do |prod|
+        executor.begin_section prod.class, prod.name, "Building"
+        prod.build(executor, self)
+        executor.end_section prod.class, prod.name
+      end
     end
 
     def build(executor, remake: false, reconfigure: false)
       executor.mkdir_p dest_dir
       executor.mkdir_p build_dir
       @toolchain.install
       [@source, @baseruby, @libyaml, @zlib, @openssl, @wasi_vfs].each do |prod|
+        executor.begin_section prod.class, prod.name, "Building"
         prod.build(executor)
+        executor.end_section prod.class, prod.name
       end
+      executor.begin_section self.class, name, "Configuring"
       configure(executor, reconfigure: reconfigure)
+      executor.end_section self.class, name
+
       build_exts(executor)
 
+      executor.begin_section self.class, name, "Building"
+      executor.mkdir_p File.dirname(extinit_obj)
+      executor.system "ruby",
+                      extinit_c_erb,
+                      *@user_exts.map(&:name),
+                      "--cc",
+                      toolchain.cc,
+                      "--output",
+                      extinit_obj
       install_dir = File.join(build_dir, "install")
       if !File.exist?(install_dir) || remake || reconfigure
-        executor.system "make install DESTDIR=#{install_dir}", chdir: build_dir
+        executor.system "make",
+                        "install",
+                        "DESTDIR=#{install_dir}",
+                        chdir: build_dir
       end
 
       executor.rm_rf dest_dir
       executor.cp_r install_dir, dest_dir
       @user_exts.each { |ext| ext.do_install_rb(executor, self) }
-      executor.system "tar cfz #{artifact} -C rubies #{name}"
+      executor.system "tar", "cfz", artifact, "-C", "rubies", name
+
+      executor.end_section self.class, name
     end
 
     def clean(executor)
@@ -241,11 +268,11 @@ def configure_args(build_triple, toolchain)
 
       args = self.system_triplet_args + ["--build", build_triple]
       args << "--with-static-linked-ext"
-      args << %Q(--with-ext="#{default_exts}")
-      args << %Q(--with-libyaml-dir="#{@libyaml.install_root}")
-      args << %Q(--with-zlib-dir="#{@zlib.install_root}")
-      args << %Q(--with-openssl-dir="#{@openssl.install_root}") if @openssl
-      args << %Q(--with-baseruby="#{baseruby_path}")
+      args << %Q(--with-ext=#{default_exts})
+      args << %Q(--with-libyaml-dir=#{@libyaml.install_root})
+      args << %Q(--with-zlib-dir=#{@zlib.install_root})
+      args << %Q(--with-openssl-dir=#{@openssl.install_root}) if @openssl
+      args << %Q(--with-baseruby=#{baseruby_path})
 
       case target
       when "wasm32-unknown-wasi"
@@ -270,13 +297,13 @@ def configure_args(build_triple, toolchain)
       xcflags << "-DWASM_FIBER_STACK_BUFFER_SIZE=24576"
       xcflags << "-DWASM_SCAN_STACK_BUFFER_SIZE=24576"
 
-      args << %Q(LDFLAGS="#{ldflags.join(" ")}")
-      args << %Q(XLDFLAGS="#{xldflags.join(" ")}")
-      args << %Q(XCFLAGS="#{xcflags.join(" ")}")
-      args << %Q(debugflags="#{@debugflags.join(" ")}")
-      args << %Q(cppflags="#{@cppflags.join(" ")}")
+      args << %Q(LDFLAGS=#{ldflags.join(" ")})
+      args << %Q(XLDFLAGS=#{xldflags.join(" ")})
+      args << %Q(XCFLAGS=#{xcflags.join(" ")})
+      args << %Q(debugflags=#{@debugflags.join(" ")})
+      args << %Q(cppflags=#{@cppflags.join(" ")})
       unless wasmoptflags.empty?
-        args << %Q(wasmoptflags="#{@wasmoptflags.join(" ")}")
+        args << %Q(wasmoptflags=#{@wasmoptflags.join(" ")})
       end
       args << "--disable-install-doc"
       args

lib/ruby_wasm/build/product/libyaml.rb

@@ -25,24 +25,43 @@ def install_root
     end
 
     def name
-      product_build_dir
+      "libyaml-#{LIBYAML_VERSION}-#{target}"
     end
 
     def build(executor)
       return if Dir.exist?(install_root)
 
       executor.mkdir_p File.dirname(product_build_dir)
       executor.rm_rf product_build_dir
-      executor.system "curl -L https://github.com/yaml/libyaml/releases/download/#{LIBYAML_VERSION}/yaml-#{LIBYAML_VERSION}.tar.gz | tar xz",
-                      chdir: File.dirname(product_build_dir)
+      executor.mkdir_p product_build_dir
+      tarball_path =
+        File.join(product_build_dir, "libyaml-#{LIBYAML_VERSION}.tar.gz")
+      executor.system "curl",
+                      "-o",
+                      tarball_path,
+                      "-L",
+                      "https://github.com/yaml/libyaml/releases/download/#{LIBYAML_VERSION}/yaml-#{LIBYAML_VERSION}.tar.gz"
+      executor.system "tar",
+                      "xzf",
+                      tarball_path,
+                      "-C",
+                      product_build_dir,
+                      "--strip-components=1"
 
       # obtain the latest config.guess and config.sub for Emscripten and WASI triple support
-      executor.system "curl -o #{product_build_dir}/config/config.guess 'https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=HEAD'"
-      executor.system "curl -o #{product_build_dir}/config/config.sub 'https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub;hb=HEAD'"
+      executor.system "curl",
+                      "-o",
+                      "#{product_build_dir}/config/config.guess",
+                      "https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=HEAD"
+      executor.system "curl",
+                      "-o",
+                      "#{product_build_dir}/config/config.sub",
+                      "https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub;hb=HEAD"
 
-      executor.system "./configure #{configure_args.join(" ")}",
-                      chdir: product_build_dir
-      executor.system "make install DESTDIR=#{destdir}",
+      executor.system "./configure", *configure_args, chdir: product_build_dir
+      executor.system "make",
+                      "install",
+                      "DESTDIR=#{destdir}",
                       chdir: product_build_dir
     end
   end

lib/ruby_wasm/build/product/openssl.rb

@@ -26,7 +26,7 @@ def install_root
     end
 
     def name
-      product_build_dir
+      "openssl-#{OPENSSL_VERSION}-#{target}"
     end
 
     def configure_args
@@ -59,14 +59,28 @@ def build(executor)
 
       executor.mkdir_p File.dirname(product_build_dir)
       executor.rm_rf product_build_dir
-      executor.system "curl -L https://www.openssl.org/source/openssl-#{OPENSSL_VERSION}.tar.gz | tar xz",
-                      chdir: File.dirname(product_build_dir)
+      executor.mkdir_p product_build_dir
+      tarball_path =
+        File.join(product_build_dir, "openssl-#{OPENSSL_VERSION}.tar.gz")
+      executor.system "curl",
+                      "-o",
+                      tarball_path,
+                      "-L",
+                      "https://www.openssl.org/source/openssl-#{OPENSSL_VERSION}.tar.gz"
+      executor.system "tar",
+                      "xzf",
+                      tarball_path,
+                      "-C",
+                      product_build_dir,
+                      "--strip-components=1"
 
-      executor.system "./Configure #{configure_args.join(" ")}",
-                      chdir: product_build_dir
+      executor.system "./Configure", *configure_args, chdir: product_build_dir
       # Use "install_sw" instead of "install" because it tries to install docs and it's very slow.
       # OpenSSL build system doesn't have well support for parallel build, so force -j1.
-      executor.system "make -j1 install_sw DESTDIR=#{destdir}",
+      executor.system "make",
+                      "-j1",
+                      "install_sw",
+                      "DESTDIR=#{destdir}",
                       chdir: product_build_dir
     end
   end