chore: Use discovery-artifact-manager to get discovery documents (#11444

)

.github/workflows/dailly-generate-updates.yml

@@ -27,4 +27,4 @@ jobs:
         run: "gem install --no-document toys"
       - name: execute
         run: |
-          toys generate-updates -v --fork --all
+          toys generate-updates -v --fork --all --use-dam

.github/workflows/generate-updates.yml

@@ -30,4 +30,4 @@ jobs:
         run: "gem install --no-document toys"
       - name: execute
         run: |
-          toys generate-updates -v --fork ${{ github.event.inputs.args }}
+          toys generate-updates -v --fork --use-dam ${{ github.event.inputs.args }}

.github/workflows/presubmit.yml

@@ -10,8 +10,6 @@ jobs:
     strategy:
       matrix:
         include:
-          - elixir: "1.10.4-otp-23"
-            otp: "23.3.4.19"
           - elixir: "1.12.3-otp-24"
             otp: "24.3.4.16"
           - elixir: "1.13.4-otp-25"

.github/workflows/update-api-list.yml

@@ -32,4 +32,4 @@ jobs:
           mix do deps.get, compile
       - name: Discover updates
         run: |
-          toys update-api-list -v --fork
+          toys update-api-list -v --fork --use-dam

.toys/generate-updates.rb

@@ -14,6 +14,9 @@
 
 desc "Run standard Google client generation."
 
+flag :use_dam do
+  desc "Get discovery documents from discovery-artifact-manager instead of the discovery service"
+end
 flag :git_remote, "--remote=NAME" do
   desc "The name of the git remote to use as the pull request head. If omitted, does not open a pull request."
 end
@@ -53,6 +56,7 @@ def run
 
   @timestamp = Time.now.utc.strftime("%Y%m%d-%H%M%S")
   setup_builds
+  setup_dam if use_dam
   api_names = list_apis
   api_names.each_with_index do |entry, index|
     handle_package entry, index + 1, api_names.size
@@ -64,6 +68,11 @@ def setup_builds
   exec ["mix", "compile"]
 end
 
+def setup_dam
+  ENV["DISCOVERIES_DIR"] = git_cache.get("https://github.com/googleapis/discovery-artifact-manager.git",
+                                         path: "discoveries", update: true)
+end
+
 def list_apis
   return requested unless all
   api_list = JSON.parse File.read "#{context_directory}/config/apis.json"
@@ -73,7 +82,7 @@ def list_apis
 def handle_package api_name, index, total
   branch_name = "gen/#{api_name}-#{@timestamp}"
   commit_message = "feat: Automated regeneration of #{api_name} client"
-  if open_pr_exists? commit_message
+  if !git_remote.nil? && open_pr_exists?(commit_message)
     puts "(#{index}/#{total}) Pull request already exists for #{api_name}", :yellow
     return
   end

.toys/update-api-list.rb

@@ -14,6 +14,9 @@
 
 desc "Update the API list from discovery"
 
+flag :use_dam do
+  desc "Get discovery index from discovery-artifact-manager instead of the discovery service"
+end
 flag :git_remote, "--remote=NAME" do
   desc "The name of the git remote to use as the pull request head. If omitted, does not open a pull request."
 end
@@ -39,6 +42,7 @@ def run
 
   @timestamp = Time.now.utc.strftime("%Y%m%d-%H%M%S")
   setup_builds
+  setup_dam if use_dam
   update_api_list
 end
 
@@ -47,10 +51,15 @@ def setup_builds
   exec ["mix", "compile"]
 end
 
+def setup_dam
+  ENV["DISCOVERIES_DIR"] = git_cache.get("https://github.com/googleapis/discovery-artifact-manager.git",
+                                         path: "discoveries", update: true)
+end
+
 def update_api_list
   branch_name = "action/auto-update-api-list"
   commit_message = "chore: Automatic update of apis.json"
-  if open_pr_exists? commit_message
+  if !git_remote.nil? && open_pr_exists?(commit_message)
     puts "Pull request already exists", :yellow
     return
   end

lib/google_apis.ex

@@ -22,6 +22,27 @@ defmodule GoogleApis do
   alias GoogleApis.ApiConfig
 
   def fetch(api_config) do
+    case System.fetch_env("DISCOVERIES_DIR") do
+      {:ok, dir} -> fetch_dir(api_config, dir)
+      :error -> fetch_discovery(api_config)
+    end
+  end
+
+  def fetch_dir(api_config, dir) do
+    dest_file = ApiConfig.google_spec_file(api_config)
+    dam_name = ApiConfig.dam_name(api_config)
+    src_file = Path.expand(dam_name, dir)
+
+    with {:ok, body} <- File.read(src_file),
+         :ok <- File.mkdir_p(Path.dirname(dest_file)),
+         :ok <- File.write(dest_file, body) do
+      {:ok, dest_file}
+    else
+      error -> IO.inspect(error)
+    end
+  end
+
+  def fetch_discovery(api_config) do
     file = ApiConfig.google_spec_file(api_config)
 
     with {:ok, {body, _format}} <- GoogleApis.Discovery.fetch(api_config.url),

lib/google_apis/api_config.ex

@@ -41,6 +41,10 @@ defmodule GoogleApis.ApiConfig do
     Macro.underscore(name)
   end
 
+  def dam_name(%{name: name, version: version}) do
+    "#{String.downcase(name)}.#{version}.json"
+  end
+
   def library_namespace(api_config) do
     "#{library_root_namespace(api_config)}.#{module_version(api_config)}"
   end

lib/google_apis/discovery.ex

@@ -54,11 +54,12 @@ defmodule GoogleApis.Discovery do
   Download the list of preferred APIs from the Discovery service
   """
   def discover(opts \\ []) do
-    conn = Connection.new()
-    preferred = Keyword.get(opts, :preferred, true)
-    exceptions = Keyword.get_lazy(opts, :exceptions, &get_exceptions/0)
-    {:ok, %{items: items}} = Apis.discovery_apis_list(conn, preferred: preferred)
+    items = case System.fetch_env("DISCOVERIES_DIR") do
+      {:ok, dir} -> items_from_dam(dir, opts)
+      :error -> items_from_service(opts)
+    end
 
+    exceptions = Keyword.get_lazy(opts, :exceptions, &get_exceptions/0)
     Enum.flat_map(items, fn %{name: name, version: version, discoveryRestUrl: url} ->
       exceptions
       |> Enum.any?(fn
@@ -74,6 +75,27 @@ defmodule GoogleApis.Discovery do
     end)
   end
 
+  defp items_from_dam(dir, opts) do
+    src_file = Path.expand("index.json", dir)
+    {:ok, body} = File.read(src_file)
+    {:ok, %{items: items}} = Jason.decode(body, keys: :atoms)
+    preferred = Keyword.get(opts, :preferred, true)
+    if preferred do
+      Enum.filter(items, fn
+        %{preferred: preferred} -> preferred
+      end)
+    else
+      items
+    end
+  end
+
+  defp items_from_service(opts) do
+    conn = Connection.new()
+    preferred = Keyword.get(opts, :preferred, true)
+    {:ok, %{items: items}} = Apis.discovery_apis_list(conn, preferred: preferred)
+    items
+  end
+
   defmodule Exception do
     defstruct type: nil,
               name: nil,