feat: add azure support

add azure support by adding a waffle compatible storage module based on azurex. the configuration mostly maps to existing s3 configuration, so for now it was not deemed necessary to separate the configuration. however, two new environment variables were added for configuration: - STORAGE_TYPE, which can be either s3 or azure - AZURE_API_URL, because it needs more flexibility than just scheme, host and port. Signed-off-by: Francesco Noacco <[email protected]>
edgehog-device-manager · Dec 4, 2024 · 43f6d52 · 43f6d52
1 parent d4b4852
commit 43f6d52
Show file tree

Hide file tree

Showing 8 changed files with 288 additions and 19 deletions.
diff --git a/.github/workflows/backend-test.yaml b/.github/workflows/backend-test.yaml
@@ -265,6 +265,83 @@ jobs:
         S3_ASSET_HOST: http://localhost:9000/edgehog
       run: mix test --only integration_storage
 
+  integration-azurite:
+    name: Integration tests using Azurite
+    runs-on: ubuntu-22.04
+    services:
+      postgres:
+        image: postgres:16
+        env:
+          POSTGRES_PASSWORD: postgres
+        ports:
+        - 5432:5432
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+      azurite:
+        image: mcr.microsoft.com/azure-storage/azurite
+        ports:
+        - "10000:10000"
+        - "10001:10001"
+        - "10002:10002"
+        options: >-
+         --health-cmd "nc -z 127.0.0.1 10000"
+         --health-interval 10s
+         --health-timeout 5s
+         --health-retries 5
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+      with:
+        show-progress: false
+
+    - name: Storage setup
+      env:
+        AZURE_STORAGE_CONNECTION_STRING: "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;TableEndpoint=http://127.0.0.1:10002/devstoreaccount1;"
+      run: |
+        # install az-cli
+        curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
+
+        # setup container
+        az storage container create --name edgehog --connection-string $AZURE_STORAGE_CONNECTION_STRING
+        az storage container set-permission --name edgehog --public-access blob --connection-string $AZURE_STORAGE_CONNECTION_STRING
+
+    - name: Install OTP and Elixir
+      uses: erlef/setup-beam@v1
+      id: beam
+      with:
+        version-file: .tool-versions
+        version-type: strict
+
+    - name: Cache dependencies
+      id: cache-deps
+      uses: actions/cache@v3
+      with:
+        path: |
+          backend/deps
+          backend/_build
+        key: "${{ runner.os }}-\
+              otp-${{ steps.beam.outputs.otp-version }}-\
+              elixir-${{ steps.beam.outputs.elixir-version }}-\
+              ${{ hashFiles('backend/mix.lock') }}"
+
+    - name: Install and compile dependencies
+      if: steps.cache-deps.outputs.cache-hit != 'true'
+      run: |
+        mix deps.get --only test
+        mix deps.compile
+
+    - name: Test
+      env:
+        STORAGE_TYPE: azure
+        AZURE_BLOB_ENDPOINT: "http://localhost:10000/devstoreaccount1"
+        AZURE_CONTAINER: "edgehog"
+        AZURE_STORAGE_ACCOUNT_KEY: "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
+        AZURE_STORAGE_ACCOUNT_NAME: "devstoreaccount1"
+      run: mix test --only integration_storage
+
   build-docker-image:
     name: Build Docker image
     runs-on: ubuntu-latest

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [Unreleased]
+### Added
+- Azure support
+
 ## [0.9.1] - 2024-10-28
 ### Fixed
 - Allow receiving `trigger_name` key in trigger payload, which is sent by Astarte >= 1.2.0.

diff --git a/backend/config/dev.exs b/backend/config/dev.exs
@@ -20,6 +20,12 @@
 
 import Config
 
+config :azurex, Azurex.Blob.Config,
+  api_url: "http://localhost:10000/devstoreaccount1",
+  default_container: "edgehog",
+  storage_account_name: "devstoreaccount1",
+  storage_account_key: "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
+
 # Configure your database
 config :edgehog, Edgehog.Repo,
   username: "postgres",

diff --git a/backend/config/runtime.exs b/backend/config/runtime.exs
@@ -20,6 +20,10 @@
 
 import Config
 
+alias Azurex.Blob.Config
+alias Waffle.Storage.Google.CloudStorage
+alias Waffle.Storage.S3
+
 if System.get_env("PHX_SERVER") && System.get_env("RELEASE_NAME") do
   config :edgehog, EdgehogWeb.Endpoint, server: true
 end
@@ -77,30 +81,106 @@ if config_env() == :prod do
     port: System.get_env("S3_PORT")
   }
 
+  azure =
+    case System.fetch_env("AZURE_CONNECTION_STRING") do
+      {:ok, connection_string} ->
+        connection_string_values = Azurex.Blob.Config.parse_connection_string(connection_string)
+
+        %{
+          api_url: connection_string_values["BlobEndpoint"],
+          storage_account_name: connection_string_values["AccountName"],
+          storage_account_key: connection_string_values["AccountKey"],
+          region: System.get_env("AZURE_REGION", ""),
+          container: System.get_env("AZURE_CONTAINER")
+        }
+
+      :error ->
+        %{
+          api_url: System.get_env("AZURE_BLOB_ENDPOINT"),
+          region: System.get_env("AZURE_REGION"),
+          container: System.get_env("AZURE_CONTAINER"),
+          storage_account_name: System.get_env("AZURE_STORAGE_ACCOUNT_NAME"),
+          storage_account_key: System.get_env("AZURE_STORAGE_ACCOUNT_KEY")
+        }
+    end
+
+  allowed_storage_types = ["s3", "azure"]
+  storage_type = "STORAGE_TYPE" |> System.get_env("s3") |> String.downcase(:ascii)
+
+  unless storage_type in allowed_storage_types do
+    raise "Invalid storage type provided: #{inspect(storage_type)}. Allowed values are #{inspect(allowed_storage_types)}."
+  end
+
+  storage_module =
+    cond do
+      storage_type == "azure" -> Edgehog.AzureStorage
+      s3.host == "storage.googleapis.com" -> CloudStorage
+      true -> S3
+    end
+
+  azure_api_url =
+    with nil <- azure.api_url do
+      azure_zone_url_str =
+        case to_string(azure.region) do
+          "" -> ""
+          zone -> "." <> zone
+        end
+
+      account_name =
+        with nil <- azure.storage_account_name do
+          if storage_module == Edgehog.AzureStorage do
+            raise """
+            When using Azure Blob Storage, either the blob endpoint (AZURE_BLOB_ENDPOINT) \
+            or the account name (AZURE_STORAGE_ACCOUNT_NAME) must be specified.
+            """
+          end
+        end
+
+      # https://learn.microsoft.com/en-us/azure/storage/common/storage-account-overview#standard-endpoints
+      "https://" <> account_name <> azure_zone_url_str <> ".blob.core.windows.net"
+    end
+
+  %{
+    waffle_bucket: waffle_bucket,
+    waffle_asset_host: waffle_asset_host,
+    waffle_virtual_host?: waffle_virtual_host?,
+    edgehog_enable_s3_storage?: edgehog_enable_s3_storage?
+  } =
+    case storage_module do
+      Edgehog.AzureStorage ->
+        %{
+          waffle_bucket: azure.container,
+          waffle_asset_host: azure_api_url,
+          waffle_virtual_host?: false,
+          edgehog_enable_s3_storage?: azure |> Map.values() |> Enum.any?(&(!is_nil(&1)))
+        }
+
+      _ ->
+        %{
+          waffle_bucket: s3.bucket,
+          waffle_asset_host: s3.asset_host,
+          waffle_virtual_host?: true,
+          edgehog_enable_s3_storage?: azure |> Map.values() |> Enum.any?(&(!is_nil(&1)))
+        }
+    end
+
   # The maximum upload size, particularly relevant for OTA updates. Default to 4 GB.
   max_upload_size_bytes =
     "MAX_UPLOAD_SIZE_BYTES"
     |> System.get_env(to_string(4_000_000_000))
     |> String.to_integer()
 
-  use_google_cloud_storage =
-    case s3.host do
-      "storage.googleapis.com" -> true
-      _ -> false
-    end
-
-  s3_storage_module =
-    if use_google_cloud_storage do
-      Waffle.Storage.Google.CloudStorage
-    else
-      Waffle.Storage.S3
-    end
-
   forwarder_secure_sessions? =
     System.get_env("EDGEHOG_FORWARDER_SECURE_SESSIONS", "true") == "true"
 
   forwarder_hostname = System.get_env("EDGEHOG_FORWARDER_HOSTNAME")
 
+  config :azurex, Config,
+    api_url: azure_api_url,
+    default_container: azure.container,
+    storage_account_name: azure.storage_account_name,
+    storage_account_key: azure.storage_account_key
+
   config :edgehog, Edgehog.Repo,
     # ssl: true,
     # socket_options: [:inet6],
@@ -129,7 +209,7 @@ if config_env() == :prod do
 
   # Enable uploaders only when the S3 storage has been configured
   config :edgehog,
-    enable_s3_storage?: Enum.any?(s3, fn {_, v} -> v != nil end),
+    enable_s3_storage?: edgehog_enable_s3_storage?,
     max_upload_size_bytes: max_upload_size_bytes
 
   config :ex_aws, :s3,
@@ -143,14 +223,14 @@ if config_env() == :prod do
     secret_access_key: s3.secret_access_key
 
   config :goth,
-    disabled: !use_google_cloud_storage,
+    disabled: storage_module != CloudStorage,
     json: s3.gcp_credentials
 
   config :waffle,
-    storage: s3_storage_module,
-    bucket: s3.bucket,
-    asset_host: s3.asset_host,
-    virtual_host: true
+    storage: storage_module,
+    asset_host: waffle_asset_host,
+    bucket: waffle_bucket,
+    virtual_host: waffle_virtual_host?
 
   if forwarder_hostname != nil &&
        (String.starts_with?(forwarder_hostname, "http://") ||

diff --git a/backend/lib/edgehog/azure_storage.ex b/backend/lib/edgehog/azure_storage.ex
@@ -0,0 +1,94 @@
+#
+# This file is part of Edgehog.
+#
+# Copyright 2024 SECO Mind Srl
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+defmodule Edgehog.AzureStorage do
+  @moduledoc """
+  Waffle module for Azure storage compatibility.
+
+  Inspiration taken from waffle's s3 module
+  https://github.com/elixir-waffle/waffle/blob/8b058e5e4aabe29481df16ab691f8d1ffce6b6fd/lib/waffle/storage/s3.ex
+  """
+
+  alias Azurex.Blob
+  alias Waffle.Definition.Versioning
+
+  def put(definition, version, {file, scope}) do
+    destination_dir = definition.storage_dir(version, {file, scope})
+    name = Path.join(destination_dir, file.file_name)
+    container = container(definition, {file, scope})
+
+    content_type =
+      version |> definition.s3_object_headers({file, scope}) |> Keyword.get(:content_type)
+
+    contents =
+      case file do
+        %Waffle.File{binary: file_binary} when is_binary(file_binary) -> file_binary
+        %Waffle.File{path: file_path} -> {:stream, File.stream!(file_path)}
+      end
+
+    with :ok <- Blob.put_blob(name, contents, content_type, container) do
+      {:ok, file.file_name}
+    end
+  end
+
+  def delete(definition, version, {file, scope}) do
+    destination_dir = definition.storage_dir(version, {file, scope})
+    filename = Path.basename(file.file_name)
+    name = Path.join(destination_dir, filename)
+    container = container(definition, {file, scope})
+
+    Blob.delete_blob(name, container)
+
+    :ok
+  end
+
+  def url(definition, version, file_and_scope, _options \\ []) do
+    host = host(definition, file_and_scope)
+    dir = definition.storage_dir(version, file_and_scope)
+    filename = Versioning.resolve_file_name(definition, version, file_and_scope)
+    container = container(definition, file_and_scope)
+
+    # TODO: replace with Blob.get_url when https://github.com/jakobht/azurex/pull/47 is merged
+    # and we can specify a custom api_url
+    Path.join([host, container, dir, filename])
+  end
+
+  defp container(definition, file_and_scope) do
+    file_and_scope |> definition.bucket() |> parse_container()
+  end
+
+  defp parse_container({:system, env_var}) when is_binary(env_var), do: System.get_env(env_var)
+  defp parse_container(name), do: name
+
+  defp host(definition, file_and_scope) do
+    case asset_host(definition, file_and_scope) do
+      {:system, env_var} when is_binary(env_var) -> System.get_env(env_var)
+      url -> url
+    end
+  end
+
+  defp asset_host(definition, _file_and_scope) do
+    case definition.asset_host() do
+      false -> Blob.Config.api_url()
+      nil -> Blob.Config.api_url()
+      host -> host
+    end
+  end
+end
diff --git a/backend/mix.exs b/backend/mix.exs
@@ -95,6 +95,7 @@ defmodule Edgehog.MixProject do
       {:envar, "~> 1.1"},
       {:ex_aws, "~> 2.2"},
       {:ex_aws_s3, "~> 2.0"},
+      {:azurex, "~> 1.1"},
       {:hackney, "~> 1.9"},
       {:sweet_xml, "~> 0.6"},
       {:waffle_gcs, "~> 0.2"},

diff --git a/backend/mix.lock b/backend/mix.lock
@@ -8,6 +8,7 @@
   "ash_postgres": {:hex, :ash_postgres, "2.3.0", "85b786019f75d12f63af00a0a7cf15d885820b1279561e2a33fbb59df1aa4af5", [:mix], [{:ash, ">= 3.4.2 and < 4.0.0-0", [hex: :ash, repo: "hexpm", optional: false]}, {:ash_sql, ">= 0.2.30 and < 1.0.0-0", [hex: :ash_sql, repo: "hexpm", optional: false]}, {:ecto, ">= 3.12.1 and < 4.0.0-0", [hex: :ecto, repo: "hexpm", optional: false]}, {:ecto_sql, "~> 3.12", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:igniter, ">= 0.3.6 and < 1.0.0-0", [hex: :igniter, repo: "hexpm", optional: false]}, {:inflex, "~> 2.1", [hex: :inflex, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:owl, "~> 0.11", [hex: :owl, repo: "hexpm", optional: false]}, {:postgrex, ">= 0.0.0", [hex: :postgrex, repo: "hexpm", optional: false]}], "hexpm", "e7f7ada1978239bfbf60ed3d4af76c34d427278aff0aa51552ac2196c7460394"},
   "ash_sql": {:hex, :ash_sql, "0.2.32", "de99255becfb9daa7991c18c870e9f276bb372acda7eda3e05c3e2ff2ca8922e", [:mix], [{:ash, ">= 3.1.7 and < 4.0.0-0", [hex: :ash, repo: "hexpm", optional: false]}, {:ecto, "~> 3.9", [hex: :ecto, repo: "hexpm", optional: false]}, {:ecto_sql, "~> 3.9", [hex: :ecto_sql, repo: "hexpm", optional: false]}], "hexpm", "43773bcd33d21319c11804d76fe11f1a1b7c8faba7aaedeab6f55fde3d2405db"},
   "astarte_client": {:git, "https://github.com/astarte-platform/astarte-client-elixir.git", "e7e66963eb6d977f73309f95b0b9d04506dff358", []},
+  "azurex": {:hex, :azurex, "1.1.0", "32c18dc3817338bc7ff794a93a0448696cfc407585b634af7a2f4caada0c0e62", [:mix], [{:httpoison, "~> 1.8", [hex: :httpoison, repo: "hexpm", optional: false]}], "hexpm", "7abe82d2b9de836428eb608db6afc383dad200294fb9b9d9adc003681d72bf50"},
   "bunt": {:hex, :bunt, "1.0.0", "081c2c665f086849e6d57900292b3a161727ab40431219529f13c4ddcf3e7a44", [:mix], [], "hexpm", "dc5f86aa08a5f6fa6b8096f0735c4e76d54ae5c9fa2c143e5a1fc7c1cd9bb6b5"},
   "castore": {:hex, :castore, "1.0.8", "dedcf20ea746694647f883590b82d9e96014057aff1d44d03ec90f36a5c0dc6e", [:mix], [], "hexpm", "0b2b66d2ee742cb1d9cb8c8be3b43c3a70ee8651f37b75a8b982e036752983f1"},
   "certifi": {:hex, :certifi, "2.12.0", "2d1cca2ec95f59643862af91f001478c9863c2ac9cb6e2f89780bfd8de987329", [:rebar3], [], "hexpm", "ee68d85df22e554040cdb4be100f33873ac6051387baf6a8f6ce82272340ff1c"},
@@ -42,6 +43,7 @@
   "guardian": {:hex, :guardian, "2.3.2", "78003504b987f2b189d76ccf9496ceaa6a454bb2763627702233f31eb7212881", [:mix], [{:jose, "~> 1.8", [hex: :jose, repo: "hexpm", optional: false]}, {:plug, "~> 1.3.3 or ~> 1.4", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "b189ff38cd46a22a8a824866a6867ca8722942347f13c33f7d23126af8821b52"},
   "hackney": {:hex, :hackney, "1.20.1", "8d97aec62ddddd757d128bfd1df6c5861093419f8f7a4223823537bad5d064e2", [:rebar3], [{:certifi, "~> 2.12.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~> 6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~> 1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~> 1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.4.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~> 1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "fe9094e5f1a2a2c0a7d10918fee36bfec0ec2a979994cff8cfe8058cd9af38e3"},
   "hpax": {:hex, :hpax, "1.0.0", "28dcf54509fe2152a3d040e4e3df5b265dcb6cb532029ecbacf4ce52caea3fd2", [:mix], [], "hexpm", "7f1314731d711e2ca5fdc7fd361296593fc2542570b3105595bb0bc6d0fad601"},
+  "httpoison": {:hex, :httpoison, "1.8.2", "9eb9c63ae289296a544842ef816a85d881d4a31f518a0fec089aaa744beae290", [:mix], [{:hackney, "~> 1.17", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "2bb350d26972e30c96e2ca74a1aaf8293d61d0742ff17f01e0279fef11599921"},
   "idna": {:hex, :idna, "6.1.1", "8a63070e9f7d0c62eb9d9fcb360a7de382448200fbbd1b106cc96d3d8099df8d", [:rebar3], [{:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "92376eb7894412ed19ac475e4a86f7b413c1b9fbb5bd16dccd57934157944cea"},
   "igniter": {:hex, :igniter, "0.3.27", "d6e8441982be6da118578d63011e91ac9c32f8104d553fdc485570ec93b711e3", [:mix], [{:glob_ex, "~> 0.1.7", [hex: :glob_ex, repo: "hexpm", optional: false]}, {:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}, {:rewrite, "~> 0.9", [hex: :rewrite, repo: "hexpm", optional: false]}, {:sourceror, "~> 1.4", [hex: :sourceror, repo: "hexpm", optional: false]}, {:spitfire, ">= 0.1.3 and < 1.0.0-0", [hex: :spitfire, repo: "hexpm", optional: false]}], "hexpm", "d377e1646a0b0b9bed4eb021e10ccaa67c7d2d676c8d577c75f02136c6c10387"},
   "inflex": {:hex, :inflex, "2.1.0", "a365cf0821a9dacb65067abd95008ca1b0bb7dcdd85ae59965deef2aa062924c", [:mix], [], "hexpm", "14c17d05db4ee9b6d319b0bff1bdf22aa389a25398d1952c7a0b5f3d93162dd8"},

diff --git a/doc/pages/admin/deploying_with_kubernetes.md b/doc/pages/admin/deploying_with_kubernetes.md
@@ -180,6 +180,11 @@ Values to be replaced
 - `ACCESS-KEY-ID`: the access key ID for your S3 storage.
 - `SECRET-ACCESS-KEY`: the secret access key for your S3 storage.
 
+#### Azure Blob Storage
+
+Refer 
+
+
 #### Google Geolocation API Key (optional)
 
 Activate the Geolocation API for your project in GCP and