feat: add azure support

add azure support by adding a waffle compatible storage module based
on azurex.

new environment variables were added for configuration:
- STORAGE_TYPE, which can be either s3 or azure
- AZURE_BLOB_ENDPOINT: equivalent to s3's asset_host. maps to azurex's api_url.
- AZURE_REGION: the region for the blob storage. only used if AZURE_BLOB_ENDPOINT is unset.
- AZURE_CONTAINER: equivalent to s3's bucket
- AZURE_STORAGE_ACCOUNT_NAME: azure's AccountName
- AZURE_STORAGE_ACCOUNT_KEY: azure's AccountKey
- AZURE_CONNECTION_STRING: an azure connection string. maps to previous values.

Signed-off-by: Francesco Noacco <[email protected]>

.github/workflows/backend-test.yaml

@@ -265,6 +265,83 @@ jobs:
         S3_ASSET_HOST: http://localhost:9000/edgehog
       run: mix test --only integration_storage
 
+  integration-azurite:
+    name: Integration tests using Azurite
+    runs-on: ubuntu-22.04
+    services:
+      postgres:
+        image: postgres:16
+        env:
+          POSTGRES_PASSWORD: postgres
+        ports:
+        - 5432:5432
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+      azurite:
+        image: mcr.microsoft.com/azure-storage/azurite
+        ports:
+        - "10000:10000"
+        - "10001:10001"
+        - "10002:10002"
+        options: >-
+         --health-cmd "nc -z 127.0.0.1 10000"
+         --health-interval 10s
+         --health-timeout 5s
+         --health-retries 5
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+      with:
+        show-progress: false
+
+    - name: Storage setup
+      env:
+        AZURE_STORAGE_CONNECTION_STRING: "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;TableEndpoint=http://127.0.0.1:10002/devstoreaccount1;"
+      run: |
+        # install az-cli
+        curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
+
+        # setup container
+        az storage container create --name edgehog --connection-string $AZURE_STORAGE_CONNECTION_STRING
+        az storage container set-permission --name edgehog --public-access blob --connection-string $AZURE_STORAGE_CONNECTION_STRING
+
+    - name: Install OTP and Elixir
+      uses: erlef/setup-beam@v1
+      id: beam
+      with:
+        version-file: .tool-versions
+        version-type: strict
+
+    - name: Cache dependencies
+      id: cache-deps
+      uses: actions/cache@v3
+      with:
+        path: |
+          backend/deps
+          backend/_build
+        key: "${{ runner.os }}-\
+              otp-${{ steps.beam.outputs.otp-version }}-\
+              elixir-${{ steps.beam.outputs.elixir-version }}-\
+              ${{ hashFiles('backend/mix.lock') }}"
+
+    - name: Install and compile dependencies
+      if: steps.cache-deps.outputs.cache-hit != 'true'
+      run: |
+        mix deps.get --only test
+        mix deps.compile
+
+    - name: Test
+      env:
+        STORAGE_TYPE: azure
+        AZURE_BLOB_ENDPOINT: "http://localhost:10000/devstoreaccount1"
+        AZURE_CONTAINER: "edgehog"
+        AZURE_STORAGE_ACCOUNT_KEY: "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
+        AZURE_STORAGE_ACCOUNT_NAME: "devstoreaccount1"
+      run: mix test --only integration_storage
+
   build-docker-image:
     name: Build Docker image
     runs-on: ubuntu-latest

CHANGELOG.md

@@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [Unreleased]
+### Added
+- Azure support
+
 ## [0.9.1] - 2024-10-28
 ### Fixed
 - Allow receiving `trigger_name` key in trigger payload, which is sent by Astarte >= 1.2.0.

backend/config/dev.exs

@@ -20,6 +20,12 @@
 
 import Config
 
+config :azurex, Azurex.Blob.Config,
+  api_url: "http://localhost:10000/devstoreaccount1",
+  default_container: "edgehog",
+  storage_account_name: "devstoreaccount1",
+  storage_account_key: "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
+
 # Configure your database
 config :edgehog, Edgehog.Repo,
   username: "postgres",

backend/config/runtime.exs

@@ -20,8 +20,10 @@
 
 import Config
 
+alias Azurex.Blob.Config
 alias Waffle.Storage.Google.CloudStorage
 alias Waffle.Storage.S3
+
 if System.get_env("PHX_SERVER") && System.get_env("RELEASE_NAME") do
   config :edgehog, EdgehogWeb.Endpoint, server: true
 end
@@ -44,12 +46,64 @@ if config_env() in [:prod, :test] do
     port: System.get_env("S3_PORT")
   }
 
+  azure =
+    case System.fetch_env("AZURE_CONNECTION_STRING") do
+      {:ok, connection_string} ->
+        connection_string_values = Azurex.Blob.Config.parse_connection_string(connection_string)
+
+        %{
+          api_url: connection_string_values["BlobEndpoint"],
+          storage_account_name: connection_string_values["AccountName"],
+          storage_account_key: connection_string_values["AccountKey"],
+          region: nil,
+          container: System.get_env("AZURE_CONTAINER")
+        }
+
+      :error ->
+        %{
+          api_url: System.get_env("AZURE_BLOB_ENDPOINT"),
+          storage_account_name: System.get_env("AZURE_STORAGE_ACCOUNT_NAME"),
+          storage_account_key: System.get_env("AZURE_STORAGE_ACCOUNT_KEY"),
+          region: System.get_env("AZURE_REGION"),
+          container: System.get_env("AZURE_CONTAINER")
+        }
+    end
+
+  allowed_storage_types = ["s3", "azure"]
+  storage_type = "STORAGE_TYPE" |> System.get_env("s3") |> String.downcase(:ascii)
+
+  unless storage_type in allowed_storage_types do
+    raise "Invalid storage type provided: #{inspect(storage_type)}. Allowed values are #{inspect(allowed_storage_types)}."
+  end
+
+  if storage_type == "azure" && azure.api_url == nil && azure.storage_account_name == nil do
+    raise """
+    When using Azure Blob Storage, either the blob endpoint (AZURE_BLOB_ENDPOINT) \
+    or the account name (AZURE_STORAGE_ACCOUNT_NAME) must be specified.
+    """
+  end
+
   storage_module =
     cond do
+      storage_type == "azure" -> Edgehog.AzureStorage
       s3.host == "storage.googleapis.com" -> CloudStorage
       true -> S3
     end
 
+  azure_api_url =
+    with nil <- azure.api_url do
+      azure_zone_url_str =
+        case to_string(azure.region) do
+          "" -> ""
+          zone -> "." <> zone
+        end
+
+      account_name = to_string(azure.storage_account_name)
+
+      # https://learn.microsoft.com/en-us/azure/storage/common/storage-account-overview#standard-endpoints
+      "https://" <> account_name <> azure_zone_url_str <> ".blob.core.windows.net"
+    end
+
   # The maximum upload size, particularly relevant for OTA updates. Default to 4 GB.
   max_upload_size_bytes =
     "MAX_UPLOAD_SIZE_BYTES"
@@ -62,19 +116,36 @@ if config_env() in [:prod, :test] do
     waffle_virtual_host?: waffle_virtual_host?,
     edgehog_enable_s3_storage?: edgehog_enable_s3_storage?
   } =
+    case storage_module do
+      Edgehog.AzureStorage ->
+        %{
+          waffle_bucket: azure.container,
+          waffle_asset_host: azure_api_url,
+          waffle_virtual_host?: false,
+          edgehog_enable_s3_storage?: azure |> Map.values() |> Enum.any?(&(!is_nil(&1)))
+        }
+
+      _ ->
         %{
           waffle_bucket: s3.bucket,
           waffle_asset_host: s3.asset_host,
           waffle_virtual_host?: true,
           edgehog_enable_s3_storage?: s3 |> Map.values() |> Enum.any?(&(!is_nil(&1)))
         }
+    end
 
   edgehog_enable_s3_storage? =
     case config_env() do
       :test -> true
       :prod -> edgehog_enable_s3_storage?
     end
 
+  config :azurex, Config,
+    api_url: azure_api_url,
+    default_container: azure.container,
+    storage_account_name: azure.storage_account_name,
+    storage_account_key: azure.storage_account_key
+
   # Enable uploaders only when the S3 storage has been configured
   config :edgehog,
     enable_s3_storage?: edgehog_enable_s3_storage?,

backend/lib/edgehog/azure_storage.ex

@@ -0,0 +1,94 @@
+#
+# This file is part of Edgehog.
+#
+# Copyright 2024 SECO Mind Srl
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+defmodule Edgehog.AzureStorage do
+  @moduledoc """
+  Waffle module for Azure storage compatibility.
+
+  Inspiration taken from waffle's s3 module
+  https://github.com/elixir-waffle/waffle/blob/8b058e5e4aabe29481df16ab691f8d1ffce6b6fd/lib/waffle/storage/s3.ex
+  """
+
+  alias Azurex.Blob
+  alias Waffle.Definition.Versioning
+
+  def put(definition, version, {file, scope}) do
+    destination_dir = definition.storage_dir(version, {file, scope})
+    name = Path.join(destination_dir, file.file_name)
+    container = container(definition, {file, scope})
+
+    content_type =
+      version |> definition.s3_object_headers({file, scope}) |> Keyword.get(:content_type)
+
+    contents =
+      case file do
+        %Waffle.File{binary: file_binary} when is_binary(file_binary) -> file_binary
+        %Waffle.File{path: file_path} -> {:stream, File.stream!(file_path)}
+      end
+
+    with :ok <- Blob.put_blob(name, contents, content_type, container) do
+      {:ok, file.file_name}
+    end
+  end
+
+  def delete(definition, version, {file, scope}) do
+    destination_dir = definition.storage_dir(version, {file, scope})
+    filename = Path.basename(file.file_name)
+    name = Path.join(destination_dir, filename)
+    container = container(definition, {file, scope})
+
+    Blob.delete_blob(name, container)
+
+    :ok
+  end
+
+  def url(definition, version, file_and_scope, _options \\ []) do
+    host = host(definition, file_and_scope)
+    dir = definition.storage_dir(version, file_and_scope)
+    filename = Versioning.resolve_file_name(definition, version, file_and_scope)
+    container = container(definition, file_and_scope)
+
+    # TODO: replace with Blob.get_url when https://github.com/jakobht/azurex/pull/47 is merged
+    # and we can specify a custom api_url
+    Path.join([host, container, dir, filename])
+  end
+
+  defp container(definition, file_and_scope) do
+    file_and_scope |> definition.bucket() |> parse_container()
+  end
+
+  defp parse_container({:system, env_var}) when is_binary(env_var), do: System.get_env(env_var)
+  defp parse_container(name), do: name
+
+  defp host(definition, file_and_scope) do
+    case asset_host(definition, file_and_scope) do
+      {:system, env_var} when is_binary(env_var) -> System.get_env(env_var)
+      url -> url
+    end
+  end
+
+  defp asset_host(definition, _file_and_scope) do
+    case definition.asset_host() do
+      false -> Blob.Config.api_url()
+      nil -> Blob.Config.api_url()
+      host -> host
+    end
+  end
+end

backend/mix.exs

@@ -95,6 +95,7 @@ defmodule Edgehog.MixProject do
       {:envar, "~> 1.1"},
       {:ex_aws, "~> 2.2"},
       {:ex_aws_s3, "~> 2.0"},
+      {:azurex, "~> 1.1"},
       {:hackney, "~> 1.9"},
       {:sweet_xml, "~> 0.6"},
       {:waffle_gcs, "~> 0.2"},

backend/mix.lock

@@ -8,6 +8,7 @@
   "ash_postgres": {:hex, :ash_postgres, "2.3.0", "85b786019f75d12f63af00a0a7cf15d885820b1279561e2a33fbb59df1aa4af5", [:mix], [{:ash, ">= 3.4.2 and < 4.0.0-0", [hex: :ash, repo: "hexpm", optional: false]}, {:ash_sql, ">= 0.2.30 and < 1.0.0-0", [hex: :ash_sql, repo: "hexpm", optional: false]}, {:ecto, ">= 3.12.1 and < 4.0.0-0", [hex: :ecto, repo: "hexpm", optional: false]}, {:ecto_sql, "~> 3.12", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:igniter, ">= 0.3.6 and < 1.0.0-0", [hex: :igniter, repo: "hexpm", optional: false]}, {:inflex, "~> 2.1", [hex: :inflex, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:owl, "~> 0.11", [hex: :owl, repo: "hexpm", optional: false]}, {:postgrex, ">= 0.0.0", [hex: :postgrex, repo: "hexpm", optional: false]}], "hexpm", "e7f7ada1978239bfbf60ed3d4af76c34d427278aff0aa51552ac2196c7460394"},
   "ash_sql": {:hex, :ash_sql, "0.2.32", "de99255becfb9daa7991c18c870e9f276bb372acda7eda3e05c3e2ff2ca8922e", [:mix], [{:ash, ">= 3.1.7 and < 4.0.0-0", [hex: :ash, repo: "hexpm", optional: false]}, {:ecto, "~> 3.9", [hex: :ecto, repo: "hexpm", optional: false]}, {:ecto_sql, "~> 3.9", [hex: :ecto_sql, repo: "hexpm", optional: false]}], "hexpm", "43773bcd33d21319c11804d76fe11f1a1b7c8faba7aaedeab6f55fde3d2405db"},
   "astarte_client": {:git, "https://github.com/astarte-platform/astarte-client-elixir.git", "e7e66963eb6d977f73309f95b0b9d04506dff358", []},
+  "azurex": {:hex, :azurex, "1.1.0", "32c18dc3817338bc7ff794a93a0448696cfc407585b634af7a2f4caada0c0e62", [:mix], [{:httpoison, "~> 1.8", [hex: :httpoison, repo: "hexpm", optional: false]}], "hexpm", "7abe82d2b9de836428eb608db6afc383dad200294fb9b9d9adc003681d72bf50"},
   "bunt": {:hex, :bunt, "1.0.0", "081c2c665f086849e6d57900292b3a161727ab40431219529f13c4ddcf3e7a44", [:mix], [], "hexpm", "dc5f86aa08a5f6fa6b8096f0735c4e76d54ae5c9fa2c143e5a1fc7c1cd9bb6b5"},
   "castore": {:hex, :castore, "1.0.8", "dedcf20ea746694647f883590b82d9e96014057aff1d44d03ec90f36a5c0dc6e", [:mix], [], "hexpm", "0b2b66d2ee742cb1d9cb8c8be3b43c3a70ee8651f37b75a8b982e036752983f1"},
   "certifi": {:hex, :certifi, "2.12.0", "2d1cca2ec95f59643862af91f001478c9863c2ac9cb6e2f89780bfd8de987329", [:rebar3], [], "hexpm", "ee68d85df22e554040cdb4be100f33873ac6051387baf6a8f6ce82272340ff1c"},
@@ -42,6 +43,7 @@
   "guardian": {:hex, :guardian, "2.3.2", "78003504b987f2b189d76ccf9496ceaa6a454bb2763627702233f31eb7212881", [:mix], [{:jose, "~> 1.8", [hex: :jose, repo: "hexpm", optional: false]}, {:plug, "~> 1.3.3 or ~> 1.4", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "b189ff38cd46a22a8a824866a6867ca8722942347f13c33f7d23126af8821b52"},
   "hackney": {:hex, :hackney, "1.20.1", "8d97aec62ddddd757d128bfd1df6c5861093419f8f7a4223823537bad5d064e2", [:rebar3], [{:certifi, "~> 2.12.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~> 6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~> 1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~> 1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.4.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~> 1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "fe9094e5f1a2a2c0a7d10918fee36bfec0ec2a979994cff8cfe8058cd9af38e3"},
   "hpax": {:hex, :hpax, "1.0.0", "28dcf54509fe2152a3d040e4e3df5b265dcb6cb532029ecbacf4ce52caea3fd2", [:mix], [], "hexpm", "7f1314731d711e2ca5fdc7fd361296593fc2542570b3105595bb0bc6d0fad601"},
+  "httpoison": {:hex, :httpoison, "1.8.2", "9eb9c63ae289296a544842ef816a85d881d4a31f518a0fec089aaa744beae290", [:mix], [{:hackney, "~> 1.17", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "2bb350d26972e30c96e2ca74a1aaf8293d61d0742ff17f01e0279fef11599921"},
   "idna": {:hex, :idna, "6.1.1", "8a63070e9f7d0c62eb9d9fcb360a7de382448200fbbd1b106cc96d3d8099df8d", [:rebar3], [{:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "92376eb7894412ed19ac475e4a86f7b413c1b9fbb5bd16dccd57934157944cea"},
   "igniter": {:hex, :igniter, "0.3.27", "d6e8441982be6da118578d63011e91ac9c32f8104d553fdc485570ec93b711e3", [:mix], [{:glob_ex, "~> 0.1.7", [hex: :glob_ex, repo: "hexpm", optional: false]}, {:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}, {:rewrite, "~> 0.9", [hex: :rewrite, repo: "hexpm", optional: false]}, {:sourceror, "~> 1.4", [hex: :sourceror, repo: "hexpm", optional: false]}, {:spitfire, ">= 0.1.3 and < 1.0.0-0", [hex: :spitfire, repo: "hexpm", optional: false]}], "hexpm", "d377e1646a0b0b9bed4eb021e10ccaa67c7d2d676c8d577c75f02136c6c10387"},
   "inflex": {:hex, :inflex, "2.1.0", "a365cf0821a9dacb65067abd95008ca1b0bb7dcdd85ae59965deef2aa062924c", [:mix], [], "hexpm", "14c17d05db4ee9b6d319b0bff1bdf22aa389a25398d1952c7a0b5f3d93162dd8"},

doc/pages/admin/deploying_with_kubernetes.md

@@ -180,6 +180,19 @@ Values to be replaced
 - `ACCESS-KEY-ID`: the access key ID for your S3 storage.
 - `SECRET-ACCESS-KEY`: the secret access key for your S3 storage.
 
+#### Azure Blob Credentials
+
+To get started, follow the 
+[documentation](https://learn.microsoft.com/en-us/azure/storage/blobs/storage-quickstart-blobs-portal#create-a-container)
+to create a container.
+
+Then, you can create a secret containing your connection string:
+
+```bash
+$ kubectl create secret generic -n edgehog edgehog-azure-credentials \
+  --from-literal="connection-string=<CONNECTION-STRING>"
+```
+
 #### Google Geolocation API Key (optional)
 
 Activate the Geolocation API for your project in GCP and
@@ -353,6 +366,30 @@ spec:
           value: <EDGEHOG-FORWARDER-PORT>
         - name: EDGEHOG_FORWARDER_SECURE_SESSIONS
           value: <EDGEHOG-FORWARDER-SECURE-SESSIONS>
+
+        # If you're using Azure instead, use the following configuration instead of the S3
+        # configuration above
+        # - name: STORAGE_TYPE
+        #   value: azure
+        # - name: AZURE_CONNECTION_STRING
+        #   valueFrom:
+        #     secretKeyRef:
+        #       name: edgehog-azure-credentials
+        #       key: connection-string 
+        # - name: AZURE_CONTAINER
+        #   value: <AZURE_CONTAINER>
+
+        # You can also use standalone values instead of a connection string
+        # - name: AZURE_REGION
+        #   value: <AZURE_REGION>
+        # - name: AZURE_STORAGE_ACCOUNT_NAME
+        #   value: <AZURE_STORAGE_ACCOUNT_NAME>
+        # - name: AZURE_STORAGE_ACCOUNT_KEY
+        #   value: <AZURE_STORAGE_ACCOUNT_KEY>
+        # - name: AZURE_BLOB_ENDPOINT
+        #   value: <AZURE_BLOB_ENDPOINT>
+
+
         image: edgehogdevicemanager/edgehog-backend:0.9.1
         imagePullPolicy: Always
         name: edgehog-backend