diff --git a/.github/workflows/backend-test.yaml b/.github/workflows/backend-test.yaml index 16221e40a..845740af8 100644 --- a/.github/workflows/backend-test.yaml +++ b/.github/workflows/backend-test.yaml @@ -265,6 +265,83 @@ jobs: S3_ASSET_HOST: http://localhost:9000/edgehog run: mix test --only integration_storage + integration-azurite: + name: Integration tests using Azurite + runs-on: ubuntu-22.04 + services: + postgres: + image: postgres:16 + env: + POSTGRES_PASSWORD: postgres + ports: + - 5432:5432 + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + azurite: + image: mcr.microsoft.com/azure-storage/azurite + ports: + - "10000:10000" + - "10001:10001" + - "10002:10002" + options: >- + --health-cmd "nc -z 127.0.0.1 10000" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + show-progress: false + + - name: Storage setup + env: + AZURE_STORAGE_CONNECTION_STRING: "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;TableEndpoint=http://127.0.0.1:10002/devstoreaccount1;" + run: | + # install az-cli + curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash + + # setup container + az storage container create --name edgehog --connection-string $AZURE_STORAGE_CONNECTION_STRING + az storage container set-permission --name edgehog --public-access blob --connection-string $AZURE_STORAGE_CONNECTION_STRING + + - name: Install OTP and Elixir + uses: erlef/setup-beam@v1 + id: beam + with: + version-file: .tool-versions + version-type: strict + + - name: Cache dependencies + id: cache-deps + uses: actions/cache@v3 + with: + path: | + backend/deps + backend/_build + key: "${{ runner.os }}-\ + otp-${{ steps.beam.outputs.otp-version }}-\ + elixir-${{ steps.beam.outputs.elixir-version }}-\ + ${{ hashFiles('backend/mix.lock') }}" + + - name: Install and compile dependencies + if: steps.cache-deps.outputs.cache-hit != 'true' + run: | + mix deps.get --only test + mix deps.compile + + - name: Test + env: + STORAGE_TYPE: azure + AZURE_BLOB_ENDPOINT: "http://localhost:10000/devstoreaccount1" + AZURE_CONTAINER: "edgehog" + AZURE_STORAGE_ACCOUNT_KEY: "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" + AZURE_STORAGE_ACCOUNT_NAME: "devstoreaccount1" + run: mix test --only integration_storage + build-docker-image: name: Build Docker image runs-on: ubuntu-latest diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e6fee0ad..a7e9e8867 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] +### Added +- Azure support + ## [0.9.1] - 2024-10-28 ### Fixed - Allow receiving `trigger_name` key in trigger payload, which is sent by Astarte >= 1.2.0. diff --git a/backend/config/dev.exs b/backend/config/dev.exs index 0ff5b13f6..604f58fdc 100644 --- a/backend/config/dev.exs +++ b/backend/config/dev.exs @@ -20,6 +20,12 @@ import Config +config :azurex, Azurex.Blob.Config, + api_url: "http://localhost:10000/devstoreaccount1", + default_container: "edgehog", + storage_account_name: "devstoreaccount1", + storage_account_key: "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" + # Configure your database config :edgehog, Edgehog.Repo, username: "postgres", diff --git a/backend/config/runtime.exs b/backend/config/runtime.exs index 3cb4337b7..a15930c34 100644 --- a/backend/config/runtime.exs +++ b/backend/config/runtime.exs @@ -20,8 +20,10 @@ import Config +alias Azurex.Blob.Config alias Waffle.Storage.Google.CloudStorage alias Waffle.Storage.S3 + if System.get_env("PHX_SERVER") && System.get_env("RELEASE_NAME") do config :edgehog, EdgehogWeb.Endpoint, server: true end @@ -44,12 +46,64 @@ if config_env() in [:prod, :test] do port: System.get_env("S3_PORT") } + azure = + case System.fetch_env("AZURE_CONNECTION_STRING") do + {:ok, connection_string} -> + connection_string_values = Azurex.Blob.Config.parse_connection_string(connection_string) + + %{ + api_url: connection_string_values["BlobEndpoint"], + storage_account_name: connection_string_values["AccountName"], + storage_account_key: connection_string_values["AccountKey"], + region: nil, + container: System.get_env("AZURE_CONTAINER") + } + + :error -> + %{ + api_url: System.get_env("AZURE_BLOB_ENDPOINT"), + region: System.get_env("AZURE_REGION"), + container: System.get_env("AZURE_CONTAINER"), + storage_account_name: System.get_env("AZURE_STORAGE_ACCOUNT_NAME"), + storage_account_key: System.get_env("AZURE_STORAGE_ACCOUNT_KEY") + } + end + + allowed_storage_types = ["s3", "azure"] + storage_type = "STORAGE_TYPE" |> System.get_env("s3") |> String.downcase(:ascii) + + unless storage_type in allowed_storage_types do + raise "Invalid storage type provided: #{inspect(storage_type)}. Allowed values are #{inspect(allowed_storage_types)}." + end + + if storage_type == "azure" && azure.api_url == nil && azure.storage_account_name == nil do + raise """ + When using Azure Blob Storage, either the blob endpoint (AZURE_BLOB_ENDPOINT) \ + or the account name (AZURE_STORAGE_ACCOUNT_NAME) must be specified. + """ + end + storage_module = cond do + storage_type == "azure" -> Edgehog.AzureStorage s3.host == "storage.googleapis.com" -> CloudStorage true -> S3 end + azure_api_url = + with nil <- azure.api_url do + azure_zone_url_str = + case to_string(azure.region) do + "" -> "" + zone -> "." <> zone + end + + account_name = to_string(azure.storage_account_name) + + # https://learn.microsoft.com/en-us/azure/storage/common/storage-account-overview#standard-endpoints + "https://" <> account_name <> azure_zone_url_str <> ".blob.core.windows.net" + end + # The maximum upload size, particularly relevant for OTA updates. Default to 4 GB. max_upload_size_bytes = "MAX_UPLOAD_SIZE_BYTES" @@ -62,12 +116,23 @@ if config_env() in [:prod, :test] do waffle_virtual_host?: waffle_virtual_host?, edgehog_enable_s3_storage?: edgehog_enable_s3_storage? } = + case storage_module do + Edgehog.AzureStorage -> + %{ + waffle_bucket: azure.container, + waffle_asset_host: azure_api_url, + waffle_virtual_host?: false, + edgehog_enable_s3_storage?: azure |> Map.values() |> Enum.any?(&(!is_nil(&1))) + } + + _ -> %{ waffle_bucket: s3.bucket, waffle_asset_host: s3.asset_host, waffle_virtual_host?: true, edgehog_enable_s3_storage?: s3 |> Map.values() |> Enum.any?(&(!is_nil(&1))) } + end edgehog_enable_s3_storage? = case config_env() do @@ -75,6 +140,12 @@ if config_env() in [:prod, :test] do :prod -> edgehog_enable_s3_storage? end + config :azurex, Config, + api_url: azure_api_url, + default_container: azure.container, + storage_account_name: azure.storage_account_name, + storage_account_key: azure.storage_account_key + # Enable uploaders only when the S3 storage has been configured config :edgehog, enable_s3_storage?: edgehog_enable_s3_storage?, diff --git a/backend/lib/edgehog/azure_storage.ex b/backend/lib/edgehog/azure_storage.ex new file mode 100644 index 000000000..7b902402f --- /dev/null +++ b/backend/lib/edgehog/azure_storage.ex @@ -0,0 +1,94 @@ +# +# This file is part of Edgehog. +# +# Copyright 2024 SECO Mind Srl +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# + +defmodule Edgehog.AzureStorage do + @moduledoc """ + Waffle module for Azure storage compatibility. + + Inspiration taken from waffle's s3 module + https://github.com/elixir-waffle/waffle/blob/8b058e5e4aabe29481df16ab691f8d1ffce6b6fd/lib/waffle/storage/s3.ex + """ + + alias Azurex.Blob + alias Waffle.Definition.Versioning + + def put(definition, version, {file, scope}) do + destination_dir = definition.storage_dir(version, {file, scope}) + name = Path.join(destination_dir, file.file_name) + container = container(definition, {file, scope}) + + content_type = + version |> definition.s3_object_headers({file, scope}) |> Keyword.get(:content_type) + + contents = + case file do + %Waffle.File{binary: file_binary} when is_binary(file_binary) -> file_binary + %Waffle.File{path: file_path} -> {:stream, File.stream!(file_path)} + end + + with :ok <- Blob.put_blob(name, contents, content_type, container) do + {:ok, file.file_name} + end + end + + def delete(definition, version, {file, scope}) do + destination_dir = definition.storage_dir(version, {file, scope}) + filename = Path.basename(file.file_name) + name = Path.join(destination_dir, filename) + container = container(definition, {file, scope}) + + Blob.delete_blob(name, container) + + :ok + end + + def url(definition, version, file_and_scope, _options \\ []) do + host = host(definition, file_and_scope) + dir = definition.storage_dir(version, file_and_scope) + filename = Versioning.resolve_file_name(definition, version, file_and_scope) + container = container(definition, file_and_scope) + + # TODO: replace with Blob.get_url when https://github.com/jakobht/azurex/pull/47 is merged + # and we can specify a custom api_url + Path.join([host, container, dir, filename]) + end + + defp container(definition, file_and_scope) do + file_and_scope |> definition.bucket() |> parse_container() + end + + defp parse_container({:system, env_var}) when is_binary(env_var), do: System.get_env(env_var) + defp parse_container(name), do: name + + defp host(definition, file_and_scope) do + case asset_host(definition, file_and_scope) do + {:system, env_var} when is_binary(env_var) -> System.get_env(env_var) + url -> url + end + end + + defp asset_host(definition, _file_and_scope) do + case definition.asset_host() do + false -> Blob.Config.api_url() + nil -> Blob.Config.api_url() + host -> host + end + end +end diff --git a/backend/mix.exs b/backend/mix.exs index 465b26c24..667a928f1 100644 --- a/backend/mix.exs +++ b/backend/mix.exs @@ -95,6 +95,7 @@ defmodule Edgehog.MixProject do {:envar, "~> 1.1"}, {:ex_aws, "~> 2.2"}, {:ex_aws_s3, "~> 2.0"}, + {:azurex, "~> 1.1"}, {:hackney, "~> 1.9"}, {:sweet_xml, "~> 0.6"}, {:waffle_gcs, "~> 0.2"}, diff --git a/backend/mix.lock b/backend/mix.lock index adad5dc03..2f6c10fb5 100644 --- a/backend/mix.lock +++ b/backend/mix.lock @@ -8,6 +8,7 @@ "ash_postgres": {:hex, :ash_postgres, "2.3.0", "85b786019f75d12f63af00a0a7cf15d885820b1279561e2a33fbb59df1aa4af5", [:mix], [{:ash, ">= 3.4.2 and < 4.0.0-0", [hex: :ash, repo: "hexpm", optional: false]}, {:ash_sql, ">= 0.2.30 and < 1.0.0-0", [hex: :ash_sql, repo: "hexpm", optional: false]}, {:ecto, ">= 3.12.1 and < 4.0.0-0", [hex: :ecto, repo: "hexpm", optional: false]}, {:ecto_sql, "~> 3.12", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:igniter, ">= 0.3.6 and < 1.0.0-0", [hex: :igniter, repo: "hexpm", optional: false]}, {:inflex, "~> 2.1", [hex: :inflex, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:owl, "~> 0.11", [hex: :owl, repo: "hexpm", optional: false]}, {:postgrex, ">= 0.0.0", [hex: :postgrex, repo: "hexpm", optional: false]}], "hexpm", "e7f7ada1978239bfbf60ed3d4af76c34d427278aff0aa51552ac2196c7460394"}, "ash_sql": {:hex, :ash_sql, "0.2.32", "de99255becfb9daa7991c18c870e9f276bb372acda7eda3e05c3e2ff2ca8922e", [:mix], [{:ash, ">= 3.1.7 and < 4.0.0-0", [hex: :ash, repo: "hexpm", optional: false]}, {:ecto, "~> 3.9", [hex: :ecto, repo: "hexpm", optional: false]}, {:ecto_sql, "~> 3.9", [hex: :ecto_sql, repo: "hexpm", optional: false]}], "hexpm", "43773bcd33d21319c11804d76fe11f1a1b7c8faba7aaedeab6f55fde3d2405db"}, "astarte_client": {:git, "https://github.com/astarte-platform/astarte-client-elixir.git", "e7e66963eb6d977f73309f95b0b9d04506dff358", []}, + "azurex": {:hex, :azurex, "1.1.0", "32c18dc3817338bc7ff794a93a0448696cfc407585b634af7a2f4caada0c0e62", [:mix], [{:httpoison, "~> 1.8", [hex: :httpoison, repo: "hexpm", optional: false]}], "hexpm", "7abe82d2b9de836428eb608db6afc383dad200294fb9b9d9adc003681d72bf50"}, "bunt": {:hex, :bunt, "1.0.0", "081c2c665f086849e6d57900292b3a161727ab40431219529f13c4ddcf3e7a44", [:mix], [], "hexpm", "dc5f86aa08a5f6fa6b8096f0735c4e76d54ae5c9fa2c143e5a1fc7c1cd9bb6b5"}, "castore": {:hex, :castore, "1.0.8", "dedcf20ea746694647f883590b82d9e96014057aff1d44d03ec90f36a5c0dc6e", [:mix], [], "hexpm", "0b2b66d2ee742cb1d9cb8c8be3b43c3a70ee8651f37b75a8b982e036752983f1"}, "certifi": {:hex, :certifi, "2.12.0", "2d1cca2ec95f59643862af91f001478c9863c2ac9cb6e2f89780bfd8de987329", [:rebar3], [], "hexpm", "ee68d85df22e554040cdb4be100f33873ac6051387baf6a8f6ce82272340ff1c"}, @@ -42,6 +43,7 @@ "guardian": {:hex, :guardian, "2.3.2", "78003504b987f2b189d76ccf9496ceaa6a454bb2763627702233f31eb7212881", [:mix], [{:jose, "~> 1.8", [hex: :jose, repo: "hexpm", optional: false]}, {:plug, "~> 1.3.3 or ~> 1.4", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "b189ff38cd46a22a8a824866a6867ca8722942347f13c33f7d23126af8821b52"}, "hackney": {:hex, :hackney, "1.20.1", "8d97aec62ddddd757d128bfd1df6c5861093419f8f7a4223823537bad5d064e2", [:rebar3], [{:certifi, "~> 2.12.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~> 6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~> 1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~> 1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.4.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~> 1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "fe9094e5f1a2a2c0a7d10918fee36bfec0ec2a979994cff8cfe8058cd9af38e3"}, "hpax": {:hex, :hpax, "1.0.0", "28dcf54509fe2152a3d040e4e3df5b265dcb6cb532029ecbacf4ce52caea3fd2", [:mix], [], "hexpm", "7f1314731d711e2ca5fdc7fd361296593fc2542570b3105595bb0bc6d0fad601"}, + "httpoison": {:hex, :httpoison, "1.8.2", "9eb9c63ae289296a544842ef816a85d881d4a31f518a0fec089aaa744beae290", [:mix], [{:hackney, "~> 1.17", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "2bb350d26972e30c96e2ca74a1aaf8293d61d0742ff17f01e0279fef11599921"}, "idna": {:hex, :idna, "6.1.1", "8a63070e9f7d0c62eb9d9fcb360a7de382448200fbbd1b106cc96d3d8099df8d", [:rebar3], [{:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "92376eb7894412ed19ac475e4a86f7b413c1b9fbb5bd16dccd57934157944cea"}, "igniter": {:hex, :igniter, "0.3.27", "d6e8441982be6da118578d63011e91ac9c32f8104d553fdc485570ec93b711e3", [:mix], [{:glob_ex, "~> 0.1.7", [hex: :glob_ex, repo: "hexpm", optional: false]}, {:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}, {:rewrite, "~> 0.9", [hex: :rewrite, repo: "hexpm", optional: false]}, {:sourceror, "~> 1.4", [hex: :sourceror, repo: "hexpm", optional: false]}, {:spitfire, ">= 0.1.3 and < 1.0.0-0", [hex: :spitfire, repo: "hexpm", optional: false]}], "hexpm", "d377e1646a0b0b9bed4eb021e10ccaa67c7d2d676c8d577c75f02136c6c10387"}, "inflex": {:hex, :inflex, "2.1.0", "a365cf0821a9dacb65067abd95008ca1b0bb7dcdd85ae59965deef2aa062924c", [:mix], [], "hexpm", "14c17d05db4ee9b6d319b0bff1bdf22aa389a25398d1952c7a0b5f3d93162dd8"}, diff --git a/doc/pages/admin/deploying_with_kubernetes.md b/doc/pages/admin/deploying_with_kubernetes.md index 464326e8f..8ddb90a76 100644 --- a/doc/pages/admin/deploying_with_kubernetes.md +++ b/doc/pages/admin/deploying_with_kubernetes.md @@ -180,6 +180,19 @@ Values to be replaced - `ACCESS-KEY-ID`: the access key ID for your S3 storage. - `SECRET-ACCESS-KEY`: the secret access key for your S3 storage. +#### Azure Blob Credentials + +To get started, follow the +[documentation](https://learn.microsoft.com/en-us/azure/storage/blobs/storage-quickstart-blobs-portal#create-a-container) +to create a container. + +Then, you can create a secret containing your connection string: + +```bash +$ kubectl create secret generic -n edgehog edgehog-azure-credentials \ + --from-literal="connection-string=" +``` + #### Google Geolocation API Key (optional) Activate the Geolocation API for your project in GCP and @@ -353,6 +366,28 @@ spec: value: - name: EDGEHOG_FORWARDER_SECURE_SESSIONS value: + + # If you're using Azure instead, use the following configuration instead of the S3 + # configuration above + # - name: AZURE_CONNECTION_STRING + # valueFrom: + # secretKeyRef: + # name: edgehog-azure-credentials + # key: connection-string + # - name: AZURE_CONTAINER + # value: + + # You can also use standalone values instead of a connection string + # - name: AZURE_REGION + # value: + # - name: AZURE_STORAGE_ACCOUNT_NAME + # value: + # - name: AZURE_STORAGE_ACCOUNT_KEY + # value: + # - name: AZURE_BLOB_ENDPOINT + # value: + + image: edgehogdevicemanager/edgehog-backend:0.9.1 imagePullPolicy: Always name: edgehog-backend