Skip to content

Commit

Permalink
test: add integration tests
Browse files Browse the repository at this point in the history
add runtime s3 configuration for the test environment as well as prod.
this is needed during CI but should not affect normal use 

Signed-off-by: Francesco Noacco <[email protected]>
  • Loading branch information
noaccOS committed Dec 9, 2024
1 parent dd0722e commit ac1524e
Show file tree
Hide file tree
Showing 4 changed files with 283 additions and 61 deletions.
80 changes: 80 additions & 0 deletions .github/workflows/backend-test.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -185,6 +185,86 @@ jobs:
if: matrix.postgres != env.postgres_version_uploading_to_coveralls
run: mix test

integration-minio:
name: Integration tests using MinIO
runs-on: ubuntu-22.04
services:
postgres:
image: postgres:16
env:
POSTGRES_PASSWORD: postgres
ports:
- 5432:5432
options: >-
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
minio:
image: bitnami/minio:latest
env:
MINIO_ROOT_USER: "minioadmin"
MINIO_ROOT_PASSWORD: "minioadmin"
ports:
- 9000:9000
options: >-
--name minio
--health-interval 10s
--health-timeout 5s
--health-retries 5
--health-cmd "curl http://localhost:9000/minio/health/live"
steps:
- name: Checkout repository
uses: actions/checkout@v4
with:
show-progress: false

- name: Bucket setup
run: |
wget https://dl.min.io/client/mc/release/linux-amd64/mc
chmod +x ./mc
./mc alias set minio http://localhost:9000 minioadmin minioadmin;
./mc mb minio/edgehog;
./mc anonymous set download minio/edgehog;
- name: Install OTP and Elixir
uses: erlef/setup-beam@v1
id: beam
with:
version-file: .tool-versions
version-type: strict

- name: Cache dependencies
id: cache-deps
uses: actions/cache@v3
with:
path: |
backend/deps
backend/_build
key: "${{ runner.os }}-\
otp-${{ steps.beam.outputs.otp-version }}-\
elixir-${{ steps.beam.outputs.elixir-version }}-\
${{ hashFiles('backend/mix.lock') }}"

- name: Install and compile dependencies
if: steps.cache-deps.outputs.cache-hit != 'true'
run: |
mix deps.get --only test
mix deps.compile
- name: Test
env:
STORAGE_TYPE: s3
S3_ACCESS_KEY_ID: minioadmin
S3_SECRET_ACCESS_KEY: minioadmin
S3_REGION: local
S3_SCHEME: http://
S3_HOST: localhost
S3_PORT: 9000
S3_BUCKET: edgehog
S3_ASSET_HOST: http://localhost:9000/edgehog
run: mix test --only integration_storage

build-docker-image:
name: Build Docker image
runs-on: ubuntu-latest
Expand Down
137 changes: 77 additions & 60 deletions backend/config/runtime.exs
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,87 @@

import Config

alias Waffle.Storage.Google.CloudStorage
alias Waffle.Storage.S3
if System.get_env("PHX_SERVER") && System.get_env("RELEASE_NAME") do
config :edgehog, EdgehogWeb.Endpoint, server: true
end

# We need s3 storage configuration both in production and in integration tests
if config_env() in [:prod, :test] do
# TODO: while you can use access key + secret key with S3-compatible storages,
# Waffle's default S3 adapter doesn't work well with Google Cloud Storage.
# To use GCP, you need to supply the JSON credentials of an authorized Service
# Account instead, which are used by the GCP adapter for Waffle.
s3 = %{
access_key_id: System.get_env("S3_ACCESS_KEY_ID"),
secret_access_key: System.get_env("S3_SECRET_ACCESS_KEY"),
gcp_credentials: System.get_env("S3_GCP_CREDENTIALS"),
region: System.get_env("S3_REGION"),
bucket: System.get_env("S3_BUCKET"),
asset_host: System.get_env("S3_ASSET_HOST"),
scheme: System.get_env("S3_SCHEME"),
host: System.get_env("S3_HOST"),
port: System.get_env("S3_PORT")
}

storage_module =
cond do
s3.host == "storage.googleapis.com" -> CloudStorage
true -> S3
end

# The maximum upload size, particularly relevant for OTA updates. Default to 4 GB.
max_upload_size_bytes =
"MAX_UPLOAD_SIZE_BYTES"
|> System.get_env(to_string(4_000_000_000))
|> String.to_integer()

%{
waffle_bucket: waffle_bucket,
waffle_asset_host: waffle_asset_host,
waffle_virtual_host?: waffle_virtual_host?,
edgehog_enable_s3_storage?: edgehog_enable_s3_storage?
} =
%{
waffle_bucket: s3.bucket,
waffle_asset_host: s3.asset_host,
waffle_virtual_host?: true,
edgehog_enable_s3_storage?: s3 |> Map.values() |> Enum.any?(&(!is_nil(&1)))
}

edgehog_enable_s3_storage? =
case config_env() do
:test -> true
:prod -> edgehog_enable_s3_storage?
end

# Enable uploaders only when the S3 storage has been configured
config :edgehog,
enable_s3_storage?: edgehog_enable_s3_storage?,
max_upload_size_bytes: max_upload_size_bytes

config :ex_aws, :s3,
scheme: s3.scheme,
host: s3.host,
port: s3.port

config :ex_aws,
region: s3.region,
access_key_id: s3.access_key_id,
secret_access_key: s3.secret_access_key

config :goth,
disabled: storage_module != CloudStorage,
json: s3.gcp_credentials

config :waffle,
storage: storage_module,
asset_host: waffle_asset_host,
bucket: waffle_bucket,
virtual_host: waffle_virtual_host?
end

# config/runtime.exs is executed for all environments, including
# during releases. It is executed after compilation and before the
# system starts, so it is typically used to load production configuration
Expand Down Expand Up @@ -61,41 +138,6 @@ if config_env() == :prod do
url_port = System.get_env("URL_PORT", "443")
url_scheme = System.get_env("URL_SCHEME", "https")

# TODO: while you can use access key + secret key with S3-compatible storages,
# Waffle's default S3 adapter doesn't work well with Google Cloud Storage.
# To use GCP, you need to supply the JSON credentials of an authorized Service
# Account instead, which are used by the GCP adapter for Waffle.
s3 = %{
access_key_id: System.get_env("S3_ACCESS_KEY_ID"),
secret_access_key: System.get_env("S3_SECRET_ACCESS_KEY"),
gcp_credentials: System.get_env("S3_GCP_CREDENTIALS"),
region: System.get_env("S3_REGION"),
bucket: System.get_env("S3_BUCKET"),
asset_host: System.get_env("S3_ASSET_HOST"),
scheme: System.get_env("S3_SCHEME"),
host: System.get_env("S3_HOST"),
port: System.get_env("S3_PORT")
}

# The maximum upload size, particularly relevant for OTA updates. Default to 4 GB.
max_upload_size_bytes =
"MAX_UPLOAD_SIZE_BYTES"
|> System.get_env(to_string(4_000_000_000))
|> String.to_integer()

use_google_cloud_storage =
case s3.host do
"storage.googleapis.com" -> true
_ -> false
end

s3_storage_module =
if use_google_cloud_storage do
Waffle.Storage.Google.CloudStorage
else
Waffle.Storage.S3
end

forwarder_secure_sessions? =
System.get_env("EDGEHOG_FORWARDER_SECURE_SESSIONS", "true") == "true"

Expand Down Expand Up @@ -127,31 +169,6 @@ if config_env() == :prod do
],
secret_key_base: secret_key_base

# Enable uploaders only when the S3 storage has been configured
config :edgehog,
enable_s3_storage?: Enum.any?(s3, fn {_, v} -> v != nil end),
max_upload_size_bytes: max_upload_size_bytes

config :ex_aws, :s3,
scheme: s3.scheme,
host: s3.host,
port: s3.port

config :ex_aws,
region: s3.region,
access_key_id: s3.access_key_id,
secret_access_key: s3.secret_access_key

config :goth,
disabled: !use_google_cloud_storage,
json: s3.gcp_credentials

config :waffle,
storage: s3_storage_module,
bucket: s3.bucket,
asset_host: s3.asset_host,
virtual_host: true

if forwarder_hostname != nil &&
(String.starts_with?(forwarder_hostname, "http://") ||
String.starts_with?(forwarder_hostname, "https://")) do
Expand Down
125 changes: 125 additions & 0 deletions backend/test/edgehog/storage_test.exs
Original file line number Diff line number Diff line change
@@ -0,0 +1,125 @@
#
# This file is part of Edgehog.
#
# Copyright 2024 SECO Mind Srl
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# SPDX-License-Identifier: Apache-2.0
#

defmodule Edgehog.StorageTest do
@moduledoc false
use Edgehog.DataCase, async: true

alias Edgehog.BaseImages.BaseImage

@moduletag :integration_storage

setup do
# Do not mock the storage for integration
Mox.stub_with(Edgehog.BaseImages.StorageMock, Edgehog.BaseImages.BucketStorage)
Mox.stub_with(Edgehog.Assets.SystemModelPictureMock, Edgehog.Assets.SystemModelPicture)
Mox.stub_with(Edgehog.OSManagement.EphemeralImageMock, Edgehog.OSManagement.EphemeralImage)

:ok
end

test "Base Images can be uploaded, read and deleted" do
tenant = Edgehog.TenantsFixtures.tenant_fixture()

base_image_collection =
Edgehog.BaseImagesFixtures.base_image_collection_fixture(tenant: tenant)

file = temporary_file_fixture()
version = "0.0.1"

base_image =
Ash.create!(
BaseImage,
%{version: version, base_image_collection_id: base_image_collection.id, file: file},
tenant: tenant
)

result =
HTTPoison.request(%HTTPoison.Request{method: :get, url: base_image.url})

assert {:ok, %{status_code: 200, body: result_body}} = result
assert File.read!(file.path) == result_body

Ash.destroy!(base_image)

result =
HTTPoison.request(%HTTPoison.Request{method: :get, url: base_image.url})

assert {:ok, %{status_code: 404}} = result
end

test "System Model Picture can be uploaded, read and deleted" do
tenant = Edgehog.TenantsFixtures.tenant_fixture()
filename = "example.png"
expected_content_type = "image/png"
file = temporary_file_fixture(file_name: filename)

system_model =
Edgehog.DevicesFixtures.system_model_fixture(picture_file: file, tenant: tenant)

result =
HTTPoison.request(%HTTPoison.Request{method: :get, url: system_model.picture_url})

assert {:ok, %{status_code: 200, body: result_body, headers: headers}} = result

{_header, content_type} =
Enum.find(headers, fn {key, _value} -> String.downcase(key) == "content-type" end)

assert content_type == expected_content_type
assert File.read!(file.path) == result_body

Ash.destroy!(system_model)

result =
HTTPoison.request(%HTTPoison.Request{method: :get, url: system_model.picture_url})

assert {:ok, %{status_code: 404}} = result
end

test "Ephimeral Images can be uploaded and read" do
tenant = Edgehog.TenantsFixtures.tenant_fixture()
file = temporary_file_fixture()
device_id = [tenant: tenant] |> Edgehog.DevicesFixtures.device_fixture() |> Map.fetch!(:id)

Mox.stub(Edgehog.Astarte.Device.OTARequestV1Mock, :update, fn _, _, _, _ -> :ok end)

ota_operation =
Edgehog.OSManagement.OTAOperation
|> Ash.Changeset.for_create(:manual, [device_id: device_id, base_image_file: file], tenant: tenant)
|> Ash.create!()

result =
HTTPoison.request(%HTTPoison.Request{method: :get, url: ota_operation.base_image_url})

assert {:ok, %{status_code: 200, body: result_body}} = result
assert File.read!(file.path) == result_body
end

def temporary_file_fixture(opts \\ []) do
file_name = Keyword.get(opts, :file_name, "example.bin")
contents = Keyword.get(opts, :contents, "example")
content_type = Keyword.get(opts, :content_type)

temp_file = Plug.Upload.random_file!(file_name)
File.write!(temp_file, contents)

%Plug.Upload{path: temp_file, filename: file_name, content_type: content_type}
end
end
2 changes: 1 addition & 1 deletion backend/test/test_helper.exs
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
# SPDX-License-Identifier: Apache-2.0
#

ExUnit.start()
ExUnit.start(exclude: [:integration_storage])
Ecto.Adapters.SQL.Sandbox.mode(Edgehog.Repo, :manual)

Mox.defmock(Edgehog.Geolocation.GeolocationProviderMock,
Expand Down

0 comments on commit ac1524e

Please sign in to comment.