Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Validate repo data fetched from data-v2.hacs.xyz #3594

Merged
merged 16 commits into from
Apr 12, 2024
Merged
Show file tree
Hide file tree
Changes from 15 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions custom_components/hacs/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -862,7 +862,7 @@ async def async_get_category_repositories_experimental(self, category: str) -> N
"""Update all category repositories."""
self.log.debug("Fetching updated content for %s", category)
try:
category_data = await self.data_client.get_data(category)
category_data = await self.data_client.get_data(category, validate=True)
except HacsNotModifiedException:
self.log.debug("No updates for %s", category)
return
Expand Down Expand Up @@ -1007,7 +1007,7 @@ async def async_handle_removed_repositories(self, _=None) -> None:

try:
if self.configuration.experimental:
removed_repositories = await self.data_client.get_data("removed")
removed_repositories = await self.data_client.get_data("removed", validate=True)
else:
removed_repositories = await self.async_github_get_hacs_default_file(
HacsCategory.REMOVED
Expand Down Expand Up @@ -1091,7 +1091,7 @@ async def async_handle_critical_repositories(self, _=None) -> None:

try:
if self.configuration.experimental:
critical = await self.data_client.get_data("critical")
critical = await self.data_client.get_data("critical", validate=True)
else:
critical = await self.async_github_get_hacs_default_file("critical")
except (GitHubNotModifiedException, HacsNotModifiedException):
Expand Down
45 changes: 43 additions & 2 deletions custom_components/hacs/data_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,21 @@
from typing import Any

from aiohttp import ClientSession, ClientTimeout
import voluptuous as vol

from .exceptions import HacsException, HacsNotModifiedException
from .utils.logger import LOGGER
from .utils.validate import (
VALIDATE_FETCHED_V2_REPO_DATA,
VALIDATE_FETCHED_V2_CRITICAL_REPO_SCHEMA,
VALIDATE_FETCHED_V2_REMOVED_REPO_SCHEMA,
)


CRITICAL_REMOVED_VALIDATORS = {
"critical": VALIDATE_FETCHED_V2_CRITICAL_REPO_SCHEMA,
"removed": VALIDATE_FETCHED_V2_REMOVED_REPO_SCHEMA,
}


class HacsDataClient:
Expand Down Expand Up @@ -48,9 +61,37 @@ async def _do_request(

return await response.json()

async def get_data(self, section: str | None) -> dict[str, dict[str, Any]]:
async def get_data(self, section: str | None, *, validate: bool) -> dict[str, dict[str, Any]]:
"""Get data."""
return await self._do_request(filename="data.json", section=section)
data = await self._do_request(filename="data.json", section=section)
if not validate:
return data

if section in VALIDATE_FETCHED_V2_REPO_DATA:
validated = {}
for key, repo_data in data.items():
try:
validated[key] = VALIDATE_FETCHED_V2_REPO_DATA[section](repo_data)
except vol.Invalid as exception:
LOGGER.info(
"Got invalid data for %s (%s)", repo_data.get("full_name", key), exception
)
continue

return validated

if not (validator := CRITICAL_REMOVED_VALIDATORS.get(section)):
raise ValueError(f"Do not know how to validate {section}")

validated = []
for repo_data in data:
try:
validated.append(validator(repo_data))
except vol.Invalid as exception:
LOGGER.info("Got invalid data for %s (%s)", section, exception)
continue

return validated

async def get_repositories(self, section: str) -> list[str]:
"""Get repositories."""
Expand Down
174 changes: 100 additions & 74 deletions custom_components/hacs/utils/validate.py
Original file line number Diff line number Diff line change
Expand Up @@ -71,18 +71,19 @@ def _country_validator(values) -> list[str]:
)


def validate_repo_data(schema: vol.Schema) -> Callable[[Any], Any]:
def validate_repo_data(schema: dict[str, Any], extra: int) -> Callable[[Any], Any]:
"""Return a validator for repo data.

This is used instead of vol.All to always try both the repo schema and
and the validate_version validator.
"""
_schema = vol.Schema(schema, extra=extra)

def validate_repo_data(data: Any) -> Any:
"""Validate integration repo data."""
schema_errors: vol.MultipleInvalid | None = None
try:
schema(data)
_schema(data)
except vol.MultipleInvalid as err:
schema_errors = err
try:
Expand All @@ -106,88 +107,113 @@ def validate_version(data: Any) -> Any:
return data


V2_COMMON_DATA_JSON_SCHEMA = vol.Schema(
{
vol.Required("description"): vol.Any(str, None),
vol.Optional("downloads"): int,
vol.Optional("etag_releases"): str,
vol.Required("etag_repository"): str,
vol.Required("full_name"): str,
vol.Optional("last_commit"): str,
vol.Required("last_fetched"): vol.Any(int, float),
vol.Required("last_updated"): str,
vol.Optional("last_version"): str,
vol.Required("manifest"): {
vol.Optional("country"): vol.Any([str], False),
vol.Optional("name"): str,
},
vol.Optional("open_issues"): int,
vol.Optional("stargazers_count"): int,
vol.Optional("topics"): [str],
V2_COMMON_DATA_JSON_SCHEMA = {
vol.Required("description"): vol.Any(str, None),
vol.Optional("downloads"): int,
vol.Optional("etag_releases"): str,
vol.Required("etag_repository"): str,
vol.Required("full_name"): str,
vol.Optional("last_commit"): str,
vol.Required("last_fetched"): vol.Any(int, float),
vol.Required("last_updated"): str,
vol.Optional("last_version"): str,
vol.Required("manifest"): {
vol.Optional("country"): vol.Any([str], False),
vol.Optional("name"): str,
},
extra=vol.PREVENT_EXTRA,
)
vol.Optional("open_issues"): int,
vol.Optional("stargazers_count"): int,
vol.Optional("topics"): [str],
}

V2_INTEGRATION_DATA_JSON_SCHEMA = V2_COMMON_DATA_JSON_SCHEMA.extend(
{
vol.Required("domain"): str,
vol.Required("manifest_name"): str,
},
)
V2_INTEGRATION_DATA_JSON_SCHEMA = {
**V2_COMMON_DATA_JSON_SCHEMA,
vol.Required("domain"): str,
vol.Required("manifest_name"): str,
}

V2_NETDAEMON_DATA_JSON_SCHEMA = V2_COMMON_DATA_JSON_SCHEMA.extend(
{
vol.Required("domain"): str,
},
)
V2_NETDAEMON_DATA_JSON_SCHEMA = {
**V2_COMMON_DATA_JSON_SCHEMA,
vol.Required("domain"): str,
}

V2_REPO_SCHEMA = {
"appdaemon": validate_repo_data(V2_COMMON_DATA_JSON_SCHEMA),
"integration": validate_repo_data(V2_INTEGRATION_DATA_JSON_SCHEMA),
"netdaemon": validate_repo_data(V2_NETDAEMON_DATA_JSON_SCHEMA),
"plugin": validate_repo_data(V2_COMMON_DATA_JSON_SCHEMA),
"python_script": validate_repo_data(V2_COMMON_DATA_JSON_SCHEMA),
"template": validate_repo_data(V2_COMMON_DATA_JSON_SCHEMA),
"theme": validate_repo_data(V2_COMMON_DATA_JSON_SCHEMA),
_V2_REPO_SCHEMAS = {
"appdaemon": V2_COMMON_DATA_JSON_SCHEMA,
"integration": V2_INTEGRATION_DATA_JSON_SCHEMA,
"netdaemon": V2_NETDAEMON_DATA_JSON_SCHEMA,
"plugin": V2_COMMON_DATA_JSON_SCHEMA,
"python_script": V2_COMMON_DATA_JSON_SCHEMA,
"template": V2_COMMON_DATA_JSON_SCHEMA,
"theme": V2_COMMON_DATA_JSON_SCHEMA,
}

V2_REPOS_SCHEMA = {
category: vol.Schema({str: V2_REPO_SCHEMA[category]}) for category in V2_REPO_SCHEMA
# Used when validating repos in the hacs integration, discards extra keys
VALIDATE_FETCHED_V2_REPO_DATA = {
category: validate_repo_data(schema, vol.REMOVE_EXTRA)
for category, schema in _V2_REPO_SCHEMAS.items()
}

V2_CRITICAL_REPO_SCHEMA = vol.Schema(
{
vol.Required("link"): str,
vol.Required("reason"): str,
vol.Required("repository"): str,
},
extra=vol.PREVENT_EXTRA,
# Used when validating repos when generating data, fails on extra keys
VALIDATE_GENERATED_V2_REPO_DATA = {
category: vol.Schema({str: validate_repo_data(schema, vol.PREVENT_EXTRA)})
for category, schema in _V2_REPO_SCHEMAS.items()
}

V2_CRITICAL_REPO_DATA_SCHEMA = {
vol.Required("link"): str,
vol.Required("reason"): str,
vol.Required("repository"): str,
}

# Used when validating critical repos in the hacs integration, discards extra keys
VALIDATE_FETCHED_V2_CRITICAL_REPO_SCHEMA = vol.Schema(
V2_CRITICAL_REPO_DATA_SCHEMA,
extra=vol.REMOVE_EXTRA,
)

V2_CRITICAL_REPOS_SCHEMA = vol.Schema([V2_CRITICAL_REPO_SCHEMA])
# Used when validating critical repos when generating data, fails on extra keys
VALIDATE_GENERATED_V2_CRITICAL_REPO_SCHEMA = vol.Schema(
[
vol.Schema(
V2_CRITICAL_REPO_DATA_SCHEMA,
extra=vol.PREVENT_EXTRA,
)
]
)

V2_REMOVED_REPO_SCHEMA = vol.Schema(
{
vol.Optional("link"): str,
vol.Optional("reason"): str,
vol.Required("removal_type"): vol.In(
[
"Integration is missing a version, and is abandoned.",
"Remove",
"archived",
"blacklist",
"critical",
"deprecated",
"removal",
"remove",
"removed",
"replaced",
"repository",
]
),
vol.Required("repository"): str,
},
extra=vol.PREVENT_EXTRA,
V2_REMOVED_REPO_DATA_SCHEMA = {
vol.Optional("link"): str,
vol.Optional("reason"): str,
vol.Required("removal_type"): vol.In(
[
"Integration is missing a version, and is abandoned.",
"Remove",
"archived",
"blacklist",
"critical",
"deprecated",
"removal",
"remove",
"removed",
"replaced",
"repository",
]
),
vol.Required("repository"): str,
}

# Used when validating removed repos in the hacs integration, discards extra keys
VALIDATE_FETCHED_V2_REMOVED_REPO_SCHEMA = vol.Schema(
V2_REMOVED_REPO_DATA_SCHEMA,
extra=vol.REMOVE_EXTRA,
)

V2_REMOVED_REPOS_SCHEMA = vol.Schema([V2_REMOVED_REPO_SCHEMA])
# Used when validating removed repos when generating data, fails on extra keys
VALIDATE_GENERATED_V2_REMOVED_REPO_SCHEMA = vol.Schema(
[
vol.Schema(
V2_REMOVED_REPO_DATA_SCHEMA,
extra=vol.PREVENT_EXTRA,
)
]
)
6 changes: 3 additions & 3 deletions scripts/data/generate_category_data.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@
from custom_components.hacs.utils.data import HacsData
from custom_components.hacs.utils.decorator import concurrent
from custom_components.hacs.utils.queue_manager import QueueManager
from custom_components.hacs.utils.validate import V2_REPOS_SCHEMA
from custom_components.hacs.utils.validate import VALIDATE_GENERATED_V2_REPO_DATA

from .common import expand_and_humanize_error, print_error_and_exit

Expand Down Expand Up @@ -347,7 +347,7 @@ async def generate_category_data(category: str, repository_name: str = None):
os.makedirs(os.path.join(OUTPUT_DIR, category), exist_ok=True)
os.makedirs(os.path.join(OUTPUT_DIR, "diff"), exist_ok=True)
force = os.environ.get("FORCE_REPOSITORY_UPDATE") == "True"
stored_data = await hacs.data_client.get_data(category)
stored_data = await hacs.data_client.get_data(category, validate=False)
current_data = (
next(
(
Expand Down Expand Up @@ -382,7 +382,7 @@ async def generate_category_data(category: str, repository_name: str = None):
did_raise = True

try:
V2_REPOS_SCHEMA[category](updated_data)
VALIDATE_GENERATED_V2_REPO_DATA[category](updated_data)
except vol.Invalid as error:
did_raise = True
errors = expand_and_humanize_error(updated_data, error)
Expand Down
6 changes: 3 additions & 3 deletions scripts/data/validate_category_data.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@

import voluptuous as vol

from custom_components.hacs.utils.validate import V2_REPOS_SCHEMA
from custom_components.hacs.utils.validate import VALIDATE_GENERATED_V2_REPO_DATA

from .common import expand_and_humanize_error, print_error_and_exit

Expand All @@ -20,7 +20,7 @@ async def validate_category_data(category: str, file_path: str) -> None:

if not os.path.isfile(target_path):
print_error_and_exit(f"File {target_path} does not exist", category, file_path)
if category not in V2_REPOS_SCHEMA:
if category not in VALIDATE_GENERATED_V2_REPO_DATA:
print_error_and_exit(f"Category {category} is not supported", category, file_path)

with open(
Expand All @@ -34,7 +34,7 @@ async def validate_category_data(category: str, file_path: str) -> None:
print_error_and_exit(f"File {target_path} is empty", category, file_path)

try:
V2_REPOS_SCHEMA[category](contents)
VALIDATE_GENERATED_V2_REPO_DATA[category](contents)
except vol.Invalid as error:
did_raise = True
errors = expand_and_humanize_error(contents, error)
Expand Down
7 changes: 6 additions & 1 deletion tests/fixtures/proxy/data-v2.hacs.xyz/appdaemon/data.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,11 @@
{
"1296265": {
"description": "This your first repo!",
"etag_repository": "321",
"full_name": "hacs-test-org/appdaemon-basic",
"last_fetched": 0.0
"last_fetched": 0.0,
"last_updated": "2020-01-20T09:34:52Z",
"last_version": "1.0.0",
"manifest": {}
}
}
8 changes: 7 additions & 1 deletion tests/fixtures/proxy/data-v2.hacs.xyz/integration/data.json
Original file line number Diff line number Diff line change
@@ -1,7 +1,13 @@
{
"1296269": {
"description": "This your first repo!",
"domain": "example",
"etag_repository": "321",
"full_name": "hacs-test-org/integration-basic",
"last_fetched": 0.0,
"domain": "example"
"last_updated": "2020-01-20T09:34:52Z",
"last_version": "1.0.0",
"manifest": {},
"manifest_name": "Basic integration"
}
}
7 changes: 6 additions & 1 deletion tests/fixtures/proxy/data-v2.hacs.xyz/netdaemon/data.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,11 @@
{
"1296263": {
"description": "This your first repo!",
"etag_repository": "321",
"full_name": "hacs-test-org/python_script-basic",
"last_fetched": 0.0
"last_fetched": 0.0,
"last_updated": "2020-01-20T09:34:52Z",
"last_version": "1.0.0",
"manifest": {}
}
}
Loading
Loading