From 76792270959835c60f22197cc06fdff11a095631 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20S=C3=B8rensen?= Date: Fri, 24 Feb 2023 13:41:05 +0100 Subject: [PATCH] Store etag for releases (#3075) --- custom_components/hacs/repositories/base.py | 38 +++++++++------ custom_components/hacs/utils/data.py | 2 + scripts/data/generate_category_data.py | 52 +++++++++++++++++++-- 3 files changed, 74 insertions(+), 18 deletions(-) diff --git a/custom_components/hacs/repositories/base.py b/custom_components/hacs/repositories/base.py index 113d8f9c2f4..c35ce69f172 100644 --- a/custom_components/hacs/repositories/base.py +++ b/custom_components/hacs/repositories/base.py @@ -102,6 +102,7 @@ ("description", ""), ("downloads", 0), ("domain", None), + ("etag_releases", None), ("etag_repository", None), ("full_name", ""), ("last_commit", None), @@ -143,6 +144,7 @@ class RepositoryData: domain: str = None downloads: int = 0 etag_repository: str = None + etag_releases: str = None file_name: str = "" first_install: bool = False full_name: str = "" @@ -505,14 +507,18 @@ async def common_registration(self) -> None: self.data.description = self.data.description @concurrent(concurrenttasks=10, backoff_time=5) - async def common_update(self, ignore_issues=False, force=False) -> bool: + async def common_update(self, ignore_issues=False, force=False, skip_releases=False) -> bool: """Common information update steps of the repository.""" self.logger.debug("%s Getting repository information", self.string) # Attach repository current_etag = self.data.etag_repository try: - await self.common_update_data(ignore_issues=ignore_issues, force=force) + await self.common_update_data( + ignore_issues=ignore_issues, + force=force, + skip_releases=skip_releases, + ) except HacsRepositoryExistException: self.data.full_name = self.hacs.common.renamed_repositories[self.data.full_name] await self.common_update_data(ignore_issues=ignore_issues, force=force) @@ -1049,6 +1055,7 @@ async def common_update_data( ignore_issues: bool = False, force: bool = False, retry=False, + skip_releases=False, ) -> None: """Common update data.""" releases = [] @@ -1097,19 +1104,20 @@ async def common_update_data( raise HacsException(f"{self} Repository has been requested to be removed.") # Get releases. - try: - releases = await self.get_releases( - prerelease=self.data.show_beta, - returnlimit=self.hacs.configuration.release_limit, - ) - if releases: - self.data.releases = True - self.releases.objects = releases - self.data.published_tags = [x.tag_name for x in self.releases.objects] - self.data.last_version = next(iter(self.data.published_tags)) - - except HacsException: - self.data.releases = False + if not skip_releases: + try: + releases = await self.get_releases( + prerelease=self.data.show_beta, + returnlimit=self.hacs.configuration.release_limit, + ) + if releases: + self.data.releases = True + self.releases.objects = releases + self.data.published_tags = [x.tag_name for x in self.releases.objects] + self.data.last_version = next(iter(self.data.published_tags)) + + except HacsException: + self.data.releases = False if not self.force_branch: self.ref = self.version_to_download() diff --git a/custom_components/hacs/utils/data.py b/custom_components/hacs/utils/data.py index eb5c2780d15..76616dc6c6e 100644 --- a/custom_components/hacs/utils/data.py +++ b/custom_components/hacs/utils/data.py @@ -284,6 +284,8 @@ def async_restore_repository(self, entry: str, repository_data: dict[str, Any]): repository.data.description = repository_data.get("description", "") repository.data.downloads = repository_data.get("downloads", 0) repository.data.last_updated = repository_data.get("last_updated", 0) + if self.hacs.system.generator: + repository.data.etag_releases = repository_data.get("etag_releases") repository.data.etag_repository = repository_data.get("etag_repository") repository.data.topics = [ topic for topic in repository_data.get("topics", []) if topic not in TOPIC_FILTER diff --git a/scripts/data/generate_category_data.py b/scripts/data/generate_category_data.py index b34ff8eb7a7..c220f38839a 100644 --- a/scripts/data/generate_category_data.py +++ b/scripts/data/generate_category_data.py @@ -7,7 +7,14 @@ import sys from typing import Any, Literal -from aiogithubapi import GitHub, GitHubAPI +from aiogithubapi import ( + GitHub, + GitHubAPI, + GitHubException, + GitHubNotFoundException, + GitHubNotModifiedException, + GitHubReleaseModel, +) from aiohttp import ClientSession from homeassistant.core import HomeAssistant, callback from homeassistant.helpers.json import JSONEncoder @@ -156,7 +163,42 @@ async def concurrent_update_repository(self, repository: HacsRepository) -> None if repository_has_missing_keys(repository, "update"): # If we have missing keys, force a full update by setting the etag to None repository.data.etag_repository = None - await repository.common_update(force=repository.data.etag_repository is None) + + if repository.data.last_version not in (None, ""): + try: + repository.logger.info("%s Fetching repository releases", repository.string) + response = await self.githubapi.generic( + endpoint=f"/repos/{repository.data.full_name}/releases/latest", + etag=repository.data.etag_releases, + ) + response.data = GitHubReleaseModel(response.data) if response.data else None + repository.data.etag_releases = response.etag + if (releases := response.data) is not None: + repository.data.releases = True + repository.releases.objects = [releases] + repository.data.published_tags = [ + x.tag_name for x in repository.releases.objects + ] + if ( + next_version := next(iter(repository.data.published_tags), None) + ) != repository.data.last_version: + repository.data.last_version = next_version + repository.data.etag_repository = None + + except GitHubNotModifiedException: + repository.data.releases = True + repository.logger.info("%s Release data is up to date", repository.string) + except GitHubNotFoundException: + repository.data.releases = False + repository.logger.info("%s No releases found", repository.string) + except GitHubException as exception: + repository.data.releases = False + repository.logger.warning("%s %s", repository.string, exception) + + await repository.common_update( + force=repository.data.etag_repository is None, + skip_releases=repository.data.releases, + ) async def generate_data_for_category( self, @@ -244,7 +286,11 @@ async def summarize_data( changed = 0 for repo_id, repo_data in updated_data.items(): - if repo_data.get("last_fetched") != current_data.get(repo_id, {}).get("last_fetched"): + if repo_data.get("etag_releases") != current_data.get(repo_id, {}).get( + "etag_releases" + ) or repo_data.get("etag_repository") != current_data.get(repo_id, {}).get( + "etag_repository" + ): changed += 1 print(