Bump HACS to 1.30.1

custom_components/hacs/__init__.py

@@ -25,6 +25,7 @@ import voluptuous as vol
 
 from .base import HacsBase
 from .const import DOMAIN, MINIMUM_HA_VERSION, STARTUP
+from .data_client import HacsDataClient
 from .enums import ConfigurationType, HacsDisabledReason, HacsStage, LovelaceMode
 from .frontend import async_register_frontend
 from .utils.configuration_schema import hacs_config_combined
@@ -87,6 +88,10 @@ async def async_initialize_integration(
     hacs.hass = hass
     hacs.queue = QueueManager(hass=hass)
     hacs.data = HacsData(hacs=hacs)
+    hacs.data_client = HacsDataClient(
+        session=clientsession,
+        client_name=f"HACS/{integration.version}",
+    )
     hacs.system.running = True
     hacs.session = clientsession
 
@@ -153,6 +158,7 @@ async def async_initialize_integration(
             hacs.disable_hacs(HacsDisabledReason.RESTORE)
             return False
 
+        if not hacs.configuration.experimental:
             can_update = await hacs.async_can_update()
             hacs.log.debug("Can update %s repositories", can_update)
 
@@ -168,7 +174,7 @@ async def async_initialize_integration(
             hacs.log.info("Update entities are only supported when using UI configuration")
 
         else:
-            hass.config_entries.async_setup_platforms(
+            await hass.config_entries.async_forward_entry_setups(
                 config_entry,
                 [Platform.SENSOR, Platform.UPDATE]
                 if hacs.configuration.experimental

custom_components/hacs/base.py

@@ -28,11 +28,17 @@ from homeassistant.config_entries import ConfigEntry, ConfigEntryState
 from homeassistant.const import EVENT_HOMEASSISTANT_FINAL_WRITE, Platform
 from homeassistant.core import HomeAssistant, callback
 from homeassistant.helpers.dispatcher import async_dispatcher_send
-from homeassistant.helpers.issue_registry import async_create_issue, IssueSeverity
+from homeassistant.helpers.issue_registry import IssueSeverity, async_create_issue
 from homeassistant.loader import Integration
 from homeassistant.util import dt
 
-from .const import DOMAIN, TV
+from custom_components.hacs.repositories.base import (
+    HACS_MANIFEST_KEYS_TO_EXPORT,
+    REPOSITORY_KEYS_TO_EXPORT,
+)
+
+from .const import DOMAIN, TV, URL_BASE
+from .data_client import HacsDataClient
 from .enums import (
     ConfigurationType,
     HacsCategory,
@@ -47,6 +53,7 @@ from .exceptions import (
     HacsException,
     HacsExecutionStillInProgress,
     HacsExpectedException,
+    HacsNotModifiedException,
     HacsRepositoryArchivedException,
     HacsRepositoryExistException,
     HomeAssistantCoreRepositoryException,
@@ -164,6 +171,9 @@ class HacsStatus:
 
     startup: bool = True
     new: bool = False
+    active_frontend_endpoint_plugin: bool = False
+    active_frontend_endpoint_theme: bool = False
+    inital_fetch_done: bool = False
 
 
 @dataclass
@@ -174,6 +184,7 @@ class HacsSystem:
     running: bool = False
     stage = HacsStage.SETUP
     action: bool = False
+    generator: bool = False
 
     @property
     def disabled(self) -> bool:
@@ -263,7 +274,7 @@ class HacsRepositories:
 
         self._default_repositories.add(repo_id)
 
-    def set_repository_id(self, repository, repo_id):
+    def set_repository_id(self, repository: HacsRepository, repo_id: str):
         """Update a repository id."""
         existing_repo_id = str(repository.data.id)
         if existing_repo_id == repo_id:
@@ -348,6 +359,7 @@ class HacsBase:
     configuration = HacsConfiguration()
     core = HacsCore()
     data: HacsData | None = None
+    data_client: HacsDataClient | None = None
     frontend_version: str | None = None
     github: GitHub | None = None
     githubapi: GitHubAPI | None = None
@@ -544,8 +556,6 @@ class HacsBase:
         if check:
             try:
                 await repository.async_registration(ref)
-                if self.status.new:
-                    repository.data.new = False
                 if repository.validate.errors:
                     self.common.skip.append(repository.data.full_name)
                     if not self.status.startup:
@@ -559,7 +569,11 @@ class HacsBase:
                     repository.logger.info("%s Validation completed", repository.string)
                 else:
                     repository.logger.info("%s Registration completed", repository.string)
-            except (HacsRepositoryExistException, HacsRepositoryArchivedException):
+            except (HacsRepositoryExistException, HacsRepositoryArchivedException) as exception:
+                if self.system.generator:
+                    repository.logger.error(
+                        "%s Registration Failed - %s", repository.string, exception
+                    )
                 return
             except AIOGitHubAPIException as exception:
                 self.common.skip.append(repository.data.full_name)
@@ -567,6 +581,9 @@ class HacsBase:
                     f"Validation for {repository_full_name} failed with {exception}."
                 ) from exception
 
+        if self.status.new:
+            repository.data.new = False
+
         if repository_id is not None:
             repository.data.id = repository_id
 
@@ -586,34 +603,7 @@ class HacsBase:
     async def startup_tasks(self, _=None) -> None:
         """Tasks that are started after setup."""
         self.set_stage(HacsStage.STARTUP)
-
+        await self.async_load_hacs_from_github()
-        try:
-            repository = self.repositories.get_by_full_name(HacsGitHubRepo.INTEGRATION)
-            if repository is None:
-                await self.async_register_repository(
-                    repository_full_name=HacsGitHubRepo.INTEGRATION,
-                    category=HacsCategory.INTEGRATION,
-                    default=True,
-                )
-                repository = self.repositories.get_by_full_name(HacsGitHubRepo.INTEGRATION)
-            if repository is None:
-                raise HacsException("Unknown error")
-
-            repository.data.installed = True
-            repository.data.installed_version = self.integration.version.string
-            repository.data.new = False
-            repository.data.releases = True
-
-            self.repository = repository.repository_object
-            self.repositories.mark_default(repository)
-        except HacsException as exception:
-            if "403" in str(exception):
-                self.log.critical(
-                    "GitHub API is ratelimited, or the token is wrong.",
-                )
-            else:
-                self.log.critical("Could not load HACS! - %s", exception)
-            self.disable_hacs(HacsDisabledReason.LOAD_HACS)
 
         if critical := await async_load_from_store(self.hass, "critical"):
             for repo in critical:
@@ -624,16 +614,38 @@ class HacsBase:
                     )
                     break
 
+        if not self.configuration.experimental:
             self.recuring_tasks.append(
                 self.hass.helpers.event.async_track_time_interval(
-                self.async_get_all_category_repositories, timedelta(hours=3)
+                    self.async_update_downloaded_repositories, timedelta(hours=48)
                 )
             )
             self.recuring_tasks.append(
                 self.hass.helpers.event.async_track_time_interval(
-                self.async_update_all_repositories, timedelta(hours=25)
+                    self.async_update_all_repositories,
+                    timedelta(hours=96),
                 )
             )
+        else:
+            self.recuring_tasks.append(
+                self.hass.helpers.event.async_track_time_interval(
+                    self.async_load_hacs_from_github,
+                    timedelta(hours=48),
+                )
+            )
+
+        self.recuring_tasks.append(
+            self.hass.helpers.event.async_track_time_interval(
+                self.async_update_downloaded_custom_repositories, timedelta(hours=48)
+            )
+        )
+
+        self.recuring_tasks.append(
+            self.hass.helpers.event.async_track_time_interval(
+                self.async_get_all_category_repositories, timedelta(hours=6)
+            )
+        )
+
         self.recuring_tasks.append(
             self.hass.helpers.event.async_track_time_interval(
                 self.async_check_rate_limit, timedelta(minutes=5)
@@ -644,14 +656,10 @@ class HacsBase:
                 self.async_prosess_queue, timedelta(minutes=10)
             )
         )
+
         self.recuring_tasks.append(
             self.hass.helpers.event.async_track_time_interval(
-                self.async_update_downloaded_repositories, timedelta(hours=2)
+                self.async_handle_critical_repositories, timedelta(hours=6)
-            )
-        )
-        self.recuring_tasks.append(
-            self.hass.helpers.event.async_track_time_interval(
-                self.async_handle_critical_repositories, timedelta(hours=2)
             )
         )
 
@@ -659,6 +667,8 @@ class HacsBase:
             EVENT_HOMEASSISTANT_FINAL_WRITE, self.data.async_force_write
         )
 
+        self.log.debug("There are %s scheduled recurring tasks", len(self.recuring_tasks))
+
         self.status.startup = False
         self.async_dispatch(HacsDispatchEvent.STATUS, {})
 
@@ -756,6 +766,42 @@ class HacsBase:
         if self.configuration.netdaemon:
             self.enable_hacs_category(HacsCategory.NETDAEMON)
 
+    async def async_load_hacs_from_github(self, _=None) -> None:
+        """Load HACS from GitHub."""
+        if self.configuration.experimental and self.status.inital_fetch_done:
+            return
+
+        try:
+            repository = self.repositories.get_by_full_name(HacsGitHubRepo.INTEGRATION)
+            if repository is None:
+                await self.async_register_repository(
+                    repository_full_name=HacsGitHubRepo.INTEGRATION,
+                    category=HacsCategory.INTEGRATION,
+                    default=True,
+                )
+                repository = self.repositories.get_by_full_name(HacsGitHubRepo.INTEGRATION)
+            elif self.configuration.experimental and not self.status.startup:
+                self.log.error("Scheduling update of hacs/integration")
+                self.queue.add(repository.common_update())
+            if repository is None:
+                raise HacsException("Unknown error")
+
+            repository.data.installed = True
+            repository.data.installed_version = self.integration.version.string
+            repository.data.new = False
+            repository.data.releases = True
+
+            self.repository = repository.repository_object
+            self.repositories.mark_default(repository)
+        except HacsException as exception:
+            if "403" in str(exception):
+                self.log.critical(
+                    "GitHub API is ratelimited, or the token is wrong.",
+                )
+            else:
+                self.log.critical("Could not load HACS! - %s", exception)
+            self.disable_hacs(HacsDisabledReason.LOAD_HACS)
+
     async def async_get_all_category_repositories(self, _=None) -> None:
         """Get all category repositories."""
         if self.system.disabled:
@@ -763,11 +809,62 @@ class HacsBase:
         self.log.info("Loading known repositories")
         await asyncio.gather(
             *[
-                self.async_get_category_repositories(HacsCategory(category))
+                self.async_get_category_repositories_experimental(category)
+                if self.configuration.experimental
+                else self.async_get_category_repositories(HacsCategory(category))
                 for category in self.common.categories or []
             ]
         )
 
+    async def async_get_category_repositories_experimental(self, category: str) -> None:
+        """Update all category repositories."""
+        self.log.debug("Fetching updated content for %s", category)
+        try:
+            category_data = await self.data_client.get_data(category)
+        except HacsNotModifiedException:
+            self.log.debug("No updates for %s", category)
+            return
+        except HacsException as exception:
+            self.log.error("Could not update %s - %s", category, exception)
+            return
+
+        await self.data.register_unknown_repositories(category_data, category)
+
+        for repo_id, repo_data in category_data.items():
+            repo = repo_data["full_name"]
+            if self.common.renamed_repositories.get(repo):
+                repo = self.common.renamed_repositories[repo]
+            if self.repositories.is_removed(repo):
+                continue
+            if repo in self.common.archived_repositories:
+                continue
+            if repository := self.repositories.get_by_full_name(repo):
+                self.repositories.set_repository_id(repository, repo_id)
+                self.repositories.mark_default(repository)
+                if repository.data.last_fetched is None or (
+                    repository.data.last_fetched.timestamp() < repo_data["last_fetched"]
+                ):
+                    repository.data.update_data({**dict(REPOSITORY_KEYS_TO_EXPORT), **repo_data})
+                    if (manifest := repo_data.get("manifest")) is not None:
+                        repository.repository_manifest.update_data(
+                            {**dict(HACS_MANIFEST_KEYS_TO_EXPORT), **manifest}
+                        )
+
+        if category == "integration":
+            self.status.inital_fetch_done = True
+
+        if self.stage == HacsStage.STARTUP:
+            for repository in self.repositories.list_all:
+                if (
+                    repository.data.category == category
+                    and not repository.data.installed
+                    and not self.repositories.is_default(repository.data.id)
+                ):
+                    repository.logger.debug(
+                        "%s Unregister stale custom repository", repository.string
+                    )
+                    self.repositories.unregister(repository)
+
     async def async_get_category_repositories(self, category: HacsCategory) -> None:
         """Get repositories from category."""
         if self.system.disabled:
@@ -843,7 +940,7 @@ class HacsBase:
                 return
             can_update = await self.async_can_update()
             self.log.debug(
-                "Can update %s repositories, " "items in queue %s",
+                "Can update %s repositories, items in queue %s",
                 can_update,
                 self.queue.pending_tasks,
             )
@@ -865,6 +962,9 @@ class HacsBase:
         self.log.info("Loading removed repositories")
 
         try:
+            if self.configuration.experimental:
+                removed_repositories = await self.data_client.get_data("removed")
+            else:
                 removed_repositories = await self.async_github_get_hacs_default_file(
                     HacsCategory.REMOVED
                 )
@@ -913,7 +1013,7 @@ class HacsBase:
 
     async def async_update_downloaded_repositories(self, _=None) -> None:
         """Execute the task."""
-        if self.system.disabled:
+        if self.system.disabled or self.configuration.experimental:
             return
         self.log.info("Starting recurring background task for downloaded repositories")
 
@@ -923,6 +1023,21 @@ class HacsBase:
 
         self.log.debug("Recurring background task for downloaded repositories done")
 
+    async def async_update_downloaded_custom_repositories(self, _=None) -> None:
+        """Execute the task."""
+        if self.system.disabled or not self.configuration.experimental:
+            return
+        self.log.info("Starting recurring background task for downloaded custom repositories")
+
+        for repository in self.repositories.list_downloaded:
+            if (
+                repository.data.category in self.common.categories
+                and not self.repositories.is_default(repository.data.id)
+            ):
+                self.queue.add(repository.update_repository(ignore_issues=True))
+
+        self.log.debug("Recurring background task for downloaded custom repositories done")
+
     async def async_handle_critical_repositories(self, _=None) -> None:
         """Handle critical repositories."""
         critical_queue = QueueManager(hass=self.hass)
@@ -931,8 +1046,11 @@ class HacsBase:
         was_installed = False
 
         try:
+            if self.configuration.experimental:
+                critical = await self.data_client.get_data("critical")
+            else:
                 critical = await self.async_github_get_hacs_default_file("critical")
-        except GitHubNotModifiedException:
+        except (GitHubNotModifiedException, HacsNotModifiedException):
             return
         except HacsException:
             pass
@@ -984,3 +1102,43 @@ class HacsBase:
         if was_installed:
             self.log.critical("Restarting Home Assistant")
             self.hass.async_create_task(self.hass.async_stop(100))
+
+    @callback
+    def async_setup_frontend_endpoint_plugin(self) -> None:
+        """Setup the http endpoints for plugins if its not already handled."""
+        if self.status.active_frontend_endpoint_plugin or not os.path.exists(
+            self.hass.config.path("www/community")
+        ):
+            return
+
+        self.log.info("Setting up plugin endpoint")
+        use_cache = self.core.lovelace_mode == "storage"
+        self.log.info(
+            "<HacsFrontend> %s mode, cache for /hacsfiles/: %s",
+            self.core.lovelace_mode,
+            use_cache,
+        )
+
+        self.hass.http.register_static_path(
+            URL_BASE,
+            self.hass.config.path("www/community"),
+            cache_headers=use_cache,
+        )
+
+        self.status.active_frontend_endpoint_plugin = True
+
+    @callback
+    def async_setup_frontend_endpoint_themes(self) -> None:
+        """Setup the http endpoints for themes if its not already handled."""
+        if (
+            self.configuration.experimental
+            or self.status.active_frontend_endpoint_theme
+            or not os.path.exists(self.hass.config.path("themes"))
+        ):
+            return
+
+        self.log.info("Setting up themes endpoint")
+        # Register themes
+        self.hass.http.register_static_path(f"{URL_BASE}/themes", self.hass.config.path("themes"))
+
+        self.status.active_frontend_endpoint_theme = True

custom_components/hacs/const.py

@@ -6,7 +6,9 @@ from aiogithubapi.common.const import ACCEPT_HEADERS
 NAME_SHORT = "HACS"
 DOMAIN = "hacs"
 CLIENT_ID = "395a8e669c5de9f7c6e8"
-MINIMUM_HA_VERSION = "2022.10.0"
+MINIMUM_HA_VERSION = "2022.11.0"
+
+URL_BASE = "/hacsfiles"
 
 TV = TypeVar("TV")
 
@@ -15,6 +17,8 @@ PACKAGE_NAME = "custom_components.hacs"
 DEFAULT_CONCURRENT_TASKS = 15
 DEFAULT_CONCURRENT_BACKOFF_TIME = 1
 
+HACS_REPOSITORY_ID = "172733314"
+
 HACS_ACTION_GITHUB_API_HEADERS = {
     "User-Agent": "HACS/action",
     "Accept": ACCEPT_HEADERS["preview"],

custom_components/hacs/enums.py

@@ -1,16 +1,30 @@
 """Helper constants."""
 # pylint: disable=missing-class-docstring
-from enum import Enum
+import sys
+
+if sys.version_info.minor >= 11:
+    # Needs Python 3.11
+    from enum import StrEnum  # # pylint: disable=no-name-in-module
+else:
+    try:
+        # https://github.com/home-assistant/core/blob/dev/homeassistant/backports/enum.py
+        # Considered internal to Home Assistant, can be removed whenever.
+        from homeassistant.backports.enum import StrEnum
+    except ImportError:
+        from enum import Enum
+
+        class StrEnum(str, Enum):
+            pass
 
 
-class HacsGitHubRepo(str, Enum):
+class HacsGitHubRepo(StrEnum):
     """HacsGitHubRepo."""
 
     DEFAULT = "hacs/default"
     INTEGRATION = "hacs/integration"
 
 
-class HacsCategory(str, Enum):
+class HacsCategory(StrEnum):
     APPDAEMON = "appdaemon"
     INTEGRATION = "integration"
     LOVELACE = "lovelace"
@@ -24,7 +38,7 @@ class HacsCategory(str, Enum):
         return str(self.value)
 
 
-class HacsDispatchEvent(str, Enum):
+class HacsDispatchEvent(StrEnum):
     """HacsDispatchEvent."""
 
     CONFIG = "hacs_dispatch_config"
@@ -37,19 +51,19 @@ class HacsDispatchEvent(str, Enum):
     STATUS = "hacs_dispatch_status"
 
 
-class RepositoryFile(str, Enum):
+class RepositoryFile(StrEnum):
     """Repository file names."""
 
     HACS_JSON = "hacs.json"
     MAINIFEST_JSON = "manifest.json"
 
 
-class ConfigurationType(str, Enum):
+class ConfigurationType(StrEnum):
     YAML = "yaml"
     CONFIG_ENTRY = "config_entry"
 
 
-class LovelaceMode(str, Enum):
+class LovelaceMode(StrEnum):
     """Lovelace Modes."""
 
     STORAGE = "storage"
@@ -58,7 +72,7 @@ class LovelaceMode(str, Enum):
     YAML = "yaml"
 
 
-class HacsStage(str, Enum):
+class HacsStage(StrEnum):
     SETUP = "setup"
     STARTUP = "startup"
     WAITING = "waiting"
@@ -66,7 +80,7 @@ class HacsStage(str, Enum):
     BACKGROUND = "background"
 
 
-class HacsDisabledReason(str, Enum):
+class HacsDisabledReason(StrEnum):
     RATE_LIMIT = "rate_limit"
     REMOVED = "removed"
     INVALID_TOKEN = "invalid_token"

custom_components/hacs/frontend.py

@@ -7,15 +7,13 @@ from aiohttp import web
 from homeassistant.components.http import HomeAssistantView
 from homeassistant.core import HomeAssistant, callback
 
-from .const import DOMAIN
+from .const import DOMAIN, URL_BASE
-from .hacs_frontend import locate_dir, VERSION as FE_VERSION
+from .hacs_frontend import VERSION as FE_VERSION, locate_dir
 from .hacs_frontend_experimental import (
-    locate_dir as experimental_locate_dir,
     VERSION as EXPERIMENTAL_FE_VERSION,
+    locate_dir as experimental_locate_dir,
 )
 
-URL_BASE = "/hacsfiles"
-
 if TYPE_CHECKING:
     from .base import HacsBase
 
@@ -24,8 +22,8 @@ if TYPE_CHECKING:
 def async_register_frontend(hass: HomeAssistant, hacs: HacsBase) -> None:
     """Register the frontend."""
 
-    # Register themes
+    # Setup themes endpoint if needed
-    hass.http.register_static_path(f"{URL_BASE}/themes", hass.config.path("themes"))
+    hacs.async_setup_frontend_endpoint_themes()
 
     # Register frontend
     if hacs.configuration.frontend_repo_url:
@@ -50,20 +48,6 @@ def async_register_frontend(hass: HomeAssistant, hacs: HacsBase) -> None:
         hass.data["frontend_extra_module_url"] = set()
     hass.data["frontend_extra_module_url"].add(f"{URL_BASE}/iconset.js")
 
-    # Register www/community for all other files
-    use_cache = hacs.core.lovelace_mode == "storage"
-    hacs.log.info(
-        "<HacsFrontend> %s mode, cache for /hacsfiles/: %s",
-        hacs.core.lovelace_mode,
-        use_cache,
-    )
-
-    hass.http.register_static_path(
-        URL_BASE,
-        hass.config.path("www/community"),
-        cache_headers=use_cache,
-    )
-
     hacs.frontend_version = (
         FE_VERSION if not hacs.configuration.experimental else EXPERIMENTAL_FE_VERSION
     )
@@ -86,6 +70,9 @@ def async_register_frontend(hass: HomeAssistant, hacs: HacsBase) -> None:
             require_admin=True,
         )
 
+    # Setup plugin endpoint if needed
+    hacs.async_setup_frontend_endpoint_plugin()
+
 
 class HacsFrontendDev(HomeAssistantView):
     """Dev View Class for HACS."""

custom_components/hacs/manifest.json

@@ -17,7 +17,7 @@
     "issue_tracker": "https://github.com/hacs/integration/issues",
     "name": "HACS",
     "requirements": [
-        "aiogithubapi>=22.2.4"
+        "aiogithubapi>=22.10.1"
     ],
-    "version": "1.28.3"
+    "version": "1.30.1"
 }

custom_components/hacs/repositories/base.py

@@ -50,16 +50,27 @@ if TYPE_CHECKING:
 
 
 TOPIC_FILTER = (
+    "add-on",
+    "addon",
+    "app",
+    "appdaemon-apps",
+    "appdaemon",
     "custom-card",
+    "custom-cards",
     "custom-component",
     "custom-components",
     "customcomponents",
     "hacktoberfest",
     "hacs-default",
     "hacs-integration",
+    "hacs-repository",
     "hacs",
     "hass",
     "hassio",
+    "home-assistant-custom",
+    "home-assistant-frontend",
+    "home-assistant-hacs",
+    "home-assistant-sensor",
     "home-assistant",
     "home-automation",
     "homeassistant-components",
@@ -68,16 +79,45 @@ TOPIC_FILTER = (
     "homeassistant",
     "homeautomation",
     "integration",
+    "lovelace-ui",
     "lovelace",
+    "media-player",
+    "mediaplayer",
+    "netdaemon",
+    "plugin",
+    "python_script",
+    "python-script",
     "python",
     "sensor",
+    "smart-home",
+    "smarthome",
     "theme",
     "themes",
-    "custom-cards",
+)
-    "home-assistant-frontend",
+
-    "home-assistant-hacs",
+
-    "home-assistant-custom",
+REPOSITORY_KEYS_TO_EXPORT = (
-    "lovelace-ui",
+    # Keys can not be removed from this list until v3
+    # If keys are added, the action need to be re-run with force
+    ("description", ""),
+    ("downloads", 0),
+    ("domain", None),
+    ("etag_repository", None),
+    ("full_name", ""),
+    ("last_commit", None),
+    ("last_updated", 0),
+    ("last_version", None),
+    ("manifest_name", None),
+    ("open_issues", 0),
+    ("stargazers_count", 0),
+    ("topics", []),
+)
+
+HACS_MANIFEST_KEYS_TO_EXPORT = (
+    # Keys can not be removed from this list until v3
+    # If keys are added, the action need to be re-run with force
+    ("country", []),
+    ("name", None),
 )
 
 
@@ -120,7 +160,6 @@ class RepositoryData:
     new: bool = True
     open_issues: int = 0
     published_tags: list[str] = []
-    pushed_at: str = ""
     releases: bool = False
     selected_tag: str = None
     show_beta: bool = False
@@ -147,32 +186,24 @@ class RepositoryData:
 
     def update_data(self, data: dict, action: bool = False) -> None:
         """Update data of the repository."""
-        for key in data:
+        for key, value in data.items():
             if key not in self.__dict__:
                 continue
-            if key == "pushed_at":
+
-                if data[key] == "":
+            if key == "last_fetched" and isinstance(value, float):
-                    continue
+                setattr(self, key, datetime.fromtimestamp(value))
-                if "Z" in data[key]:
-                    setattr(
-                        self,
-                        key,
-                        datetime.strptime(data[key], "%Y-%m-%dT%H:%M:%SZ"),
-                    )
-                else:
-                    setattr(self, key, datetime.strptime(data[key], "%Y-%m-%dT%H:%M:%S"))
             elif key == "id":
-                setattr(self, key, str(data[key]))
+                setattr(self, key, str(value))
             elif key == "country":
-                if isinstance(data[key], str):
+                if isinstance(value, str):
-                    setattr(self, key, [data[key]])
+                    setattr(self, key, [value])
                 else:
-                    setattr(self, key, data[key])
+                    setattr(self, key, value)
             elif key == "topics" and not action:
-                setattr(self, key, [topic for topic in data[key] if topic not in TOPIC_FILTER])
+                setattr(self, key, [topic for topic in value if topic not in TOPIC_FILTER])
 
             else:
-                setattr(self, key, data[key])
+                setattr(self, key, value)
 
 
 @attr.s(auto_attribs=True)
@@ -215,6 +246,20 @@ class HacsManifest:
                 setattr(manifest_data, key, value)
         return manifest_data
 
+    def update_data(self, data: dict) -> None:
+        """Update the manifest data."""
+        for key, value in data.items():
+            if key not in self.__dict__:
+                continue
+
+            if key == "country":
+                if isinstance(value, str):
+                    setattr(self, key, [value])
+                else:
+                    setattr(self, key, value)
+            else:
+                setattr(self, key, value)
+
 
 class RepositoryReleases:
     """RepositoyReleases."""
@@ -449,6 +494,10 @@ class HacsRepository:
                 self.logger.debug("%s Did not update, content was not modified", self.string)
                 return
 
+        if self.repository_object:
+            self.data.last_updated = self.repository_object.attributes.get("pushed_at", 0)
+            self.data.last_fetched = datetime.utcnow()
+
         # Set topics
         self.data.topics = self.data.topics
 
@@ -497,7 +546,7 @@ class HacsRepository:
         self.additional_info = await self.async_get_info_file_contents()
 
         # Set last fetch attribute
-        self.data.last_fetched = datetime.now()
+        self.data.last_fetched = datetime.utcnow()
 
         return True
 
@@ -1011,7 +1060,11 @@ class HacsRepository:
                 self.hacs.common.renamed_repositories[
                     self.data.full_name
                 ] = repository_object.full_name
+                if not self.hacs.system.generator:
                     raise HacsRepositoryExistException
+                self.logger.error(
+                    "%s Repository has been renamed - %s", self.string, repository_object.full_name
+                )
             self.data.update_data(
                 repository_object.attributes,
                 action=self.hacs.system.action,

custom_components/hacs/repositories/integration.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 
 from typing import TYPE_CHECKING, Any
 
-from homeassistant.helpers.issue_registry import async_create_issue, IssueSeverity
+from homeassistant.helpers.issue_registry import IssueSeverity, async_create_issue
 from homeassistant.loader import async_get_custom_components
 
 from ..const import DOMAIN

custom_components/hacs/repositories/plugin.py

@@ -53,6 +53,10 @@ class HacsPluginRepository(HacsRepository):
                     self.logger.error("%s %s", self.string, error)
         return self.validate.success
 
+    async def async_post_installation(self):
+        """Run post installation steps."""
+        self.hacs.async_setup_frontend_endpoint_plugin()
+
     @concurrent(concurrenttasks=10, backoff_time=5)
     async def update_repository(self, ignore_issues=False, force=False):
         """Update."""

custom_components/hacs/repositories/theme.py

@@ -37,6 +37,8 @@ class HacsThemeRepository(HacsRepository):
         except BaseException:  # lgtm [py/catch-base-exception] pylint: disable=broad-except
             pass
 
+        self.hacs.async_setup_frontend_endpoint_themes()
+
     async def validate_repository(self):
         """Validate."""
         # Run common validation steps.

custom_components/hacs/sensor.py

@@ -1,5 +1,6 @@
 """Sensor platform for HACS."""
 from __future__ import annotations
+
 from typing import TYPE_CHECKING
 
 from homeassistant.components.sensor import SensorEntity

custom_components/hacs/system_health.py

@@ -7,6 +7,7 @@ from .base import HacsBase
 from .const import DOMAIN
 
 GITHUB_STATUS = "https://www.githubstatus.com/"
+CLOUDFLARE_STATUS = "https://www.cloudflarestatus.com/"
 
 
 @callback
@@ -39,4 +40,9 @@ async def system_health_info(hass):
     if hacs.system.disabled:
         data["Disabled"] = hacs.system.disabled_reason
 
+    if hacs.configuration.experimental:
+        data["HACS Data"] = system_health.async_check_can_reach_url(
+            hass, "https://data-v2.hacs.xyz/data.json", CLOUDFLARE_STATUS
+        )
+
     return data

custom_components/hacs/update.py

@@ -90,6 +90,17 @@ class HacsRepositoryUpdateEntity(HacsRepositoryEntity, UpdateEntity):
         if self.repository.pending_restart or not self.repository.can_download:
             return None
 
+        if self.latest_version not in self.repository.data.published_tags:
+            releases = await self.repository.get_releases(
+                prerelease=self.repository.data.show_beta,
+                returnlimit=self.hacs.configuration.release_limit,
+            )
+            if releases:
+                self.repository.data.releases = True
+                self.repository.releases.objects = releases
+                self.repository.data.published_tags = [x.tag_name for x in releases]
+                self.repository.data.last_version = next(iter(self.repository.data.published_tags))
+
         release_notes = ""
         if len(self.repository.releases.objects) > 0:
             release = self.repository.releases.objects[0]

custom_components/hacs/utils/data.py

@@ -1,38 +1,45 @@
 """Data handler for HACS."""
+from __future__ import annotations
+
 import asyncio
 from datetime import datetime
+from typing import Any
 
 from homeassistant.core import callback
 from homeassistant.exceptions import HomeAssistantError
 from homeassistant.util import json as json_util
 
 from ..base import HacsBase
-from ..enums import HacsDisabledReason, HacsDispatchEvent, HacsGitHubRepo
+from ..const import HACS_REPOSITORY_ID
+from ..enums import HacsDisabledReason, HacsDispatchEvent
 from ..repositories.base import TOPIC_FILTER, HacsManifest, HacsRepository
 from .logger import LOGGER
 from .path import is_safe
 from .store import async_load_from_store, async_save_to_store
 
-DEFAULT_BASE_REPOSITORY_DATA = (
+EXPORTED_BASE_DATA = (
+    ("new", False),
+    ("full_name", ""),
+)
+
+EXPORTED_REPOSITORY_DATA = EXPORTED_BASE_DATA + (
     ("authors", []),
     ("category", ""),
     ("description", ""),
     ("domain", None),
     ("downloads", 0),
     ("etag_repository", None),
-    ("full_name", ""),
-    ("last_updated", 0),
     ("hide", False),
+    ("last_updated", 0),
     ("new", False),
     ("stargazers_count", 0),
     ("topics", []),
 )
 
-DEFAULT_EXTENDED_REPOSITORY_DATA = (
+EXPORTED_DOWNLOADED_REPOSITORY_DATA = EXPORTED_REPOSITORY_DATA + (
     ("archived", False),
     ("config_flow", False),
     ("default_branch", None),
-    ("description", ""),
     ("first_install", False),
     ("installed_commit", None),
     ("installed", False),
@@ -41,12 +48,9 @@ DEFAULT_EXTENDED_REPOSITORY_DATA = (
     ("manifest_name", None),
     ("open_issues", 0),
     ("published_tags", []),
-    ("pushed_at", ""),
     ("releases", False),
     ("selected_tag", None),
     ("show_beta", False),
-    ("stargazers_count", 0),
-    ("topics", []),
 )
 
 
@@ -80,6 +84,8 @@ class HacsData:
                 "ignored_repositories": self.hacs.common.ignored_repositories,
             },
         )
+        if self.hacs.configuration.experimental:
+            await self._async_store_experimental_content_and_repos()
         await self._async_store_content_and_repos()
 
     async def _async_store_content_and_repos(self, _=None):  # bb: ignore
@@ -94,40 +100,94 @@ class HacsData:
         for event in (HacsDispatchEvent.REPOSITORY, HacsDispatchEvent.CONFIG):
             self.hacs.async_dispatch(event, {})
 
+    async def _async_store_experimental_content_and_repos(self, _=None):  # bb: ignore
+        """Store the main repos file and each repo that is out of date."""
+        # Repositories
+        self.content = {}
+        for repository in self.hacs.repositories.list_all:
+            if repository.data.category in self.hacs.common.categories:
+                self.async_store_experimental_repository_data(repository)
+
+        await async_save_to_store(self.hacs.hass, "data", {"repositories": self.content})
+
     @callback
     def async_store_repository_data(self, repository: HacsRepository) -> dict:
         """Store the repository data."""
         data = {"repository_manifest": repository.repository_manifest.manifest}
 
-        for key, default_value in DEFAULT_BASE_REPOSITORY_DATA:
+        for key, default in (
-            if (value := repository.data.__getattribute__(key)) != default_value:
+            EXPORTED_DOWNLOADED_REPOSITORY_DATA
+            if repository.data.installed
+            else EXPORTED_REPOSITORY_DATA
+        ):
+            if (value := getattr(repository.data, key, default)) != default:
                 data[key] = value
 
-        if repository.data.installed:
+        if repository.data.installed_version:
-            for key, default_value in DEFAULT_EXTENDED_REPOSITORY_DATA:
-                if (value := repository.data.__getattribute__(key)) != default_value:
-                    data[key] = value
             data["version_installed"] = repository.data.installed_version
-
         if repository.data.last_fetched:
             data["last_fetched"] = repository.data.last_fetched.timestamp()
 
         self.content[str(repository.data.id)] = data
 
+    @callback
+    def async_store_experimental_repository_data(self, repository: HacsRepository) -> None:
+        """Store the experimental repository data for non downloaded repositories."""
+        data = {}
+        self.content.setdefault(repository.data.category, [])
+
+        if repository.data.installed:
+            data["repository_manifest"] = repository.repository_manifest.manifest
+            for key, default in EXPORTED_DOWNLOADED_REPOSITORY_DATA:
+                if (value := getattr(repository.data, key, default)) != default:
+                    data[key] = value
+
+            if repository.data.installed_version:
+                data["version_installed"] = repository.data.installed_version
+            if repository.data.last_fetched:
+                data["last_fetched"] = repository.data.last_fetched.timestamp()
+        else:
+            for key, default in EXPORTED_BASE_DATA:
+                if (value := getattr(repository.data, key, default)) != default:
+                    data[key] = value
+
+        self.content[repository.data.category].append({"id": str(repository.data.id), **data})
+
     async def restore(self):
         """Restore saved data."""
         self.hacs.status.new = False
-        try:
+        repositories = {}
-            hacs = await async_load_from_store(self.hacs.hass, "hacs") or {}
-        except HomeAssistantError:
         hacs = {}
 
         try:
-            repositories = await async_load_from_store(self.hacs.hass, "repositories") or {}
+            hacs = await async_load_from_store(self.hacs.hass, "hacs") or {}
+        except HomeAssistantError:
+            pass
+
+        try:
+            data = (
+                await async_load_from_store(
+                    self.hacs.hass,
+                    "data" if self.hacs.configuration.experimental else "repositories",
+                )
+                or {}
+            )
+            if data and self.hacs.configuration.experimental:
+                for category, entries in data.get("repositories", {}).items():
+                    for repository in entries:
+                        repositories[repository["id"]] = {"category": category, **repository}
+            else:
+                repositories = (
+                    data or await async_load_from_store(self.hacs.hass, "repositories") or {}
+                )
         except HomeAssistantError as exception:
             self.hacs.log.error(
                 "Could not read %s, restore the file from a backup - %s",
-                self.hacs.hass.config.path(".storage/hacs.repositories"),
+                self.hacs.hass.config.path(
+                    ".storage/hacs.data"
+                    if self.hacs.configuration.experimental
+                    else ".storage/hacs.repositories"
+                ),
                 exception,
             )
             self.hacs.disable_hacs(HacsDisabledReason.RESTORE)
@@ -136,6 +196,8 @@ class HacsData:
         if not hacs and not repositories:
             # Assume new install
             self.hacs.status.new = True
+            if self.hacs.configuration.experimental:
+                return True
             self.logger.info("<HacsData restore> Loading base repository information")
             repositories = await self.hacs.hass.async_add_executor_job(
                 json_util.load_json,
@@ -186,28 +248,34 @@ class HacsData:
             return False
         return True
 
-    async def register_unknown_repositories(self, repositories):
+    async def register_unknown_repositories(self, repositories, category: str | None = None):
         """Registry any unknown repositories."""
         register_tasks = [
             self.hacs.async_register_repository(
                 repository_full_name=repo_data["full_name"],
-                category=repo_data["category"],
+                category=repo_data.get("category", category),
                 check=False,
                 repository_id=entry,
             )
             for entry, repo_data in repositories.items()
-            if entry != "0" and not self.hacs.repositories.is_registered(repository_id=entry)
+            if entry != "0"
+            and not self.hacs.repositories.is_registered(repository_id=entry)
+            and repo_data.get("category", category) is not None
         ]
         if register_tasks:
             await asyncio.gather(*register_tasks)
 
     @callback
-    def async_restore_repository(self, entry, repository_data):
+    def async_restore_repository(self, entry: str, repository_data: dict[str, Any]):
         """Restore repository."""
-        full_name = repository_data["full_name"]
+        repository: HacsRepository | None = None
-        if not (repository := self.hacs.repositories.get_by_full_name(full_name)):
+        if full_name := repository_data.get("full_name"):
-            self.logger.error("<HacsData restore> Did not find %s (%s)", full_name, entry)
+            repository = self.hacs.repositories.get_by_full_name(full_name)
+        if not repository:
+            repository = self.hacs.repositories.get_by_id(entry)
+        if not repository:
             return
+
         # Restore repository attributes
         self.hacs.repositories.set_repository_id(repository, entry)
         repository.data.authors = repository_data.get("authors", [])
@@ -238,7 +306,7 @@ class HacsData:
             repository.data.last_fetched = datetime.fromtimestamp(last_fetched)
 
         repository.repository_manifest = HacsManifest.from_dict(
-            repository_data.get("repository_manifest", {})
+            repository_data.get("manifest") or repository_data.get("repository_manifest") or {}
         )
 
         if repository.localpath is not None and is_safe(self.hacs, repository.localpath):
@@ -248,6 +316,6 @@ class HacsData:
         if repository.data.installed:
             repository.data.first_install = False
 
-        if full_name == HacsGitHubRepo.INTEGRATION:
+        if entry == HACS_REPOSITORY_ID:
             repository.data.installed_version = self.hacs.version
             repository.data.installed = True