add roundlookup_uk serving Malvern Hills, Wychavon, Worcester City

README.md

@@ -1486,6 +1486,8 @@ If your service provider is not listed, feel free to open a [source request issu
 - [Luton](/doc/source/apps_imactivate_com.md) / luton.gov.uk
 - [Maidstone Borough Council](/doc/source/maidstone_gov_uk.md) / maidstone.gov.uk
 - [Maldon District Council](/doc/source/maldon_gov_uk.md) / maldon.gov.uk
+- [Malvern Hills](/doc/source/roundlookup_uk.md) / malvernhills.gov.uk
+- [Malvern Hills District Council](/doc/source/roundlookup_uk.md) / malvernhills.gov.uk
 - [Manchester City Council](/doc/source/manchester_uk.md) / manchester.gov.uk
 - [Mansfield District Council](/doc/source/mansfield_gov_uk.md) / mansfield.gov.uk
 - [Mendip District Council](/doc/source/iweb_itouchvision_com.md) / somerset.gov.uk
@@ -1586,7 +1588,9 @@ If your service provider is not listed, feel free to open a [source request issu
 - [Wirral Council](/doc/source/wirral_gov_uk.md) / wirral.gov.uk
 - [Woking Borough Council](/doc/source/jointwastesolutions_org.md) / woking.gov.uk
 - [Wokingham Borough Council](/doc/source/wokingham_gov_uk.md) / wokingham.gov.uk
-- [Wychavon District Council](/doc/source/wychavon_gov_uk.md) / wychavon.gov.uk
+- [Worcester City](/doc/source/roundlookup_uk.md) / worcester.gov.uk
+- [Wychavon](/doc/source/roundlookup_uk.md) / wychavon.gov.uk
+- [Wychavon District Council (Deprecated)](/doc/source/wychavon_gov_uk.md) / wychavon.gov.uk
 - [Wyre Forest District Council](/doc/source/wyreforestdc_gov_uk.md) / wyreforestdc.gov.uk
 </details>
 

custom_components/waste_collection_schedule/const.py

@@ -7,7 +7,7 @@ DOMAIN: Final = "waste_collection_schedule"
 
 UPDATE_SENSORS_SIGNAL: Final = "wcs_update_sensors_signal"
 
-CONFIG_VERSION: Final = 1
+CONFIG_VERSION: Final = 2
 
 # Config var names
 CONF_SOURCES: Final = "sources"

custom_components/waste_collection_schedule/init_ui.py

@@ -7,6 +7,7 @@ from typing import Any
 
 import homeassistant.helpers.config_validation as cv
 import homeassistant.util.dt as dt_util
+from homeassistant.config_entries import ConfigEntry
 from homeassistant.core import HomeAssistant, callback
 from homeassistant.helpers.device_registry import DeviceEntryType, DeviceInfo
 from homeassistant.helpers.dispatcher import dispatcher_send
@@ -98,7 +99,7 @@ async def async_unload_entry(hass, entry):
     return await hass.config_entries.async_unload_platforms(entry, PLATFORMS)
 
 
-async def async_migrate_entry(hass, config_entry) -> bool:
+async def async_migrate_entry(hass, config_entry: ConfigEntry) -> bool:
     """Migrate old entry."""
     # Version number has gone backwards
     if const.CONFIG_VERSION < config_entry.version:
@@ -110,10 +111,17 @@ async def async_migrate_entry(hass, config_entry) -> bool:
     # Version number has gone up
     if config_entry.version < const.CONFIG_VERSION:
         _LOGGER.debug("Migrating from version %s", config_entry.version)
-        new_data = config_entry.data
+        new_data = {**config_entry.data}
 
-        config_entry.version = const.CONFIG_VERSION
-        hass.config_entries.async_update_entry(config_entry, data=new_data)
+        if config_entry.version < 2 and const.CONFIG_VERSION >= 2:
+            if new_data.get("name", "") == "wychavon_gov_uk":
+                _LOGGER.debug("Migrating from wychavon_gov_uk to roundlookup_uk")
+                new_data["name"] = "roundlookup_uk"
+                new_data["args"]["council"] = "Wychavon"
+
+        hass.config_entries.async_update_entry(
+            config_entry, data=new_data, version=const.CONFIG_VERSION
+        )
 
         _LOGGER.debug("Migration to version %s successful", config_entry.version)
 

custom_components/waste_collection_schedule/sources.json

@@ -8005,6 +8005,18 @@
       "module": "maldon_gov_uk",
       "default_params": {}
     },
+    {
+      "title": "Malvern Hills",
+      "module": "roundlookup_uk",
+      "default_params": {
+        "council": "Malvern Hills"
+      }
+    },
+    {
+      "title": "Malvern Hills District Council",
+      "module": "roundlookup_uk",
+      "default_params": {}
+    },
     {
       "title": "Manchester City Council",
       "module": "manchester_uk",
@@ -8534,7 +8546,21 @@
       "default_params": {}
     },
     {
-      "title": "Wychavon District Council",
+      "title": "Worcester City",
+      "module": "roundlookup_uk",
+      "default_params": {
+        "council": "Worcester City"
+      }
+    },
+    {
+      "title": "Wychavon",
+      "module": "roundlookup_uk",
+      "default_params": {
+        "council": "Wychavon"
+      }
+    },
+    {
+      "title": "Wychavon District Council (Deprecated)",
       "module": "wychavon_gov_uk",
       "default_params": {}
     },

custom_components/waste_collection_schedule/waste_collection_schedule/source/roundlookup_uk.py

@@ -0,0 +1,131 @@
+import logging
+from datetime import datetime
+from typing import Literal
+
+import requests
+import urllib3
+from bs4 import BeautifulSoup
+from waste_collection_schedule import Collection  # type: ignore[attr-defined]
+
+# With verify=True the POST fails due to a SSLCertVerificationError.
+# Using verify=False works, but is not ideal. The following links may provide a better way of dealing with this:
+# https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings
+# https://urllib3.readthedocs.io/en/1.26.x/user-guide.html#ssl
+# This line suppresses the InsecureRequestWarning when using verify=False
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
+
+_LOGGER = logging.getLogger(__name__)
+
+TITLE = "Malvern Hills District Council"
+DESCRIPTION = "Source for Malvern Hills District Council."
+URL = "https://www.malvernhills.gov.uk/"
+TEST_CASES = {
+    "1Malvern Hill: s00120597618": {"uprn": 100120597618, "council": "Malvern Hills"},
+    "Malvern Hills: 100121268004": {
+        "uprn": "100121268004",
+        "council": "Malvern Hills",
+    },
+    "Worcester City: 100120656169": {
+        "uprn": 100120656169,
+        "council": "Worcester City",
+    },
+    "Wychavon: 10095592085": {"uprn": 10095592085, "council": "Wychavon"},
+}
+
+
+ICON_MAP = {
+    "Non-recyclable": "mdi:trash-can",
+    "Garden": "mdi:leaf",
+    "Recycling": "mdi:recycle",
+}
+
+
+SERVICE_MAP = {
+    "Malvern Hills": {
+        "api_url": "https://swict.malvernhills.gov.uk/mhdcroundlookup/HandleSearchScreen",
+        "url": "https://www.malvernhills.gov.uk/",
+    },
+    "Wychavon": {
+        "api_url": "https://selfservice.wychavon.gov.uk/wdcroundlookup/HandleSearchScreen",
+        "url": "https://www.wychavon.gov.uk/",
+    },
+    "Worcester City": {
+        "api_url": "https://selfserve.worcester.gov.uk/wccroundlookup/HandleSearchScreen",
+        "url": "https://www.worcester.gov.uk/",
+    },
+}
+
+EXTRA_INFO = [
+    {
+        "url": value["url"],
+        "title": key,
+        "default_params": {"council": key},
+    }
+    for key, value in SERVICE_MAP.items()
+]
+
+COUNIL_LITERAL = Literal["Malvern Hills", "Wychavon", "Worcester City"]
+
+
+class Source:
+    def __init__(self, uprn: str | int, council: COUNIL_LITERAL) -> None:
+        self._api_url = SERVICE_MAP.get(council, {}).get("api_url", "")
+        if not self._api_url:
+            raise ValueError(
+                f"District '{council}' not supported, use one of {SERVICE_MAP.keys()}"
+            )
+        self._uprn: str | int = uprn
+
+    def fetch(self) -> list[Collection]:
+        data: dict[str, str | int] = {
+            "alAddrsel": self._uprn,
+            "txtPage": "std",
+            "txtSearchPerformedFlag": "false",
+            "futuredate": "",
+            "address": "",
+            "btnSubmit": "Next",
+        }
+
+        r = requests.post(self._api_url, data=data, verify=False)
+        r.raise_for_status()
+
+        soup = BeautifulSoup(r.text, "html.parser")
+
+        table = soup.select_one("table")
+        if not table:
+            raise ValueError("collection table not, maybe wrong/unsupported address")
+
+        entries = []
+        for tr in table.select("tr"):
+            tds = tr.select("td")
+            if not len(tds) == 3:
+                continue
+
+            bin_type_tag = tds[1]
+            # remove all divs
+            div = bin_type_tag.select_one("div")
+            if div:
+                div.decompose()
+
+            bin_type = bin_type_tag.text.strip().removesuffix("collection").strip()
+            date_text = tds[2].text.strip()
+            if "Not applicable" in date_text:
+                continue
+            icon = ICON_MAP.get(bin_type.split()[0])
+
+            # Thursday 22/08/2024
+            for date_str in date_text.split("\n"):
+                date_str = date_str.strip()
+                if not date_str:
+                    continue
+                try:
+                    date = datetime.strptime(date_str, "%A %d/%m/%Y").date()
+                except ValueError:
+                    _LOGGER.warning(
+                        "Could not parse date '%s', unknown format", date_str
+                    )
+                    continue
+                entries.append(Collection(date=date, t=bin_type, icon=icon))
+
+        return entries

custom_components/waste_collection_schedule/waste_collection_schedule/source/wychavon_gov_uk.py

@@ -1,18 +1,10 @@
-from datetime import datetime
+import logging
 
-import requests
-import urllib3
-from bs4 import BeautifulSoup, Tag
-from waste_collection_schedule import Collection  # type: ignore[attr-defined]
+from waste_collection_schedule.source.roundlookup_uk import \
+    Source as Roundlookup  # type: ignore[attr-defined]
 
-# With verify=True the POST fails due to a SSLCertVerificationError.
-# Using verify=False works, but is not ideal. The following links may provide a better way of dealing with this:
-# https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings
-# https://urllib3.readthedocs.io/en/1.26.x/user-guide.html#ssl
-# This line suppresses the InsecureRequestWarning when using verify=False
-urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
-
-TITLE = "Wychavon District Council"
+_LOGGER = logging.getLogger(__name__)
+TITLE = "Wychavon District Council (Deprecated)"
 DESCRIPTION = "Source for Wychavon District Council."
 URL = "https://wychavon.gov.uk/"
 TEST_CASES = {
@@ -29,54 +21,9 @@ ICON_MAP = {
 }
 
 
-API_URL = "https://selfservice.wychavon.gov.uk/wdcroundlookup/HandleSearchScreen"
-
-
-class Source:
+class Source(Roundlookup):
     def __init__(self, uprn: str | int):
-        self._uprn: str | int = uprn
-
-    def fetch(self) -> list[Collection]:
-        data = {"alAddrsel": self._uprn}
-
-        # get json file
-        r = requests.post(API_URL, data=data, verify=False)
-        r.raise_for_status()
-
-        soup = BeautifulSoup(r.text, "html.parser")
-
-        table = soup.find("table", {"class": "table table-striped"})
-        rows = table.find_all("tr")
-
-        entries: list[Collection] = []
-        for row in rows:
-            if not isinstance(row, Tag):
-                continue
-            tds: list[Tag] = row.find_all("td")
-            if len(tds) < 3:
-                continue
-
-            # remove everything inside tds[1] that's not text directly inside the tag
-            for tag in tds[1].find_all():
-                if not isinstance(tag, Tag):
-                    continue
-                if tag.name != "br":
-                    tag.decompose()
-
-            collection_type = tds[1].text.strip()
-            date_strs = [date.text.strip() for date in tds[2] if date.text.strip()]
-
-            for date_str in date_strs:
-                try:
-                    # Like Thursday 11/07/2024
-                    date = datetime.strptime(date_str, "%A %d/%m/%Y").date()
-                except ValueError:
-                    continue
-
-                entries.append(
-                    Collection(
-                        date, collection_type, ICON_MAP.get(collection_type.split()[0])
-                    )
-                )
-
-        return entries
+        super().__init__(uprn, "Wychavon")
+        _LOGGER.warning(
+            "This source is deprecated, please use the 'roundlookup_uk' source instead"
+        )

doc/source/roundlookup_uk.md

@@ -0,0 +1,53 @@
+# Malvern Hills District Council
+
+Support for schedules provided by multiple UK councils via the roundlookup service.
+
+## Configuration via configuration.yaml
+
+```yaml
+waste_collection_schedule:
+    sources:
+    - name: roundlookup_uk
+      args:
+        council: "DISTRRICT" # see below
+        uprn: "UPRN"
+```
+
+### Configuration Variables
+
+**uprn**  
+*(String | Integer) (required)*
+
+**council**  
+*(String) (required)*
+
+should be one of the following:
+
+- "Malvern Hills": <https://swict.malvernhills.gov.uk/mhdcroundlookup/HandleSearchScreen>
+- "Wychavon": <https://selfservice.wychavon.gov.uk/wdcroundlookup/HandleSearchScreen>
+- "Worcester City": <https://selfserve.worcester.gov.uk/wccroundlookup/HandleSearchScreen>
+
+## Example
+
+```yaml
+waste_collection_schedule:
+    sources:
+    - name: roundlookup_uk
+      args:
+        uprn: "100120597618"
+```
+
+## How to get the source argument
+
+### Easy way (with external tool)
+
+An easy way to discover your Unique Property Reference Number (UPRN) is by going to <https://www.findmyaddress.co.uk/> and entering in your address details.
+
+### Harder way (with browser developer tools)
+
+1. Go to the Bin day form of your council (see above)
+1. Enter your postcode and click "Find address".
+1. Right click -> inspect element on the address dropdown.
+1. Your UPRN will be in the value attribute of the option tag containing your address.
+
+