added Source for Korneuburg

README.md

@@ -75,6 +75,7 @@ Currently the following service providers are supported:
 
 - [BMV.at](./doc/source/bmv_at.md)
 - [Data.Umweltprofis](./doc/source/data_umweltprofis_at.md)
+- [Korneuburg Stadtservice](./doc/source/korneuburg_stadtservice_at.md)
 - [WSZ-Moosburg.at](./doc/source/wsz_moosburg_at.md)
 
 ### Belgium

custom_components/waste_collection_schedule/waste_collection_schedule/source/korneuburg_stadtservice_at.py

@@ -0,0 +1,181 @@
+import json
+from urllib.parse import urljoin
+
+import requests
+from bs4 import BeautifulSoup
+from requests.adapters import HTTPAdapter, Retry
+from waste_collection_schedule import Collection  # type: ignore[attr-defined]
+from waste_collection_schedule.service.ICS import ICS  # type: ignore[attr-defined]
+
+TITLE = 'Müllabfuhr Korneuburg'
+DESCRIPTION = 'Source for Stadtservice Korneuburg'
+URL = 'https://www.korneuburg.gv.at'
+
+# Mapping of teilgebiete to calendar urls
+WASTE_TYPE_URLS = {
+    '1': ('Biomuell_3', 'Restmuell_3', 'Papier_2', 'Gelber_Sack_4'),
+    '2': ('Biomuell_4', 'Restmuell_2', 'Papier_3', 'Gelber_Sack_1'),
+    '3': ('Biomuell_1', 'Restmuell_1', 'Papier_1', 'Gelber_Sack_2'),
+    '4': ('Biomuell_2', 'Restmuell',   'Papier',   'Gelber_Sack_3')
+}
+
+TEST_CASES = {
+    "Rathaus": {"street_name": "Hauptplatz", "street_number": 39},                                         # Teilgebiet 4
+    "Rathaus using Teilgebiet": {"street_name": "Some Street", "street_number": "1A", "teilgebiet": "4"},  # Teilgebiet 4
+    "Werft": {"street_name": "Am Hafen", "street_number": 6}                                               # Teilgebiet 2
+}
+
+
+def http_get_retry(params, n_retries=5):
+    """HTTP GET with n retries """
+    s = requests.Session()
+
+    retries = Retry(total=n_retries,
+                    backoff_factor=0.1,
+                    status_forcelist=[500, 502, 503, 504])
+
+    s.mount('https://', HTTPAdapter(max_retries=retries))
+
+    r = s.get(**params)
+
+    return r
+
+
+class Source:
+    def __init__(self, street_name, street_number, teilgebiet=-1):
+        self.street_name = street_name
+        self.street_number = street_number
+
+        self._street_name_id = -1
+        self._street_number_id = -1
+        self._headers = {'User-Agent': 'Mozilla/5.0'}
+        self._cookies = {'ris_cookie_setting': 'g7750'}  # Accept Cookie Consent
+        self._ics = ICS()
+
+        if 0 < int(teilgebiet) <= 4:
+            self.region = str(teilgebiet)
+        else:
+            self.region = self.determine_region()
+
+    @staticmethod
+    def extract_street_numbers(soup):
+
+        scripts = soup.findAll("script", {"type": "text/javascript"})
+
+        street_number_idx = 0
+        for s in scripts:
+            if s.string and "var strassenArr" in s.string:
+                break
+            street_number_idx += 1
+
+        possible_numbers = json.loads(
+            scripts[street_number_idx].string[19:].replace('\r\n', '').replace(', ]', ']').replace('\'', '"'))
+
+        number_dict = dict()
+
+        for idx, street_id in enumerate(possible_numbers):
+            number_dict[street_id[0]] = {e[1]: (e[0], e[2]) for _idx, e in enumerate(possible_numbers[idx][1])}
+
+        return number_dict
+
+    @staticmethod
+    def extract_street_names(soup):
+        street_selector = soup.find("select", {"id": "225991280_boxmuellkalenderstrassedd"}).findAll("option")
+        available_streets = {street.string: int(street["value"]) for _idx, street in enumerate(street_selector)}
+
+        return available_streets
+
+    @staticmethod
+    def extract_region(soup):
+        region = -1
+
+        for span in soup.findAll("span"):
+            if span.parent.name == 'td' and "teilgebiet" in span.string.lower():
+                region = span.string.split(' ')[1]
+                break
+
+        return region
+
+    def determine_region(self):
+        """finds the target region for the street and street number"""
+
+        # request address selection form
+        url = urljoin(URL, "Rathaus/Buergerservice/Muellabfuhr")
+        page = http_get_retry(params=dict(url=url, headers=self._headers, cookies=self._cookies))
+        soup = BeautifulSoup(page.content, "html.parser")
+
+        # extract possible street and number combinations from html source
+        available_streets = self.extract_street_names(soup)
+        number_dict = self.extract_street_numbers(soup)
+
+        street_found = self.street_name in available_streets.keys()
+
+        if not street_found:
+            raise Exception(f"{self.street_name} not found. Please check back spelling with the official site: {url}")
+
+        self._street_name_id = available_streets.get(self.street_name)
+
+        self._street_number_id, street_number_link = number_dict.get(
+            available_streets.get(self.street_name)).get(str(self.street_number), (-1, 'not found'))
+
+        if street_number_link == 'not found':
+            raise Exception(f"{self.street_number} not found. Available numbers for {self.street_name} are\
+             {list(number_dict.get(available_streets['Am Hafen']).keys())}")
+
+        # add selection cookie
+        self._cookies['riscms_muellkalender'] = str(f"{self._street_name_id}_{self._street_number_id}")
+
+        # request overview with address selection to get the region
+        url = urljoin(URL, "system/web/kalender.aspx?sprache=1&menuonr=225991280&typids=" + street_number_link)
+        page = http_get_retry(params=dict(url=url, headers=self._headers, cookies=self._cookies))
+        soup = BeautifulSoup(page.content, "html.parser")
+
+        region = self.extract_region(soup)
+
+        if region == -1:
+            raise Exception(f"Region could not be found")
+
+        return str(region)
+
+    def get_region_links(self):
+        """traverses the pages for different waste types and collects download links for the iCals"""
+
+        # create waste type urls
+        ical_urls = []
+        urls = [urljoin(URL, u) for u in WASTE_TYPE_URLS.get(self.region)]
+
+        for u in urls:
+            r = http_get_retry(params=dict(url=u, headers=self._headers, cookies=self._cookies))
+            soup = BeautifulSoup(r.content, "html.parser")
+            download_link = soup.findAll("a", {"class": "piwik_download_tracker", "data-trackingtyp": "iCal/Kalender"})
+            if len(download_link):
+                ical_urls.append(urljoin(URL, download_link[0].get("href")))
+
+        return ical_urls
+
+    def process_waste_type(self, url):
+        """downloads one calendar and returns list with entries"""
+
+        r = http_get_retry(params=dict(url=url, headers=self._headers, cookies=self._cookies))
+        r.encoding = r.apparent_encoding
+
+        dates = self._ics.convert(r.text)
+
+        entries = [Collection(d[0], d[1]) for d in dates]
+
+        return entries
+
+    def fetch(self):
+
+        ical_urls = self.get_region_links()
+
+        for u in ical_urls:
+            print(u)
+
+        all_entries = []
+
+        for ical in ical_urls:
+            entries = self.process_waste_type(url=ical)
+            all_entries = all_entries + entries
+
+        return all_entries

doc/source/korneuburg_stadtservice_at.md

@@ -0,0 +1,66 @@
+# Abfallkalender der Gemeinde Korneuburg
+
+Support for schedules provided by [Stadtservice Korneuburg](https://www.korneuburg.gv.at/Rathaus/Buergerservice/Muellabfuhr) located in Korneuburg, Austria.
+
+## Configuration via configuration.yaml
+
+```yaml
+waste_collection_schedule:
+  sources:
+    - name: korneuburg_stadtservice_at
+      args:
+        street_name: Straßenname
+        street_number: 1A
+        teilgebiet: 1
+```
+
+### Configuration Variables
+
+**street_name**<br>
+*(string) (required)*
+
+**street_number**<br>
+*(string) (required)*
+
+**teilgebiet**<br>
+*(string) (optional)*
+
+### How to get the source arguments
+The arguments can be found on [Stadtservice Korneuburg](https://www.korneuburg.gv.at/Rathaus/Buergerservice/Muellabfuhr).
+
+Check if your address details are available on the official site. If not use something that is close by or the same region.
+
+You can enter your region number (`teilgebiet`) directly to skip the step that determines your region based on your address.
+Still some values need to be set for `street_name` and `street_number` which are then not used.
+<br>
+
+## Example
+
+**First Entry**
+```yaml
+waste_collection_schedule:
+  sources:
+    - name: korneuburg_stadtservice_at
+      args:
+        street_name: "Albrecht Dürer-Gasse"
+        street_number: 2
+```
+**Rathaus**
+```yaml
+waste_collection_schedule:
+  sources:
+    - name: korneuburg_stadtservice_at
+      args:
+        street_name: Hauptplatz
+        street_number: 39
+```
+**Rathaus using Teilgebiet**
+```yaml
+waste_collection_schedule:
+  sources:
+    - name: korneuburg_stadtservice_at
+      args:
+        street_name: Some Street
+        street_number: 1A
+        teilgebiet: 4
+```

info.md

@@ -61,6 +61,7 @@ Currently the following service providers are supported:
 
 - [BMV.at](https://github.com/mampfes/hacs_waste_collection_schedule/doc/source/bmv_at.md)
 - [Data.Umweltprofis](https://github.com/mampfes/hacs_waste_collection_schedule/doc/source/data_umweltprofis_at.md)
+- [Korneuburg Stadtservice](https://github.com/mampfes/hacs_waste_collection_schedule/doc/source/korneuburg_stadtservice_at.md)
 - [WSZ-Moosburg.at](https://github.com/mampfes/hacs_waste_collection_schedule/doc/source/wsz_moosburg_at.md)
 
 ### Belgium