refactor: improve duunitori parser with better error handling and code organization

src/lib/parsers/duunitori.py

@@ -8,99 +8,225 @@ from lib.exceptions import FetchException
 from models.job_listing import JobListing
 
 
-class _fetch_details(BaseModel):
+class _DuunitoriDetails(BaseModel):
+    """Model for storing job details fetched from Duunitori."""
+
     description: str
     apply_url: str
     business_id: str
 
 
-async def _get_description(url: str, client: AsyncClient) -> _fetch_details:
+async def _fetch_job_details(url: str, client: AsyncClient) -> _DuunitoriDetails:
-    logger.debug(f"Fetching description from {url}")
+    """Fetch job details from a Duunitori job posting page.
-    response = await client.get(url)
+
-    if response.status_code != 200:
+    Args:
-        logger.error(
+        url: URL of the job posting
-            f"Failed to fetch description from {url}", extra={"response": response}
+        client: HTTP client for making requests
-        )
+
-        raise FetchException(response.url, response.status_code)
+    Returns:
-    html = HTMLParser(response.text)
+        _DuunitoriDetails: Parsed job details
-    description = html.css_first(".description-box").html
+
-    try:
+    Raises:
-        apply_url = html.css_first(".gtm-apply-top-button").attributes["href"]
+        FetchException: If the request fails
-    except AttributeError:
+    """
-        apply_url = ""
+    logger.debug(f"Fetching job details from {url}")
 
     try:
-        business_id = html.css_first("span[itemprop='vatId']").text()
+        response = await client.get(url)
-    except AttributeError:
-        business_id = ""
-
-    client.timeout = None
-    return _fetch_details(
-        description=markdownify(description),
-        apply_url=apply_url,
-        business_id=business_id,
-    )
-
-
-async def parse_duunitori(search_term: str):
-    retval: list[JobListing] = []
-    async with AsyncClient() as client:
-        # Fetch job postings
-        response = await client.get(
-            f"https://duunitori.fi/tyopaikat?haku={search_term}"
-        )
-
-        # Check if request was successful
         if response.status_code != 200:
+            logger.error(
+                f"Failed to fetch job details from {url}", extra={"response": response}
+            )
             raise FetchException(response.url, response.status_code)
 
         html = HTMLParser(response.text)
-        # Get pagination total pages
-        pagination_lastpage = int(html.css(".pagination__pagenum")[-1].text())
 
-        logger.debug(f"Pagination last page: {pagination_lastpage}")
+        # Extract description
+        description_element = html.css_first(".description-box")
+        description = description_element.html if description_element else ""
 
-        # Page 1 results
+        # Extract apply URL
-        for job_listing in html.css(".gtm-search-result"):
+        apply_url = ""
-            if "data-job-slug" in job_listing.attributes:
+        try:
-                url = f"https://duunitori.fi/tyopaikat/tyo/{job_listing.attributes['data-job-slug']}"
+            apply_button = html.css_first(".gtm-apply-top-button")
-                company = job_listing.attributes["data-company"]
+            if apply_button and apply_button.attributes:
-                title = job_listing.text()
+                apply_url = apply_button.attributes.get("href", "")
-                details = await _get_description(url, client)
+        except AttributeError:
-                retval.append(
+            pass
-                    JobListing(
+
-                        url=url,
+        # Extract business ID
-                        company=company,
+        business_id = ""
-                        title=title,
+        try:
-                        description=details.description,
+            business_id_element = html.css_first("span[itemprop='vatId']")
-                        apply_url=details.apply_url,
+            if business_id_element:
-                        business_id=details.business_id,
+                business_id = business_id_element.text()
+        except AttributeError:
+            pass
+
+        return _DuunitoriDetails(
+            description=markdownify(description) if description else "",
+            apply_url=apply_url,
+            business_id=business_id,
+        )
+    except Exception as e:
+        logger.error(
+            f"Error fetching job details from {url}",
+            extra={"error": str(e), "response": response},
+        )
+        raise
+
+
+def _parse_job_listing_element(job_element) -> dict[str, str] | None:
+    """Parse a single job listing element from the search results page.
+
+    Args:
+        job_element: HTML element representing a job listing
+
+    Returns:
+        dict with job data or None if invalid
+    """
+    if "data-job-slug" not in job_element.attributes:
+        return None
+
+    slug = job_element.attributes["data-job-slug"]
+    company = job_element.attributes.get("data-company", "")
+    title = job_element.text()
+    url = f"https://duunitori.fi/tyopaikat/tyo/{slug}"
+
+    return {"url": url, "company": company, "title": title}
+
+
+async def _fetch_search_results_page(
+    search_term: str, page: int, client: AsyncClient
+) -> HTMLParser:
+    """Fetch a single page of search results.
+
+    Args:
+        search_term: Term to search for
+        page: Page number to fetch
+        client: HTTP client for making requests
+
+    Returns:
+        HTMLParser: Parsed HTML content
+
+    Raises:
+        FetchException: If the request fails
+    """
+    url = f"https://duunitori.fi/tyopaikat?haku={search_term}"
+    if page > 1:
+        url += f"&page={page}"
+
+    logger.debug(f"Fetching search results page {page} from {url}")
+
+    response = await client.get(url)
+    if response.status_code != 200:
+        logger.error(
+            f"Failed to fetch search results page {page} from {url}",
+            extra={"response": response},
+        )
+        raise FetchException(response.url, response.status_code)
+
+    return HTMLParser(response.text)
+
+
+def _get_total_pages(html: HTMLParser) -> int:
+    """Extract total number of pages from the search results.
+
+    Args:
+        html: Parsed HTML of the first search results page
+
+    Returns:
+        int: Total number of pages
+    """
+    pagination_elements = html.css(".pagination__pagenum")
+    if not pagination_elements:
+        return 1
+
+    try:
+        return int(pagination_elements[-1].text())
+    except (ValueError, IndexError):
+        return 1
+
+
+async def _process_job_listings(
+    html: HTMLParser, client: AsyncClient
+) -> list[JobListing]:
+    """Process job listings from a search results page.
+
+    Args:
+        html: Parsed HTML of a search results page
+        client: HTTP client for making requests
+
+    Returns:
+        list[JobListing]: List of parsed job listings
+    """
+    job_listings = []
+
+    for job_element in html.css(".gtm-search-result"):
+        job_data = _parse_job_listing_element(job_element)
+        if not job_data:
+            continue
+
+        try:
+            # Fetch additional details from the job page
+            details = await _fetch_job_details(job_data["url"], client)
+
+            # Create JobListing object
+            job_listing = JobListing(
+                url=job_data["url"],
+                company=job_data["company"],
+                title=job_data["title"],
+                description=details.description,
+                apply_url=details.apply_url,
+                business_id=details.business_id,
+            )
+
+            job_listings.append(job_listing)
+        except Exception as e:
+            logger.error(f"Error processing job {job_data['url']}: {str(e)}")
+            # Continue with other jobs even if one fails
+            continue
+
+    return job_listings
+
+
+async def parse_duunitori(search_term: str) -> list[JobListing]:
+    """Parse job listings from Duunitori.fi based on a search term.
+
+    Args:
+        search_term: Term to search for job listings
+
+    Returns:
+        list[JobListing]: List of parsed job listings
+
+    Raises:
+        FetchException: If unable to fetch data from Duunitori
+    """
+    all_job_listings = []
+
+    async with AsyncClient() as client:
+        # Fetch first page to determine pagination
+        first_page_html = await _fetch_search_results_page(search_term, 1, client)
+        total_pages = _get_total_pages(first_page_html)
+
+        logger.debug(f"Found {total_pages} pages of search results")
+
+        # Process first page
+        first_page_listings = await _process_job_listings(first_page_html, client)
+        all_job_listings.extend(first_page_listings)
+
+        # Process additional pages if they exist
+        if total_pages > 1:
+            for page in range(2, total_pages + 1):
+                try:
+                    page_html = await _fetch_search_results_page(
+                        search_term, page, client
                     )
-                )
+                    page_listings = await _process_job_listings(page_html, client)
-        if pagination_lastpage > 1:
+                    all_job_listings.extend(page_listings)
-            for page in range(2, pagination_lastpage + 1):
+                except Exception as e:
-                response = await client.get(
+                    logger.error(f"Error processing page {page}: {str(e)}")
-                    f"https://duunitori.fi/tyopaikat?haku={search_term}&page={page}"
+                    # Continue with other pages even if one fails
-                )
+                    continue
-                if response.status_code != 200:
-                    raise FetchException(response.url, response.status_code)
-                html = HTMLParser(response.text)
-                job_listings = html.css(".gtm-search-result")
-                for job_listing in job_listings:
-                    if "data-job-slug" in job_listing.attributes:
-                        url = f"https://duunitori.fi/tyopaikat/tyo/{job_listing.attributes['data-job-slug']}"
-                        company = job_listing.attributes["data-company"]
-                        title = job_listing.text()
-                        details = await _get_description(url, client)
 
-                        retval.append(
+    return all_job_listings
-                            JobListing(
-                                url=url,
-                                company=company,
-                                title=title,
-                                description=details.description,
-                                apply_url=details.apply_url,
-                                business_id=details.business_id,
-                            )
-                        )
-    return retval