diff --git a/src/lib/parsers/duunitori.py b/src/lib/parsers/duunitori.py index d6d1287..3208fe5 100644 --- a/src/lib/parsers/duunitori.py +++ b/src/lib/parsers/duunitori.py @@ -8,99 +8,225 @@ from lib.exceptions import FetchException from models.job_listing import JobListing -class _fetch_details(BaseModel): +class _DuunitoriDetails(BaseModel): + """Model for storing job details fetched from Duunitori.""" + description: str apply_url: str business_id: str -async def _get_description(url: str, client: AsyncClient) -> _fetch_details: - logger.debug(f"Fetching description from {url}") - response = await client.get(url) - if response.status_code != 200: - logger.error( - f"Failed to fetch description from {url}", extra={"response": response} - ) - raise FetchException(response.url, response.status_code) - html = HTMLParser(response.text) - description = html.css_first(".description-box").html - try: - apply_url = html.css_first(".gtm-apply-top-button").attributes["href"] - except AttributeError: - apply_url = "" +async def _fetch_job_details(url: str, client: AsyncClient) -> _DuunitoriDetails: + """Fetch job details from a Duunitori job posting page. + + Args: + url: URL of the job posting + client: HTTP client for making requests + + Returns: + _DuunitoriDetails: Parsed job details + + Raises: + FetchException: If the request fails + """ + logger.debug(f"Fetching job details from {url}") try: - business_id = html.css_first("span[itemprop='vatId']").text() - except AttributeError: - business_id = "" - - client.timeout = None - return _fetch_details( - description=markdownify(description), - apply_url=apply_url, - business_id=business_id, - ) - - -async def parse_duunitori(search_term: str): - retval: list[JobListing] = [] - async with AsyncClient() as client: - # Fetch job postings - response = await client.get( - f"https://duunitori.fi/tyopaikat?haku={search_term}" - ) - - # Check if request was successful + response = await client.get(url) if response.status_code != 200: + logger.error( + f"Failed to fetch job details from {url}", extra={"response": response} + ) raise FetchException(response.url, response.status_code) html = HTMLParser(response.text) - # Get pagination total pages - pagination_lastpage = int(html.css(".pagination__pagenum")[-1].text()) - logger.debug(f"Pagination last page: {pagination_lastpage}") + # Extract description + description_element = html.css_first(".description-box") + description = description_element.html if description_element else "" - # Page 1 results - for job_listing in html.css(".gtm-search-result"): - if "data-job-slug" in job_listing.attributes: - url = f"https://duunitori.fi/tyopaikat/tyo/{job_listing.attributes['data-job-slug']}" - company = job_listing.attributes["data-company"] - title = job_listing.text() - details = await _get_description(url, client) - retval.append( - JobListing( - url=url, - company=company, - title=title, - description=details.description, - apply_url=details.apply_url, - business_id=details.business_id, + # Extract apply URL + apply_url = "" + try: + apply_button = html.css_first(".gtm-apply-top-button") + if apply_button and apply_button.attributes: + apply_url = apply_button.attributes.get("href", "") + except AttributeError: + pass + + # Extract business ID + business_id = "" + try: + business_id_element = html.css_first("span[itemprop='vatId']") + if business_id_element: + business_id = business_id_element.text() + except AttributeError: + pass + + return _DuunitoriDetails( + description=markdownify(description) if description else "", + apply_url=apply_url, + business_id=business_id, + ) + except Exception as e: + logger.error( + f"Error fetching job details from {url}", + extra={"error": str(e), "response": response}, + ) + raise + + +def _parse_job_listing_element(job_element) -> dict[str, str] | None: + """Parse a single job listing element from the search results page. + + Args: + job_element: HTML element representing a job listing + + Returns: + dict with job data or None if invalid + """ + if "data-job-slug" not in job_element.attributes: + return None + + slug = job_element.attributes["data-job-slug"] + company = job_element.attributes.get("data-company", "") + title = job_element.text() + url = f"https://duunitori.fi/tyopaikat/tyo/{slug}" + + return {"url": url, "company": company, "title": title} + + +async def _fetch_search_results_page( + search_term: str, page: int, client: AsyncClient +) -> HTMLParser: + """Fetch a single page of search results. + + Args: + search_term: Term to search for + page: Page number to fetch + client: HTTP client for making requests + + Returns: + HTMLParser: Parsed HTML content + + Raises: + FetchException: If the request fails + """ + url = f"https://duunitori.fi/tyopaikat?haku={search_term}" + if page > 1: + url += f"&page={page}" + + logger.debug(f"Fetching search results page {page} from {url}") + + response = await client.get(url) + if response.status_code != 200: + logger.error( + f"Failed to fetch search results page {page} from {url}", + extra={"response": response}, + ) + raise FetchException(response.url, response.status_code) + + return HTMLParser(response.text) + + +def _get_total_pages(html: HTMLParser) -> int: + """Extract total number of pages from the search results. + + Args: + html: Parsed HTML of the first search results page + + Returns: + int: Total number of pages + """ + pagination_elements = html.css(".pagination__pagenum") + if not pagination_elements: + return 1 + + try: + return int(pagination_elements[-1].text()) + except (ValueError, IndexError): + return 1 + + +async def _process_job_listings( + html: HTMLParser, client: AsyncClient +) -> list[JobListing]: + """Process job listings from a search results page. + + Args: + html: Parsed HTML of a search results page + client: HTTP client for making requests + + Returns: + list[JobListing]: List of parsed job listings + """ + job_listings = [] + + for job_element in html.css(".gtm-search-result"): + job_data = _parse_job_listing_element(job_element) + if not job_data: + continue + + try: + # Fetch additional details from the job page + details = await _fetch_job_details(job_data["url"], client) + + # Create JobListing object + job_listing = JobListing( + url=job_data["url"], + company=job_data["company"], + title=job_data["title"], + description=details.description, + apply_url=details.apply_url, + business_id=details.business_id, + ) + + job_listings.append(job_listing) + except Exception as e: + logger.error(f"Error processing job {job_data['url']}: {str(e)}") + # Continue with other jobs even if one fails + continue + + return job_listings + + +async def parse_duunitori(search_term: str) -> list[JobListing]: + """Parse job listings from Duunitori.fi based on a search term. + + Args: + search_term: Term to search for job listings + + Returns: + list[JobListing]: List of parsed job listings + + Raises: + FetchException: If unable to fetch data from Duunitori + """ + all_job_listings = [] + + async with AsyncClient() as client: + # Fetch first page to determine pagination + first_page_html = await _fetch_search_results_page(search_term, 1, client) + total_pages = _get_total_pages(first_page_html) + + logger.debug(f"Found {total_pages} pages of search results") + + # Process first page + first_page_listings = await _process_job_listings(first_page_html, client) + all_job_listings.extend(first_page_listings) + + # Process additional pages if they exist + if total_pages > 1: + for page in range(2, total_pages + 1): + try: + page_html = await _fetch_search_results_page( + search_term, page, client ) - ) - if pagination_lastpage > 1: - for page in range(2, pagination_lastpage + 1): - response = await client.get( - f"https://duunitori.fi/tyopaikat?haku={search_term}&page={page}" - ) - if response.status_code != 200: - raise FetchException(response.url, response.status_code) - html = HTMLParser(response.text) - job_listings = html.css(".gtm-search-result") - for job_listing in job_listings: - if "data-job-slug" in job_listing.attributes: - url = f"https://duunitori.fi/tyopaikat/tyo/{job_listing.attributes['data-job-slug']}" - company = job_listing.attributes["data-company"] - title = job_listing.text() - details = await _get_description(url, client) + page_listings = await _process_job_listings(page_html, client) + all_job_listings.extend(page_listings) + except Exception as e: + logger.error(f"Error processing page {page}: {str(e)}") + # Continue with other pages even if one fails + continue - retval.append( - JobListing( - url=url, - company=company, - title=title, - description=details.description, - apply_url=details.apply_url, - business_id=details.business_id, - ) - ) - return retval + return all_job_listings