from httpx import AsyncClient from pydantic import BaseModel from lib import logger from lib.exceptions import FetchException from models.job_listing import JobListing class _TyomarkkinatoriPaging(BaseModel): pageSize: int = 90 pageNumber: int = 0 class _TyomarkkinatoriFilters(BaseModel): publishedAfter: str | None = None closesBefore: str | None = None query: str = "" class _TyomarkkinatoriParams(BaseModel): """Model for storing parameters for Tyomarkkinatori API.""" query: str paging: _TyomarkkinatoriPaging = _TyomarkkinatoriPaging() filters: _TyomarkkinatoriFilters = _TyomarkkinatoriFilters() def __post_init__(self): self.query = self.query.strip() self.filters.query = self.query.strip() async def _parse_item(item: dict) -> JobListing: """Parse a single job listing from the search results page.""" return JobListing( url=f"https://tyomarkkinatori.fi/tyopaikat/{item['id']}", company=item["employer"]["businessName"]["fi"] if "fi" in item["employer"]["businessName"] else item["employer"]["name"], title=( item["title"].get("fi") or item["title"].get("sv") or item["title"].get("en") or "" ).strip(), description=( ( item["lead"].get("fi") or item["lead"].get("sv") or item["lead"].get("en") or "" ).strip() or item["lead"].get("sv") or item["lead"].get("en") or "" ).strip(), apply_url="", business_id=item["employer"]["businessId"], due_date=item["applicationPeriodEndDate"], posted_at=item["publishDate"], ) async def parse_tyomarkkinatori(search_term: str) -> list[JobListing]: """Parse job listings from Tyomarkkinatori based on a search term.""" base_url = "https://tyomarkkinatori.fi/api/jobpostingfulltext/search/v1/search" all_job_listings = [] async with AsyncClient() as client: params = _TyomarkkinatoriParams(query=search_term) response = await client.post(base_url, json=params.model_dump(mode="json")) if response.status_code != 200: logger.error( f"Failed to fetch job listings from {base_url}", extra={"response": response}, ) raise FetchException(base_url, response.status_code) data = response.json() total_pages = data["totalPages"] total_items = data["totalElements"] logger.debug( f"Found {total_pages} pages of search results with {total_items} items." ) # Parse 1st page for item in data["content"]: job_listing = await _parse_item(item) all_job_listings.append(job_listing) # Parse additional pages if they exist for page in range(2, total_pages + 1): try: params.paging.pageNumber = page response = await client.post( base_url, json=params.model_dump(mode="json") ) if response.status_code != 200: logger.error( f"Failed to fetch job listings from {base_url}", extra={"response": response}, ) raise FetchException(base_url, response.status_code) data = response.json() for item in data["content"]: job_listing = await _parse_item(item) all_job_listings.append(job_listing) except Exception as e: logger.error(f"Error processing page {page}: {str(e)}") # Continue with other pages even if one fails continue return all_job_listings