diff --git a/src/lib/parsers/tyomarkkinatori.py b/src/lib/parsers/tyomarkkinatori.py index 6b8baee..9627c23 100644 --- a/src/lib/parsers/tyomarkkinatori.py +++ b/src/lib/parsers/tyomarkkinatori.py @@ -31,7 +31,6 @@ class _TyomarkkinatoriParams(BaseModel): async def _parse_item(item: dict) -> JobListing: """Parse a single job listing from the search results page.""" - logger.debug(f"Parsing job listing {item['id']}", extra={"item": item}) return JobListing( url=f"https://tyomarkkinatori.fi/tyopaikat/{item['id']}", company=item["employer"]["businessName"]["fi"] @@ -78,11 +77,37 @@ async def parse_tyomarkkinatori(search_term: str) -> list[JobListing]: data = response.json() total_pages = data["totalPages"] - logger.debug(f"Found {total_pages} pages of search results") + total_items = data["totalElements"] + logger.debug( + f"Found {total_pages} pages of search results with {total_items} items." + ) # Parse 1st page for item in data["content"]: job_listing = await _parse_item(item) all_job_listings.append(job_listing) + # Parse additional pages if they exist + for page in range(2, total_pages + 1): + try: + params.paging.pageNumber = page + response = await client.post( + base_url, json=params.model_dump(mode="json") + ) + if response.status_code != 200: + logger.error( + f"Failed to fetch job listings from {base_url}", + extra={"response": response}, + ) + raise FetchException(base_url, response.status_code) + data = response.json() + + for item in data["content"]: + job_listing = await _parse_item(item) + all_job_listings.append(job_listing) + except Exception as e: + logger.error(f"Error processing page {page}: {str(e)}") + # Continue with other pages even if one fails + continue + return all_job_listings