feat: implement pagination support for Tyomarkkinatori job listings parser

src/lib/parsers/tyomarkkinatori.py

@@ -31,7 +31,6 @@ class _TyomarkkinatoriParams(BaseModel):
 
 async def _parse_item(item: dict) -> JobListing:
     """Parse a single job listing from the search results page."""
-    logger.debug(f"Parsing job listing {item['id']}", extra={"item": item})
     return JobListing(
         url=f"https://tyomarkkinatori.fi/tyopaikat/{item['id']}",
         company=item["employer"]["businessName"]["fi"]
@@ -78,11 +77,37 @@ async def parse_tyomarkkinatori(search_term: str) -> list[JobListing]:
         data = response.json()
 
         total_pages = data["totalPages"]
-        logger.debug(f"Found {total_pages} pages of search results")
+        total_items = data["totalElements"]
+        logger.debug(
+            f"Found {total_pages} pages of search results with {total_items} items."
+        )
 
         # Parse 1st page
         for item in data["content"]:
             job_listing = await _parse_item(item)
             all_job_listings.append(job_listing)
 
+        # Parse additional pages if they exist
+        for page in range(2, total_pages + 1):
+            try:
+                params.paging.pageNumber = page
+                response = await client.post(
+                    base_url, json=params.model_dump(mode="json")
+                )
+                if response.status_code != 200:
+                    logger.error(
+                        f"Failed to fetch job listings from {base_url}",
+                        extra={"response": response},
+                    )
+                    raise FetchException(base_url, response.status_code)
+                data = response.json()
+
+                for item in data["content"]:
+                    job_listing = await _parse_item(item)
+                    all_job_listings.append(job_listing)
+            except Exception as e:
+                logger.error(f"Error processing page {page}: {str(e)}")
+                # Continue with other pages even if one fails
+                continue
+
     return all_job_listings