From 5602702b08c3aaadf72a11506edad4922248f566 Mon Sep 17 00:00:00 2001 From: Esa Kataja Date: Mon, 4 Aug 2025 13:19:30 +0300 Subject: [PATCH] feat: add Tyomarkkinatori job listing parser and API integration --- src/lib/parsers/__init__.py | 3 + src/lib/parsers/tyomarkkinatori.py | 88 ++++++++++++++++++++++++++++++ src/models/job_listing.py | 5 ++ src/routes/job_postings.py | 10 ++-- 4 files changed, 102 insertions(+), 4 deletions(-) create mode 100644 src/lib/parsers/tyomarkkinatori.py diff --git a/src/lib/parsers/__init__.py b/src/lib/parsers/__init__.py index ad9404d..d2446ed 100644 --- a/src/lib/parsers/__init__.py +++ b/src/lib/parsers/__init__.py @@ -1 +1,4 @@ from .duunitori import parse_duunitori as duunitori +from .tyomarkkinatori import parse_tyomarkkinatori as tyomarkkinatori + +__all__ = ["duunitori", "tyomarkkinatori"] diff --git a/src/lib/parsers/tyomarkkinatori.py b/src/lib/parsers/tyomarkkinatori.py new file mode 100644 index 0000000..6b8baee --- /dev/null +++ b/src/lib/parsers/tyomarkkinatori.py @@ -0,0 +1,88 @@ +from httpx import AsyncClient +from pydantic import BaseModel + +from lib import logger +from lib.exceptions import FetchException +from models.job_listing import JobListing + + +class _TyomarkkinatoriPaging(BaseModel): + pageSize: int = 90 + pageNumber: int = 0 + + +class _TyomarkkinatoriFilters(BaseModel): + publishedAfter: str | None = None + closesBefore: str | None = None + query: str = "" + + +class _TyomarkkinatoriParams(BaseModel): + """Model for storing parameters for Tyomarkkinatori API.""" + + query: str + paging: _TyomarkkinatoriPaging = _TyomarkkinatoriPaging() + filters: _TyomarkkinatoriFilters = _TyomarkkinatoriFilters() + + def __post_init__(self): + self.query = self.query.strip() + self.filters.query = self.query.strip() + + +async def _parse_item(item: dict) -> JobListing: + """Parse a single job listing from the search results page.""" + logger.debug(f"Parsing job listing {item['id']}", extra={"item": item}) + return JobListing( + url=f"https://tyomarkkinatori.fi/tyopaikat/{item['id']}", + company=item["employer"]["businessName"]["fi"] + if "fi" in item["employer"]["businessName"] + else item["employer"]["name"], + title=( + item["title"].get("fi") + or item["title"].get("sv") + or item["title"].get("en") + or "" + ).strip(), + description=( + ( + item["lead"].get("fi") + or item["lead"].get("sv") + or item["lead"].get("en") + or "" + ).strip() + or item["lead"].get("sv") + or item["lead"].get("en") + or "" + ).strip(), + apply_url="", + business_id=item["employer"]["businessId"], + due_date=item["applicationPeriodEndDate"], + posted_at=item["publishDate"], + ) + + +async def parse_tyomarkkinatori(search_term: str) -> list[JobListing]: + """Parse job listings from Tyomarkkinatori based on a search term.""" + base_url = "https://tyomarkkinatori.fi/api/jobpostingfulltext/search/v1/search" + all_job_listings = [] + + async with AsyncClient() as client: + params = _TyomarkkinatoriParams(query=search_term) + response = await client.post(base_url, json=params.model_dump(mode="json")) + if response.status_code != 200: + logger.error( + f"Failed to fetch job listings from {base_url}", + extra={"response": response}, + ) + raise FetchException(base_url, response.status_code) + data = response.json() + + total_pages = data["totalPages"] + logger.debug(f"Found {total_pages} pages of search results") + + # Parse 1st page + for item in data["content"]: + job_listing = await _parse_item(item) + all_job_listings.append(job_listing) + + return all_job_listings diff --git a/src/models/job_listing.py b/src/models/job_listing.py index fca8530..2e9d8a8 100644 --- a/src/models/job_listing.py +++ b/src/models/job_listing.py @@ -1,4 +1,6 @@ from pydantic import BaseModel, Field +from datetime import datetime +from zoneinfo import ZoneInfo class JobListing(BaseModel): @@ -10,3 +12,6 @@ class JobListing(BaseModel): business_id: str | None = Field(None, description="Business ID") url: str | None = Field(None, description="Job URL") apply_url: str | None = Field(None, description="Apply URL") + due_date: datetime | None = Field(None, description="Due date") + posted_at: datetime | None = Field(None, description="Posted at") + fetch_at: datetime = Field(datetime.now(ZoneInfo("UTC")), description="Fetch date") diff --git a/src/routes/job_postings.py b/src/routes/job_postings.py index 5ea3a81..7a01b7e 100644 --- a/src/routes/job_postings.py +++ b/src/routes/job_postings.py @@ -3,7 +3,7 @@ from enum import Enum from lib.exceptions import NotImplementedException from lib import logger -from lib.parsers import duunitori +from lib.parsers import duunitori, tyomarkkinatori from models.job_listing import JobListing router = APIRouter(prefix="/job_postings", tags=["Job Postings"]) @@ -21,14 +21,16 @@ async def get_job_postings(search_term: str, source: PostingSource = PostingSour postings: list[JobListing] = [] match source: case PostingSource.ALL: - logger.warning("Only duunitori is implemented at the moment.") + logger.warning( + "Only Duunitori and Tyomarkkinatori are implemented at the moment." + ) postings = await duunitori(search_term) + postings.extend(await tyomarkkinatori(search_term)) case PostingSource.LINKEDIN: logger.error(f"Job postings from {source.value} Not implemented yet") raise NotImplementedException() case PostingSource.DUUNITORI: postings = await duunitori(search_term) case PostingSource.TYOMARKKINATORI: - logger.error(f"Job postings from {source.value} Not implemented yet") - raise NotImplementedException() + postings = await tyomarkkinatori(search_term) return postings