Archived
114 lines
3.8 KiB
Python
114 lines
3.8 KiB
Python
from httpx import AsyncClient
|
|
from pydantic import BaseModel
|
|
|
|
from lib import logger
|
|
from lib.exceptions import FetchException
|
|
from models.job_listing import JobListing
|
|
|
|
|
|
class _TyomarkkinatoriPaging(BaseModel):
|
|
pageSize: int = 90
|
|
pageNumber: int = 0
|
|
|
|
|
|
class _TyomarkkinatoriFilters(BaseModel):
|
|
publishedAfter: str | None = None
|
|
closesBefore: str | None = None
|
|
query: str = ""
|
|
|
|
|
|
class _TyomarkkinatoriParams(BaseModel):
|
|
"""Model for storing parameters for Tyomarkkinatori API."""
|
|
|
|
query: str
|
|
paging: _TyomarkkinatoriPaging = _TyomarkkinatoriPaging()
|
|
filters: _TyomarkkinatoriFilters = _TyomarkkinatoriFilters()
|
|
|
|
def __post_init__(self):
|
|
self.query = self.query.strip()
|
|
self.filters.query = self.query.strip()
|
|
|
|
|
|
async def _parse_item(item: dict) -> JobListing:
|
|
"""Parse a single job listing from the search results page."""
|
|
return JobListing(
|
|
url=f"https://tyomarkkinatori.fi/tyopaikat/{item['id']}",
|
|
company=item["employer"]["businessName"]["fi"]
|
|
if "fi" in item["employer"]["businessName"]
|
|
else item["employer"]["name"],
|
|
title=(
|
|
item["title"].get("fi")
|
|
or item["title"].get("sv")
|
|
or item["title"].get("en")
|
|
or ""
|
|
).strip(),
|
|
description=(
|
|
(
|
|
item["lead"].get("fi")
|
|
or item["lead"].get("sv")
|
|
or item["lead"].get("en")
|
|
or ""
|
|
).strip()
|
|
or item["lead"].get("sv")
|
|
or item["lead"].get("en")
|
|
or ""
|
|
).strip(),
|
|
apply_url="",
|
|
business_id=item["employer"]["businessId"],
|
|
due_date=item["applicationPeriodEndDate"],
|
|
posted_at=item["publishDate"],
|
|
)
|
|
|
|
|
|
async def parse_tyomarkkinatori(search_term: str) -> list[JobListing]:
|
|
"""Parse job listings from Tyomarkkinatori based on a search term."""
|
|
base_url = "https://tyomarkkinatori.fi/api/jobpostingfulltext/search/v1/search"
|
|
all_job_listings = []
|
|
|
|
async with AsyncClient() as client:
|
|
params = _TyomarkkinatoriParams(query=search_term)
|
|
response = await client.post(base_url, json=params.model_dump(mode="json"))
|
|
if response.status_code != 200:
|
|
logger.error(
|
|
f"Failed to fetch job listings from {base_url}",
|
|
extra={"response": response},
|
|
)
|
|
raise FetchException(base_url, response.status_code)
|
|
data = response.json()
|
|
|
|
total_pages = data["totalPages"]
|
|
total_items = data["totalElements"]
|
|
logger.debug(
|
|
f"Found {total_pages} pages of search results with {total_items} items."
|
|
)
|
|
|
|
# Parse 1st page
|
|
for item in data["content"]:
|
|
job_listing = await _parse_item(item)
|
|
all_job_listings.append(job_listing)
|
|
|
|
# Parse additional pages if they exist
|
|
for page in range(2, total_pages + 1):
|
|
try:
|
|
params.paging.pageNumber = page
|
|
response = await client.post(
|
|
base_url, json=params.model_dump(mode="json")
|
|
)
|
|
if response.status_code != 200:
|
|
logger.error(
|
|
f"Failed to fetch job listings from {base_url}",
|
|
extra={"response": response},
|
|
)
|
|
raise FetchException(base_url, response.status_code)
|
|
data = response.json()
|
|
|
|
for item in data["content"]:
|
|
job_listing = await _parse_item(item)
|
|
all_job_listings.append(job_listing)
|
|
except Exception as e:
|
|
logger.error(f"Error processing page {page}: {str(e)}")
|
|
# Continue with other pages even if one fails
|
|
continue
|
|
|
|
return all_job_listings
|