diff --git a/src/lib/exceptions.py b/src/lib/exceptions.py index b706af2..1ee37a5 100644 --- a/src/lib/exceptions.py +++ b/src/lib/exceptions.py @@ -7,3 +7,13 @@ class NotImplementedException(HTTPException): status_code=status.HTTP_501_NOT_IMPLEMENTED, detail="Not implemented yet", ) + + +class FetchException(HTTPException): + def __init__( + self, url: str = "", status_code: int = status.HTTP_500_INTERNAL_SERVER_ERROR + ): + super().__init__( + status_code=status_code, + detail=f"Failed to fetch job postings from {url} with status code {status_code}", + ) diff --git a/src/lib/parsers/__init__.py b/src/lib/parsers/__init__.py new file mode 100644 index 0000000..ad9404d --- /dev/null +++ b/src/lib/parsers/__init__.py @@ -0,0 +1 @@ +from .duunitori import parse_duunitori as duunitori diff --git a/src/lib/parsers/duunitori.py b/src/lib/parsers/duunitori.py new file mode 100644 index 0000000..d6d1287 --- /dev/null +++ b/src/lib/parsers/duunitori.py @@ -0,0 +1,106 @@ +from httpx import AsyncClient +from selectolax.parser import HTMLParser +from markdownify import markdownify +from pydantic import BaseModel + +from lib import logger +from lib.exceptions import FetchException +from models.job_listing import JobListing + + +class _fetch_details(BaseModel): + description: str + apply_url: str + business_id: str + + +async def _get_description(url: str, client: AsyncClient) -> _fetch_details: + logger.debug(f"Fetching description from {url}") + response = await client.get(url) + if response.status_code != 200: + logger.error( + f"Failed to fetch description from {url}", extra={"response": response} + ) + raise FetchException(response.url, response.status_code) + html = HTMLParser(response.text) + description = html.css_first(".description-box").html + try: + apply_url = html.css_first(".gtm-apply-top-button").attributes["href"] + except AttributeError: + apply_url = "" + + try: + business_id = html.css_first("span[itemprop='vatId']").text() + except AttributeError: + business_id = "" + + client.timeout = None + return _fetch_details( + description=markdownify(description), + apply_url=apply_url, + business_id=business_id, + ) + + +async def parse_duunitori(search_term: str): + retval: list[JobListing] = [] + async with AsyncClient() as client: + # Fetch job postings + response = await client.get( + f"https://duunitori.fi/tyopaikat?haku={search_term}" + ) + + # Check if request was successful + if response.status_code != 200: + raise FetchException(response.url, response.status_code) + + html = HTMLParser(response.text) + # Get pagination total pages + pagination_lastpage = int(html.css(".pagination__pagenum")[-1].text()) + + logger.debug(f"Pagination last page: {pagination_lastpage}") + + # Page 1 results + for job_listing in html.css(".gtm-search-result"): + if "data-job-slug" in job_listing.attributes: + url = f"https://duunitori.fi/tyopaikat/tyo/{job_listing.attributes['data-job-slug']}" + company = job_listing.attributes["data-company"] + title = job_listing.text() + details = await _get_description(url, client) + retval.append( + JobListing( + url=url, + company=company, + title=title, + description=details.description, + apply_url=details.apply_url, + business_id=details.business_id, + ) + ) + if pagination_lastpage > 1: + for page in range(2, pagination_lastpage + 1): + response = await client.get( + f"https://duunitori.fi/tyopaikat?haku={search_term}&page={page}" + ) + if response.status_code != 200: + raise FetchException(response.url, response.status_code) + html = HTMLParser(response.text) + job_listings = html.css(".gtm-search-result") + for job_listing in job_listings: + if "data-job-slug" in job_listing.attributes: + url = f"https://duunitori.fi/tyopaikat/tyo/{job_listing.attributes['data-job-slug']}" + company = job_listing.attributes["data-company"] + title = job_listing.text() + details = await _get_description(url, client) + + retval.append( + JobListing( + url=url, + company=company, + title=title, + description=details.description, + apply_url=details.apply_url, + business_id=details.business_id, + ) + ) + return retval diff --git a/src/models/job_listing.py b/src/models/job_listing.py index 551f4c3..fca8530 100644 --- a/src/models/job_listing.py +++ b/src/models/job_listing.py @@ -2,9 +2,11 @@ from pydantic import BaseModel, Field class JobListing(BaseModel): - title: str = Field(..., description="Job title") - company: str = Field(..., description="Company name") - location: str = Field(..., description="Location") - description: str = Field(..., description="Job description") + title: str | None = Field(None, description="Job title") + company: str | None = Field(None, description="Company name") + location: str | None = Field(None, description="Location") + description: str | None = Field(None, description="Job description") salary: int | None = Field(None, description="Salary") - url: str = Field(..., description="Job URL") + business_id: str | None = Field(None, description="Business ID") + url: str | None = Field(None, description="Job URL") + apply_url: str | None = Field(None, description="Apply URL") diff --git a/src/routes/job_postings.py b/src/routes/job_postings.py index 8c2aa49..5ea3a81 100644 --- a/src/routes/job_postings.py +++ b/src/routes/job_postings.py @@ -3,6 +3,8 @@ from enum import Enum from lib.exceptions import NotImplementedException from lib import logger +from lib.parsers import duunitori +from models.job_listing import JobListing router = APIRouter(prefix="/job_postings", tags=["Job Postings"]) @@ -14,18 +16,19 @@ class PostingSource(Enum): TYOMARKKINATORI = "tyomarkkinatori" -@router.get("/") -def get_job_postings(source: PostingSource = PostingSource.ALL): +@router.get("/{search_term}", responses={200: {"model": list[JobListing]}}) +async def get_job_postings(search_term: str, source: PostingSource = PostingSource.ALL): + postings: list[JobListing] = [] match source: case PostingSource.ALL: - logger.error(f"Job postings from {source.value} Not implemented yet") - raise NotImplementedException() + logger.warning("Only duunitori is implemented at the moment.") + postings = await duunitori(search_term) case PostingSource.LINKEDIN: logger.error(f"Job postings from {source.value} Not implemented yet") raise NotImplementedException() case PostingSource.DUUNITORI: - logger.error(f"Job postings from {source.value} Not implemented yet") - raise NotImplementedException() + postings = await duunitori(search_term) case PostingSource.TYOMARKKINATORI: logger.error(f"Job postings from {source.value} Not implemented yet") raise NotImplementedException() + return postings