feat: implement Duunitori job posting parser and API endpoint

src/lib/exceptions.py

@@ -7,3 +7,13 @@ class NotImplementedException(HTTPException):
             status_code=status.HTTP_501_NOT_IMPLEMENTED,
             detail="Not implemented yet",
         )
+
+
+class FetchException(HTTPException):
+    def __init__(
+        self, url: str = "", status_code: int = status.HTTP_500_INTERNAL_SERVER_ERROR
+    ):
+        super().__init__(
+            status_code=status_code,
+            detail=f"Failed to fetch job postings from {url} with status code {status_code}",
+        )

src/lib/parsers/__init__.py

@@ -0,0 +1 @@
+from .duunitori import parse_duunitori as duunitori

src/lib/parsers/duunitori.py

@@ -0,0 +1,106 @@
+from httpx import AsyncClient
+from selectolax.parser import HTMLParser
+from markdownify import markdownify
+from pydantic import BaseModel
+
+from lib import logger
+from lib.exceptions import FetchException
+from models.job_listing import JobListing
+
+
+class _fetch_details(BaseModel):
+    description: str
+    apply_url: str
+    business_id: str
+
+
+async def _get_description(url: str, client: AsyncClient) -> _fetch_details:
+    logger.debug(f"Fetching description from {url}")
+    response = await client.get(url)
+    if response.status_code != 200:
+        logger.error(
+            f"Failed to fetch description from {url}", extra={"response": response}
+        )
+        raise FetchException(response.url, response.status_code)
+    html = HTMLParser(response.text)
+    description = html.css_first(".description-box").html
+    try:
+        apply_url = html.css_first(".gtm-apply-top-button").attributes["href"]
+    except AttributeError:
+        apply_url = ""
+
+    try:
+        business_id = html.css_first("span[itemprop='vatId']").text()
+    except AttributeError:
+        business_id = ""
+
+    client.timeout = None
+    return _fetch_details(
+        description=markdownify(description),
+        apply_url=apply_url,
+        business_id=business_id,
+    )
+
+
+async def parse_duunitori(search_term: str):
+    retval: list[JobListing] = []
+    async with AsyncClient() as client:
+        # Fetch job postings
+        response = await client.get(
+            f"https://duunitori.fi/tyopaikat?haku={search_term}"
+        )
+
+        # Check if request was successful
+        if response.status_code != 200:
+            raise FetchException(response.url, response.status_code)
+
+        html = HTMLParser(response.text)
+        # Get pagination total pages
+        pagination_lastpage = int(html.css(".pagination__pagenum")[-1].text())
+
+        logger.debug(f"Pagination last page: {pagination_lastpage}")
+
+        # Page 1 results
+        for job_listing in html.css(".gtm-search-result"):
+            if "data-job-slug" in job_listing.attributes:
+                url = f"https://duunitori.fi/tyopaikat/tyo/{job_listing.attributes['data-job-slug']}"
+                company = job_listing.attributes["data-company"]
+                title = job_listing.text()
+                details = await _get_description(url, client)
+                retval.append(
+                    JobListing(
+                        url=url,
+                        company=company,
+                        title=title,
+                        description=details.description,
+                        apply_url=details.apply_url,
+                        business_id=details.business_id,
+                    )
+                )
+        if pagination_lastpage > 1:
+            for page in range(2, pagination_lastpage + 1):
+                response = await client.get(
+                    f"https://duunitori.fi/tyopaikat?haku={search_term}&page={page}"
+                )
+                if response.status_code != 200:
+                    raise FetchException(response.url, response.status_code)
+                html = HTMLParser(response.text)
+                job_listings = html.css(".gtm-search-result")
+                for job_listing in job_listings:
+                    if "data-job-slug" in job_listing.attributes:
+                        url = f"https://duunitori.fi/tyopaikat/tyo/{job_listing.attributes['data-job-slug']}"
+                        company = job_listing.attributes["data-company"]
+                        title = job_listing.text()
+                        details = await _get_description(url, client)
+
+                        retval.append(
+                            JobListing(
+                                url=url,
+                                company=company,
+                                title=title,
+                                description=details.description,
+                                apply_url=details.apply_url,
+                                business_id=details.business_id,
+                            )
+                        )
+    return retval

src/models/job_listing.py

@@ -2,9 +2,11 @@ from pydantic import BaseModel, Field
 
 
 class JobListing(BaseModel):
-    title: str = Field(..., description="Job title")
-    company: str = Field(..., description="Company name")
-    location: str = Field(..., description="Location")
-    description: str = Field(..., description="Job description")
+    title: str | None = Field(None, description="Job title")
+    company: str | None = Field(None, description="Company name")
+    location: str | None = Field(None, description="Location")
+    description: str | None = Field(None, description="Job description")
     salary: int | None = Field(None, description="Salary")
-    url: str = Field(..., description="Job URL")
+    business_id: str | None = Field(None, description="Business ID")
+    url: str | None = Field(None, description="Job URL")
+    apply_url: str | None = Field(None, description="Apply URL")

src/routes/job_postings.py

@@ -3,6 +3,8 @@ from enum import Enum
 
 from lib.exceptions import NotImplementedException
 from lib import logger
+from lib.parsers import duunitori
+from models.job_listing import JobListing
 
 router = APIRouter(prefix="/job_postings", tags=["Job Postings"])
 
@@ -14,18 +16,19 @@ class PostingSource(Enum):
     TYOMARKKINATORI = "tyomarkkinatori"
 
 
-@router.get("/")
-def get_job_postings(source: PostingSource = PostingSource.ALL):
+@router.get("/{search_term}", responses={200: {"model": list[JobListing]}})
+async def get_job_postings(search_term: str, source: PostingSource = PostingSource.ALL):
+    postings: list[JobListing] = []
     match source:
         case PostingSource.ALL:
-            logger.error(f"Job postings from {source.value} Not implemented yet")
-            raise NotImplementedException()
+            logger.warning("Only duunitori is implemented at the moment.")
+            postings = await duunitori(search_term)
         case PostingSource.LINKEDIN:
             logger.error(f"Job postings from {source.value} Not implemented yet")
             raise NotImplementedException()
         case PostingSource.DUUNITORI:
-            logger.error(f"Job postings from {source.value} Not implemented yet")
-            raise NotImplementedException()
+            postings = await duunitori(search_term)
         case PostingSource.TYOMARKKINATORI:
             logger.error(f"Job postings from {source.value} Not implemented yet")
             raise NotImplementedException()
+    return postings