chore: add example environment config with port and reload settings

.dockerignore

@@ -0,0 +1,19 @@
+# Python-generated files
+__pycache__/
+*.py[oc]
+build/
+dist/
+wheels/
+*.egg-info
+
+# Virtual environments
+.venv
+.env
+
+# Log files
+*.log
+*.gz
+
+
+# Data logs directory
+src/data/logs/*

.env.example

@@ -0,0 +1,2 @@
+PORT=8000
+RELOAD=true

Dockerfile

@@ -0,0 +1,14 @@
+FROM python:3.12-slim
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
+
+WORKDIR /app
+
+COPY pyproject.toml uv.lock /app/
+
+RUN uv sync --frozen
+
+COPY src .
+
+EXPOSE 8000
+
+CMD ["uv", "run", "main.py"]

docker-compose.yml

@@ -0,0 +1,6 @@
+services:
+  jobpostingparser:
+    image: localhost/jobpostingparser:latest
+    ports:
+      - "9000:8000"
+    

makefile

@@ -0,0 +1,10 @@
+.PHONY: build run save
+
+build:
+	podman build . -t localhost/n8n_helper:latest
+
+run:
+	podman-compose up -d --build
+
+save:
+	podman save -o n8n_helper.tar localhost/n8n_helper:latest

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
-name = "jobpostingparser"
-version = "0.1.0"
+name = "n8n_helper"
+version = "1.0.0"
 description = "Add your description here"
 readme = "README.md"
 requires-python = ">=3.12"

src/lib/exceptions.py

@@ -17,3 +17,11 @@ class FetchException(HTTPException):
             status_code=status_code,
             detail=f"Failed to fetch job postings from {url} with status code {status_code}",
         )
+
+
+class ParseException(HTTPException):
+    def __init__(self):
+        super().__init__(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail="Parsing failed.",
+        )

src/lib/parsers/tyomarkkinatori.py

@@ -31,7 +31,6 @@ class _TyomarkkinatoriParams(BaseModel):
 
 async def _parse_item(item: dict) -> JobListing:
     """Parse a single job listing from the search results page."""
-    logger.debug(f"Parsing job listing {item['id']}", extra={"item": item})
     return JobListing(
         url=f"https://tyomarkkinatori.fi/tyopaikat/{item['id']}",
         company=item["employer"]["businessName"]["fi"]
@@ -78,11 +77,37 @@ async def parse_tyomarkkinatori(search_term: str) -> list[JobListing]:
         data = response.json()
 
         total_pages = data["totalPages"]
-        logger.debug(f"Found {total_pages} pages of search results")
+        total_items = data["totalElements"]
+        logger.debug(
+            f"Found {total_pages} pages of search results with {total_items} items."
+        )
 
         # Parse 1st page
         for item in data["content"]:
             job_listing = await _parse_item(item)
             all_job_listings.append(job_listing)
 
+        # Parse additional pages if they exist
+        for page in range(2, total_pages + 1):
+            try:
+                params.paging.pageNumber = page
+                response = await client.post(
+                    base_url, json=params.model_dump(mode="json")
+                )
+                if response.status_code != 200:
+                    logger.error(
+                        f"Failed to fetch job listings from {base_url}",
+                        extra={"response": response},
+                    )
+                    raise FetchException(base_url, response.status_code)
+                data = response.json()
+
+                for item in data["content"]:
+                    job_listing = await _parse_item(item)
+                    all_job_listings.append(job_listing)
+            except Exception as e:
+                logger.error(f"Error processing page {page}: {str(e)}")
+                # Continue with other pages even if one fails
+                continue
+
     return all_job_listings

src/main.py

@@ -1,9 +1,13 @@
 from fastapi import FastAPI
 from contextlib import asynccontextmanager
+from dotenv import load_dotenv
+from os import getenv
 
 from routes import routes
 from lib import logger
 
+load_dotenv()
+
 
 @asynccontextmanager
 async def lifespan(app: FastAPI):
@@ -14,7 +18,9 @@ async def lifespan(app: FastAPI):
     logger.info("Application shutdown")
 
 
-app = FastAPI(lifespan=lifespan)
+app = FastAPI(
+    lifespan=lifespan, title="N8N Helper", description="N8N Helper", version="1.0.0"
+)
 
 
 for route in routes:
@@ -24,4 +30,9 @@ for route in routes:
 if __name__ == "__main__":
     import uvicorn
 
-    uvicorn.run("main:app", host="0.0.0.0", port=8000, reload=True)
+    uvicorn.run(
+        "main:app",
+        host="0.0.0.0",
+        port=int(getenv("PORT", 80)),
+        reload=getenv("RELOAD", "false") == "true",
+    )

src/models/energy.py

@@ -0,0 +1,15 @@
+from pydantic import BaseModel, Field
+from datetime import datetime
+
+
+class Spot(BaseModel):
+    price: float = Field(..., description="Price in cents per kWh")
+    start_date: datetime = Field(..., description="Start date")
+    end_date: datetime = Field(..., description="End date")
+
+
+class Consumption(BaseModel):
+    timestamp: datetime = Field(..., description="Timestamp")
+    consumption: float = Field(..., description="Consumption in kWh")
+    total_cost: float = Field(..., description="Total cost in euros")
+    vat_percentage: float = Field(..., description="VAT percentage")

src/models/weather.py

@@ -0,0 +1,25 @@
+from pydantic import BaseModel, Field, field_validator
+from datetime import datetime
+from zoneinfo import ZoneInfo
+
+
+class Weather(BaseModel):
+    name: str = Field(..., description="Name of the weather station")
+    localtime: datetime | str = Field(..., description="Local time")
+    t2m: float = Field(..., description="Temperature")
+    DewPoint: float | None = Field(None, description="Dew point")
+    Precipitation1h: float | None = Field(
+        None, description="Precipitation in the last hour"
+    )
+    TotalCloudCover: int | None = Field(None, description="Total cloud cover")
+    WindSpeedMS: float | None = Field(None, description="Wind speed in m/s")
+    WindDirection: int | None = Field(None, description="Wind direction in degrees")
+    WindGust: float | None = Field(None, description="Wind gust in m/s")
+    Pressure: float | None = Field(None, description="Pressure in hPa")
+    Humidity: int | None = Field(None, description="Humidity in %")
+    Visibility: int | None = Field(None, description="Visibility in m")
+    SnowDepth: int | None = Field(None, description="Snow depth in cm")
+
+    @field_validator("localtime", mode="before")
+    def parse_localtime(cls, v):
+        return datetime.fromisoformat(v).replace(tzinfo=ZoneInfo("Europe/Helsinki"))

src/routes/__init__.py

@@ -1,6 +1,15 @@
 from .default import router as default_router
 from .job_postings import router as job_postings_router
+from .energy import router as energy_router
+from .weather import router as weather_router
+from .consumption import router as consumption_router
 
-routes = [default_router, job_postings_router]
+routes = [
+    default_router,
+    job_postings_router,
+    energy_router,
+    weather_router,
+    consumption_router,
+]
 
 __all__ = ["routes"]

src/routes/consumption.py

@@ -0,0 +1,115 @@
+from fastapi import APIRouter
+from httpx import AsyncClient
+from pydantic import BaseModel, Field
+from datetime import datetime, timedelta
+from zoneinfo import ZoneInfo
+import json
+
+from lib.exceptions import FetchException, ParseException
+from models.energy import Consumption
+
+# from models.consumption import Consumption
+from lib import logger
+
+
+router = APIRouter(prefix="/consumption", tags=["Consumption"])
+
+
+class _CookieData(BaseModel):
+    Host_next_auth_csrf_token: str = Field(..., description="CSRF token")
+    Secure_next_auth_session_token_0: str = Field(..., description="Session token 0")
+    Secure_next_auth_session_token_1: str = Field(..., description="Session token 1")
+
+
+@router.post("/")
+async def get_consumption(
+    cookie_data: _CookieData,
+    to_date: datetime = datetime.now() - timedelta(days=1),
+    from_date: datetime = datetime.now() - timedelta(days=14),
+):
+    """Get consumption data for a given date range. Max 14 days.
+
+    Args:
+        cookie_data: Cookie data containing CSRF token and session tokens
+        start_date: Start date of the date range
+        end_date: End date of the date range
+    """
+
+    base_url = (
+        "https://www.fortum.com/fi/sahkoa/api/trpc/loggedIn.timeSeries.listTimeSeries"
+    )
+    print(from_date.astimezone(ZoneInfo("UTC")), to_date.astimezone(ZoneInfo("UTC")))
+    input_param = {
+        "0": {
+            "json": {
+                "meteringPointNo": ["6907998"],
+                "fromDate": from_date.astimezone(ZoneInfo("UTC"))
+                .isoformat()
+                .replace("+00:00", "Z"),
+                "toDate": to_date.astimezone(ZoneInfo("UTC"))
+                .isoformat()
+                .replace("+00:00", "Z"),
+                "resolution": "HOUR",
+                "type": "CONSUMPTION",
+            }
+        }
+    }
+    params = {"batch": 1, "input": json.dumps(input_param)}
+    cookies = {
+        "NEXT_LOCALE": "fi",
+        "__Host-next-auth.csrf-token": cookie_data.Host_next_auth_csrf_token,
+        "__Secure-next-auth.callback-url": "https://www.fortum.com/fi/sahkoa/kirjautunut/sahko",
+        "__Secure-next-auth.session-token.0": cookie_data.Secure_next_auth_session_token_0,
+        "__Secure-next-auth.session-token.1": cookie_data.Secure_next_auth_session_token_1,
+    }
+    headers = {
+        "accept": "application/json",
+        "accept-language": "en-US,en;q=0.7",
+        "cache-control": "no-cache",
+        "content-type": "application/json",
+        "pragma": "no-cache",
+        "priority": "u=1, i",
+        "referer": "https://www.fortum.com/fi/sahkoa/kirjautunut/sahko",
+        "sec-ch-ua": '"Not)A;Brand";v="8", "Chromium";v="138", "Brave";v="138"',
+        "sec-ch-ua-mobile": "?0",
+        "sec-ch-ua-platform": '"Linux"',
+        "sec-fetch-dest": "empty",
+        "sec-fetch-mode": "cors",
+        "sec-fetch-site": "same-origin",
+        "sec-gpc": "1",
+        "user-agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36",
+    }
+    async with AsyncClient() as client:
+        response = await client.get(
+            base_url, params=params, cookies=cookies, headers=headers, timeout=30
+        )
+    if response.status_code != 200:
+        logger.error(
+            f"Failed to fetch data from {response.url}. Status code: {response.status_code}. Response text: {response.text}"
+        )
+        raise FetchException(response.url, response.status_code)
+    data = response.json()
+    if not data:
+        logger.error(
+            f"Failed to parse data from {response.url}. Status code: {response.status_code}. Response text: {response.text}"
+        )
+        raise FetchException(response.url, response.status_code)
+
+    series_data = data[0]["result"]["data"]["json"][0]["series"]
+    retval: list[Consumption] = []
+    for series in series_data:
+        if series["cost"] is not None:
+            try:
+                consumption = Consumption(
+                    timestamp=series["atUTC"],
+                    consumption=series["energy"][0]["value"],
+                    total_cost=series["price"]["total"],
+                    vat_percentage=series["price"]["vatPercentage"],
+                )
+                retval.append(consumption)
+            except Exception as e:
+                logger.error(
+                    f"Error parsing consumption: {str(e)}", extra={"data": series}
+                )
+                raise ParseException()
+    return retval

src/routes/energy.py

@@ -0,0 +1,58 @@
+from fastapi import APIRouter
+from httpx import AsyncClient
+from datetime import date, datetime, timedelta
+
+from lib.exceptions import FetchException, ParseException
+from models.energy import Spot
+from lib import logger
+
+router = APIRouter(prefix="/energy", tags=["Energy"])
+
+
+@router.get("/spot/", responses={200: {"model": Spot}})
+async def spot(date: date = date.today(), hour: int = 0) -> Spot:
+    """Get spot price for a given date and hour."""
+    start_date = datetime(date.year, date.month, date.day, hour)
+
+    url = f"https://api.porssisahko.net/v1/price.json?date={date}&hour={hour}"
+    async with AsyncClient() as client:
+        response = await client.get(url)
+    if response.status_code != 200:
+        raise FetchException(response.url, response.status_code)
+    data = response.json()
+    if not data:
+        raise FetchException(response.url, response.status_code)
+    try:
+        spot = Spot(
+            price=data["price"],
+            start_date=start_date,
+            end_date=start_date + timedelta(hours=1),
+        )
+    except Exception as e:
+        logger.error(f"Error parsing spot price: {str(e)}", extra={"data": data})
+        raise ParseException()
+    return spot
+
+
+@router.get("/spot48/", responses={200: {"model": list[Spot]}})
+async def spot48() -> list[Spot]:
+    """Get spot prices for the next 48 hours."""
+
+    url = "https://api.porssisahko.net/v1/latest-prices.json"
+    async with AsyncClient() as client:
+        response = await client.get(url)
+    if response.status_code != 200:
+        raise FetchException(response.url, response.status_code)
+    data = response.json()
+    if not data:
+        raise FetchException(response.url, response.status_code)
+    try:
+        spot = [
+            Spot(price=d["price"], start_date=d["startDate"], end_date=d["endDate"])
+            for d in data["prices"]
+        ]
+    except Exception as e:
+        logger.error(f"Error parsing spot prices: {str(e)}", extra={"data": data})
+        raise ParseException()
+
+    return spot

src/routes/job_postings.py

@@ -22,12 +22,12 @@ async def get_job_postings(search_term: str, source: PostingSource = PostingSour
     match source:
         case PostingSource.ALL:
             logger.warning(
-                "Only Duunitori and Tyomarkkinatori are implemented at the moment."
+                "LinkedIn is not implemented at the moment. Only Duunitori and Tyomarkkinatori are implemented."
             )
             postings = await duunitori(search_term)
             postings.extend(await tyomarkkinatori(search_term))
         case PostingSource.LINKEDIN:
-            logger.error(f"Job postings from {source.value} Not implemented yet")
+            logger.error(f"Job postings from {source.value} are not implemented yet")
             raise NotImplementedException()
         case PostingSource.DUUNITORI:
             postings = await duunitori(search_term)

src/routes/weather.py

@@ -0,0 +1,26 @@
+from fastapi import APIRouter
+from httpx import AsyncClient
+
+from lib.exceptions import FetchException, ParseException
+from models.weather import Weather
+from lib import logger
+
+router = APIRouter(prefix="/weather", tags=["Weather"])
+
+
+@router.get("/")
+async def weather():
+    url = "https://www.ilmatieteenlaitos.fi/api/weather/observations?fmisid=101237&observations=true&radar=false&daily=false"
+    async with AsyncClient() as client:
+        response = await client.get(url)
+    if response.status_code != 200:
+        raise FetchException(response.url, response.status_code)
+    data = response.json()
+    if not data:
+        raise FetchException(response.url, response.status_code)
+    try:
+        weather = [Weather(**d) for d in data["observations"] if d["t2m"] is not None]
+    except Exception as e:
+        logger.error(f"Error parsing weather: {str(e)}")
+        raise ParseException()
+    return weather

uv.lock

@@ -129,8 +129,8 @@ wheels = [
 ]
 
 [[package]]
-name = "jobpostingparser"
-version = "0.1.0"
+name = "n8n_helper"
+version = "1.0.0"
 source = { virtual = "." }
 dependencies = [
     { name = "fastapi" },