Refactor codebase to use a savefile and fetch data online

src/json_parser.py

@@ -1,42 +0,0 @@
-import json
-from pathlib import Path
-import click
-
-def _get_fields(json_data: dict) -> list[str]:
-    """
-    Extract field names from a JSON data structure.
-
-    Args:
-        json_data (dict): The JSON data from which to extract field names.
-
-    Returns:
-        list[str]: A list of field names extracted from the JSON data.
-    """
-    namelist = []
-    for attr in json_data["conceptCodes"]:
-        for name in attr["attributes"]:
-            if name["attributeName"] not in namelist:
-                namelist.append(name["attributeName"])
-            
-        
-    return namelist
-
-def parse_json(json_file: Path) -> list[str]:
-    """
-    Parse a JSON file and extract field names.
-
-    Args:
-        json_file (Path): The path to the JSON file to parse.
-
-    Returns:
-        list[str]: A list of field names extracted from the JSON file.
-
-    Raises:
-        click.FileError: If the file is not a valid JSON file.
-    """
-    try:
-        with open(json_file, "r") as f:
-            json_data = json.load(f)
-    except json.JSONDecodeError:
-        raise click.FileError(json_file, "Not a valid JSON file.")
-    return _get_fields(json_data)

src/main.py

@@ -1,23 +1,122 @@
 import click
 from pathlib import Path
+import json
+from datetime import datetime
+from pydantic import HttpUrl
+import httpx
+from urllib.parse import urlparse, urlencode
 
-from json_parser import parse_json
+from models.savefile import SaveFile, CodeItem
 from psm_parser import write_psm
-from xml_parser import get_xml_data, add_connection, write_xml_data
+from xml_parser import write_xml_data
 
+def _strip_params(url: HttpUrl) -> str:
+    parsed_url = urlparse(str(url))
+    return f"{parsed_url.scheme}://{parsed_url.netloc}{parsed_url.path}"
 
-@click.command
+def get_attributes(url: str) -> list[str]:
-@click.option('-j', '--json', type=Path, required=True, help='Path to the JSON file')
+    """
-@click.option('-x', '--xml', type=Path, required=True, help='Path to the XML file. If the file does not exist, a default configuration will be generated.')
+    Get attributes from a URL.
-@click.option('-n', '--name', type=str, required=True, help='Name of the connection')
+
-@click.option('-d', '--description', type=str, required=True, help='Description of the connection')
+    Args:
-@click.option('-p', '--pretty', is_flag=True, default=False, help='Enable pretty output')
+        url (str): The URL to get attributes from.
-def main(json: Path, xml: Path, name: str, description: str, pretty: bool):
+
-    xml_data = get_xml_data(xml)
+    Returns:
-    field_names = parse_json(json)
+        list[str]: A list of attributes.
-    root = add_connection(xml_data, name, field_names, description=description)
+    """
-    write_xml_data(root, xml, pretty=pretty)
+    query_params = {"pageSize": 500}
-    write_psm(name, field_names)
+
+    new_url = f"{url}?{urlencode(query_params, doseq=True)}"
+    
+    response = httpx.get(new_url)
+    if response.status_code != 200:
+        click.echo(f"Failed to get attributes from {url}")
+        return []
+    
+    conseptCodes = response.json()["conceptCodes"]
+    retval = []
+    for attr in conseptCodes:
+        for name in attr["attributes"]:
+            if name["attributeName"] not in retval:
+                retval.append(name["attributeName"])
+    
+    return retval
+
+def write_json(json_file: Path, data: SaveFile):
+    with open(json_file, "w") as f:
+        f.write(data.model_dump_json())
+
+@click.group
+def main():
+    click.echo("Varha CodeServer Connector")
+    pass
+
+@main.command
+@click.argument('json_file', type=Path, required=True)
+@click.option('--url', '-u', type=str, required=True, help="URL of the connection")
+@click.option('--name', '-n', type=str, required=True, help="Name of the connection")
+def add(json_file: Path, url: str, name: str):
+    """
+    Add a new connection to the JSON file.
+
+    Args:
+        json_file (Path): The path to the JSON file.
+        url (str): The URL of the connection.
+        name (str): The name of the connection.
+    """
+    server_data = SaveFile()
+    if json_file.exists():
+        with open(json_file, "r") as f:
+            server_data = SaveFile(**json.load(f))
+    
+    # Check if name already exists
+    existing_names = [code.name for code in server_data.conceptCodes]
+    if name in existing_names:
+        click.echo(f"Name {name} already exists.")
+        return
+
+    # Get attributes from the URL
+    attributes = get_attributes(_strip_params(url))
+    server_data.conceptCodes.append(CodeItem(name=name, url=_strip_params(url), attributes=attributes))
+    server_data.updated_at = datetime.now()
+    write_json(json_file, server_data)
+
+@main.command
+@click.argument('json_file', type=Path, required=True)
+def listUrls(json_file: Path):
+    #TODO: Format the output
+    if not json_file.exists():
+        click.echo(f"File {json_file} does not exist.")
+        return
+    with open(json_file, "r") as f:
+        server_data = SaveFile(**json.load(f))
+    click.echo(server_data.model_dump_json(indent=4))
+
+@main.command
+@click.argument('json_file', type=Path, required=True)
+@click.option('--name', '-n', type=str, required=True, help="Name of the connection")
+def remove(json_file: Path, name: str):
+    if not json_file.exists():
+        click.echo(f"File {json_file} does not exist.")
+        return
+    with open(json_file, "r") as f:
+        server_data = SaveFile(**json.load(f))
+    server_data.conceptCodes = [code for code in server_data.conceptCodes if code.name != name]
+    server_data.updated_at = datetime.now()
+    write_json(json_file, server_data)
+
+@main.command
+@click.argument('json_file', type=Path, required=True)
+def write_output(json_file: Path):
+    if not json_file.exists():
+        click.echo(f"File {json_file} does not exist.")
+        return
+    server_data = None
+    with open(json_file, "r") as f:
+        server_data = SaveFile(**json.load(f))
+    
+    write_psm(server_data)
+    write_xml_data(server_data)
 
 
 if __name__ == "__main__":

src/models/savefile.py

@@ -0,0 +1,14 @@
+from pydantic import BaseModel, Field
+from datetime import datetime
+
+class CodeItem(BaseModel):
+    name: str
+    url: str
+    attributes: list[str]
+    checked_at: datetime = Field(default_factory=datetime.now)
+
+class SaveFile(BaseModel):
+    version: int = 1
+    conceptCodes: list[CodeItem] = Field(default_factory=list)
+    created_at: datetime = Field(default_factory=datetime.now)
+    updated_at: datetime = Field(default_factory=datetime.now)

src/psm_parser.py

@@ -1,8 +1,7 @@
-import shutil
 from pathlib import Path
-from datetime import datetime
 
 from xml_parser import sanitize_name
+from models.savefile import SaveFile
 
 get_all_code = """
 function Get-AllCodeServerPages!NAME! {
@@ -145,17 +144,19 @@ def define_strings(items: list[str]) -> str:
     return "\n".join(["\t" + "[string]$" + "{" + sanitize_name(item) + "}" for item in items])
 
 
-def write_psm(name, items: list[str], filename: Path = Path("CodeServerConnector.psm1")) -> str:
+def write_psm(savefile: SaveFile) -> None:
-    if filename.exists():
-        backup_filename = str(filename) + f".{datetime.now().strftime('%Y-%m-%d.%H%M%S')}.bak"
-        shutil.copy2(filename, backup_filename)
 
-    output = "# ------- BEGIN CODE BLOCK -------\n"
+    filename = Path("output/CodeServerConnector.psm1")
-    output += get_all_code.replace("!NAME!", name)
+    if not filename.parent.exists():
-    output += get_code.replace("!NAME!", name)
+        filename.parent.mkdir(parents=True, exist_ok=True)
-    output += convert_from.replace("!NAME!", name).replace("!MAPS!", define_maps(items))
+
-    output += class_definition.replace("!NAME!", name).replace("!VARIABLES!", define_variables(items)).replace("!STRINGS!", define_strings(items))
+    with open(filename, "w", encoding="utf-8-sig") as f:
-    output += "# ------- END CODE BLOCK -------\n"
+        f.write("# ------- BEGIN CODE BLOCK -------\n")
-    with open(filename, "a", encoding="utf-8-sig") as f:
+        for code in savefile.conceptCodes:
-        f.write(output)
+            f.write(get_all_code.replace("!NAME!", code.name))
-    pass
+            f.write(get_code.replace("!NAME!", code.name))
+            f.write(convert_from.replace("!NAME!", code.name).replace("!MAPS!", define_maps(code.attributes)))
+            f.write(class_definition.replace("!NAME!", code.name).replace("!VARIABLES!", define_variables(code.attributes)).replace("!STRINGS!", define_strings(code.attributes)))
+        f.write("# ------- END CODE BLOCK -------\n")
+
+    return

src/xml_parser.py

@@ -3,16 +3,20 @@ from pathlib import Path
 import click
 from datetime import datetime
 
-def write_xml_data(root: ET.Element, file_path: Path, pretty: bool = True) -> None:
+from models.savefile import SaveFile
-    if file_path.exists():
+
-        file_path.rename(str(file_path) + f".{datetime.now().strftime('%Y-%m-%d.%H%M%S')}.bak")
+def write_xml_data(savefile: SaveFile) -> None:
-    xml_str = ET.tostring(root, encoding="utf-8", xml_declaration=True, pretty_print=pretty)
+    filename = Path("output/CodeServerConnector.xml")
-    if pretty:
+    if not filename.parent.exists():
-        # Parse the XML string to ensure proper indentation
+        filename.parent.mkdir(parents=True, exist_ok=True)
-        parser = ET.XMLParser(remove_blank_text=True, strip_cdata=False)
+    
-        root = ET.fromstring(xml_str, parser=parser)
+    root = _generate_xml_configuration()
-        xml_str = ET.tostring(root, encoding="utf-8", xml_declaration=True, pretty_print=pretty)
+    for code in savefile.conceptCodes:
-    file_path.write_bytes(xml_str)
+        root = add_connection(root, code.name, code.attributes)
+    
+    xml_str = ET.tostring(root, encoding="utf-8", xml_declaration=True, pretty_print=True).decode("utf-8-sig")
+    with open(filename, "w", encoding="utf-8-sig") as f:
+        f.write(xml_str)
 
 def sanitize_name(name: str) -> str:
     """