From 21aa75e1700edd88605970dcc7b42319f51bbcfe Mon Sep 17 00:00:00 2001 From: Esa Kataja Date: Wed, 14 May 2025 11:32:01 +0300 Subject: [PATCH] v1.0rc1 --- .gitignore | 2 + src/app.py | 23 ++++++++ src/excludes.txt | 36 ++++++++++++ src/helpers.py | 149 +++++++++++++++++++++++++++++++++++++++++++++++ src/models.py | 110 ++++++++++++++++++++++++++++++++++ src/targets.json | 26 +++++++++ 6 files changed, 346 insertions(+) create mode 100644 src/app.py create mode 100644 src/excludes.txt create mode 100644 src/helpers.py create mode 100644 src/models.py create mode 100644 src/targets.json diff --git a/.gitignore b/.gitignore index 505a3b1..8927364 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,5 @@ wheels/ # Virtual environments .venv + +.vscode \ No newline at end of file diff --git a/src/app.py b/src/app.py new file mode 100644 index 0000000..f39c5b0 --- /dev/null +++ b/src/app.py @@ -0,0 +1,23 @@ +import typer +import os +from helpers import get_targets, run_backup, ask_repokey + +app = typer.Typer() + + +@app.command() +def backup(): + if not os.environ.get("BORG_PASSPHRASE"): + ask_repokey() + targets = get_targets() + + for target in targets: + if not target.enabled: + print(f"Skipping disabled target: {target.path}") + continue + result_info = run_backup(target) + print(result_info.model_dump_json(indent=2)) + + +if __name__ == "__main__": + app() diff --git a/src/excludes.txt b/src/excludes.txt new file mode 100644 index 0000000..9954652 --- /dev/null +++ b/src/excludes.txt @@ -0,0 +1,36 @@ +.Trash-1000 +.Trash* +.Trashes +Thumbs.db +desktop.ini +.DS_Store +._* +*.tmp +*.temp +~* +*~ +*.swp +.*.swp +.cache/ +__pycache__/ +node_modules/ +.npm/ +.yarn/ +dist/ +build/ +out/ +*.o +*.obj +*.exe +*.dll +*.py[cod] +*.log +logs/ +*.db +*.sqlite +*.sqlite3 +.git/ +.svn/ +.hg/ +.idea/ +.vscode/ diff --git a/src/helpers.py b/src/helpers.py new file mode 100644 index 0000000..16dfa10 --- /dev/null +++ b/src/helpers.py @@ -0,0 +1,149 @@ +from pathlib import Path +import json +from models import BackupItem, ResultInfo +from datetime import datetime +from subprocess import run +import os +import typer + +# Default exclude patterns for backups +default_excludes = [ + ".Trash-1000", + ".Trash*", + ".Trashes", # Trash directories + "Thumbs.db", + "desktop.ini", # Windows system files + ".DS_Store", + "._*", # macOS system files + "*.tmp", + "*.temp", + "~*", + "*~", # Temporary files + "*.swp", + ".*.swp", # Vim swap files + ".cache/", + "__pycache__/", # Cache directories + "node_modules/", + ".npm/", + ".yarn/", # Node.js directories + "dist/", + "build/", + "out/", # Build directories + "*.o", + "*.obj", + "*.exe", + "*.dll", # Compiled binaries + "*.py[cod]", # Python compiled files + "*.log", + "logs/", # Log files + "*.db", + "*.sqlite", + "*.sqlite3", # Database files + ".git/", + ".svn/", + ".hg/", # Version control directories + ".idea/", + ".vscode/", # IDE directories +] + + +def ask_repokey() -> None: + repokey = typer.prompt("Enter repository key: ", hide_input=True) + os.environ["BORG_PASSPHRASE"] = repokey + + +def get_targets() -> list[BackupItem]: + JSON_TARGETS = Path(__file__).parent / "targets.json" + with open(JSON_TARGETS, "r") as f: + targets = json.load(f) + return [BackupItem(**target) for target in targets] + + +def add_target(target: BackupItem) -> None: + # Get current targets + targets = get_targets() + + # Add new target + targets.append(target) + + # Save targets + write_targets(targets) + + +def write_targets(targets: list[BackupItem]) -> None: + """Save targets to JSON file""" + JSON_TARGETS = Path(__file__).parent / "targets.json" + with open(JSON_TARGETS, "w") as f: + json.dump( + [target.json_serializable for target in targets], + f, + ensure_ascii=False, + indent=2, + ) + + +def set_targets(targets: list[BackupItem]) -> None: + JSON_TARGETS = Path(__file__).parent / "targets.json" + with open(JSON_TARGETS, "w") as f: + json.dump( + [target.json_serializable for target in targets], + f, + ensure_ascii=False, + indent=2, + ) + + +def prune_backups(targets: list[BackupItem]) -> None: + for target in targets: + if not target.enabled: + continue + cmd = [ + "borg", + "prune", + "--keep-daily", + "1", + "--keep-weekly", + "2", + "--keep-monthly", + "3", + "--keep-yearly", + "1", + target.target, + ] + print("Pruning target:", target.path) + run(cmd, capture_output=True) + print("Pruned target:", target.path) + print("Compacting cache") + run(["borg", "compact", target.target], capture_output=True) + print("Compacted cache") + + +def run_backup(target: BackupItem) -> None: + date_str = datetime.now().strftime("%Y-%m-%d_%H-%M-%S") + # Build borg command with base options + cmd = [ + "borg", + "create", + "-C", + f"{target.compression.value},{target.comp_ratio}", + "--exclude-from", + "/home/esa/bin/backupper/src/excludes.txt", + "--stats", + "--json", + f"{target.target}::{date_str}", + target.path, + ] + + # Add exclude option only if there are excludes defined + if target.excludes: # This is true if the list is not empty + cmd.insert(3, "--exclude") + cmd.insert(4, ",".join(target.excludes)) + print("Running target:", target.path) + result = run(cmd, capture_output=True) + if result.returncode != 0: + print("Result:", result.stderr.decode("utf-8")) + raise Exception(result.stderr.decode("utf-8")) + result_info = ResultInfo(**json.loads(result.stdout.decode("utf-8"))) + prune_backups([target]) + + return result_info diff --git a/src/models.py b/src/models.py new file mode 100644 index 0000000..c46ffce --- /dev/null +++ b/src/models.py @@ -0,0 +1,110 @@ +from pydantic import BaseModel +from enum import Enum +from pathlib import Path +from datetime import datetime + + +class Compression(Enum): + GZIP = "gzip" + ZSTD = "zstd" + + +class BackupItem(BaseModel): + path: Path + compression: Compression = Compression.ZSTD + comp_ratio: int = 19 + target: Path + excludes: list[str] = [] + enabled: bool = True + + @property + def json_serializable(self): + return { + "path": str(self.path), + "compression": self.compression.value, + "comp_ratio": self.comp_ratio, + "target": str(self.target), + "excludes": self.excludes, + "enabled": self.enabled, + } + + +{ + "archive": { + "command_line": [ + "/usr/bin/borg", + "create", + "-C", + "zstd,19", + "--exclude-from", + "/home/esa/bin/backupper/src/excludes.txt", + "--stats", + "--json", + "/home/esa/pCloudDrive/Backups/borgBackup/documents::2025-05-14_10-54-45", + "/home/esa/Documents", + ], + "duration": 57.788986, + "end": "2025-05-14T10:55:43.000000", + "id": "9119a981b8e3b2a860f9fd5ed4543f9ec126d39bccfe510fb4e36b725331983b", + "limits": {"max_archive_size": 3.2854144431110464e-05}, + "name": "2025-05-14_10-54-45", + "start": "2025-05-14T10:54:46.000000", + "stats": { + "compressed_size": 1285729209, + "deduplicated_size": 1285495760, + "nfiles": 1744, + "original_size": 1321765294, + }, + }, + "cache": { + "path": "/home/esa/.cache/borg/bf32695b5fde1b8eb6a1a278e8be9a5dde89c46b19ebe8831ccae3e329e6c09c", + "stats": { + "total_chunks": 2138, + "total_csize": 1285728520, + "total_size": 1321764559, + "total_unique_chunks": 2104, + "unique_csize": 1285644534, + "unique_size": 1320800483, + }, + }, + "encryption": {"mode": "repokey-blake2"}, + "repository": { + "id": "bf32695b5fde1b8eb6a1a278e8be9a5dde89c46b19ebe8831ccae3e329e6c09c", + "last_modified": "2025-05-14T10:55:43.000000", + "location": "/home/esa/pCloudDrive/Backups/borgBackup/documents", + }, +} + + +class _Result_Archive_Stats(BaseModel): + compressed_size: int + deduplicated_size: int + nfiles: int + original_size: int + + +class _Result_Archive(BaseModel): + command_line: list[str] + duration: float + end: datetime + id: str + limits: dict + name: str + start: datetime + stats: _Result_Archive_Stats + + +class _Result_Encryption(BaseModel): + mode: str + + +class _Result_Repository(BaseModel): + id: str + last_modified: str + location: str + + +class ResultInfo(BaseModel): + archive: _Result_Archive + encryption: _Result_Encryption + repository: _Result_Repository diff --git a/src/targets.json b/src/targets.json new file mode 100644 index 0000000..6c170c6 --- /dev/null +++ b/src/targets.json @@ -0,0 +1,26 @@ +[ + { + "path": "/home/esa/Pictures", + "compression": "zstd", + "comp_ratio": 19, + "target": "/home/esa/pCloudDrive/Backups/borgBackup/pictures", + "excludes": [], + "enabled": true + }, + { + "path": "/home/esa/Music", + "compression": "zstd", + "comp_ratio": 19, + "target": "/home/esa/pCloudDrive/Backups/borgBackup/music", + "excludes": [], + "enabled": true + }, + { + "path": "/home/esa/Documents", + "compression": "zstd", + "comp_ratio": 19, + "target": "/home/esa/pCloudDrive/Backups/borgBackup/documents", + "excludes": [], + "enabled": true + } +] \ No newline at end of file