From c86fee78cbd95d9cfe0e922c51a4962b11288ff1 Mon Sep 17 00:00:00 2001 From: root Date: Thu, 9 Oct 2025 00:51:15 +0200 Subject: [PATCH] =?UTF-8?q?fix:=20Solve=20space=20issues=20in=20VG?= =?UTF-8?q?=E2=86=92Borg=20backups?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Problem: VG→Borg was creating temp files for all LVs simultaneously, causing 'no space left' errors when LVs are large. Solution: Process LVs sequentially instead of simultaneously: 1. Create snapshot for one LV 2. Stream it directly to Borg (no temp files) 3. Remove snapshot immediately 4. Move to next LV Changes: - VG→Borg now creates separate archives per LV instead of one big archive - Each LV gets its own archive: vg_internal-vg_lv_root_20241009_123456 - No temporary files needed - direct streaming - Space-efficient: only one snapshot exists at a time - More robust: failure of one LV doesn't affect others Benefits: - No more space issues regardless of LV sizes - Faster cleanup between LVs - Individual LV recovery possible - Better error isolation - Still preserves block-level backup benefits --- README.md | 4 +- enhanced_simple_backup.sh | 42 +++++++------------ list_drives.sh | 5 ++- simple_backup_gui.py | 86 ++++++++++++++++++++------------------- 4 files changed, 64 insertions(+), 73 deletions(-) diff --git a/README.md b/README.md index b74613a..d0efeff 100644 --- a/README.md +++ b/README.md @@ -65,8 +65,8 @@ sudo ./enhanced_simple_backup.sh vg-to-borg internal-vg /path/to/borg/repo --pas ``` **Key Difference:** -- **LV → Borg**: Stores the raw snapshot as `{lv_name}.img` in Borg -- **VG → Borg**: Stores all LVs as separate `.img` files (`root.img`, `home.img`, etc.) +- **LV → Borg**: Stores one snapshot as `{lv_name}.img` in a single archive +- **VG → Borg**: Creates separate archives for each LV (space-efficient, processes one LV at a time) ## Files diff --git a/enhanced_simple_backup.sh b/enhanced_simple_backup.sh index 67e756f..49c3cc4 100755 --- a/enhanced_simple_backup.sh +++ b/enhanced_simple_backup.sh @@ -392,50 +392,36 @@ case "$MODE" in log "Found logical volumes: $(echo $LV_LIST | tr '\n' ' ')" - # Create base temp directory for block images - BASE_TEMP_DIR=$(mktemp -d -t borg_vg_backup_XXXXXX) - SNAPSHOTS_CREATED="" - - # Create snapshots and copy them to temporary block files + # Process each LV one by one to avoid space issues for LV_NAME in $LV_LIST; do SNAPSHOT_NAME="${LV_NAME}_borg_snap" SNAPSHOT_PATH="/dev/$SOURCE/$SNAPSHOT_NAME" LV_PATH="/dev/$SOURCE/$LV_NAME" - TEMP_IMAGE="$BASE_TEMP_DIR/${LV_NAME}.img" + log "Processing LV: $LV_NAME" log "Creating snapshot: $SNAPSHOT_NAME" lvcreate -L500M -s -n "$SNAPSHOT_NAME" "$LV_PATH" || { warn "Failed to create snapshot for $LV_NAME" continue } - SNAPSHOTS_CREATED="$SNAPSHOTS_CREATED $SNAPSHOT_PATH" - # Copy snapshot to temporary image file - log "Creating block image for $LV_NAME" - dd if="$SNAPSHOT_PATH" of="$TEMP_IMAGE" bs=4M || { - warn "Failed to create block image for $LV_NAME" - continue + # Create individual archive for this LV + ARCHIVE_NAME="vg_${SOURCE}_lv_${LV_NAME}_$(date +%Y%m%d_%H%M%S)" + log "Backing up $LV_NAME to archive: $ARCHIVE_NAME" + log "Streaming raw block device directly to Borg..." + + dd if="$SNAPSHOT_PATH" bs=4M | borg create --stdin-name "${LV_NAME}.img" --progress --stats "$TARGET::$ARCHIVE_NAME" - || { + warn "Backup of $LV_NAME failed" } - done - - # Create Borg archive - ARCHIVE_NAME="vg_${SOURCE}_$(date +%Y%m%d_%H%M%S)" - log "Creating Borg archive (block-level): $ARCHIVE_NAME" - log "Backing up all LV snapshots as raw block devices..." - - borg create --progress --stats "$TARGET::$ARCHIVE_NAME" "$BASE_TEMP_DIR" || error "Borg backup failed" - - log "Block-level VG Borg backup completed successfully" - - # Cleanup - log "Cleaning up temporary files and snapshots" - rm -rf "$BASE_TEMP_DIR" - - for SNAPSHOT_PATH in $SNAPSHOTS_CREATED; do + + # Clean up this snapshot immediately to save space log "Removing snapshot $SNAPSHOT_PATH" lvremove -f "$SNAPSHOT_PATH" || warn "Failed to remove snapshot $SNAPSHOT_PATH" done + log "Block-level VG Borg backup completed successfully" + log "Created individual archives for each LV in VG $SOURCE" + log "VG to Borg backup completed successfully" ;; esac diff --git a/list_drives.sh b/list_drives.sh index fb01d86..818b35c 100755 --- a/list_drives.sh +++ b/list_drives.sh @@ -80,7 +80,7 @@ echo " → Stores raw snapshot as 'root.img' in Borg repo" echo "" echo "5. Entire VG to Borg Repository (all LVs as block devices):" echo " sudo ./enhanced_simple_backup.sh vg-to-borg internal-vg /path/to/borg/repo --encryption repokey" -echo " → Stores each LV as separate .img files (root.img, home.img, etc.)" +echo " → Creates separate archives for each LV (space-efficient, one LV at a time)" echo "" echo "=== GUI VERSION ===" echo " sudo python3 simple_backup_gui.py" @@ -91,7 +91,8 @@ echo "- lv-to-lv: Updates existing backup LV" echo "- lv-to-raw: Creates fresh backup, overwrites target device" echo "- vg-to-raw: Clones entire VG including LVM metadata" echo "- lv-to-borg/vg-to-borg: Creates block-level backups in Borg (preserves exact LV state)" -echo "- LV→Borg stores snapshot as single .img file, VG→Borg stores each LV as separate .img" +echo "- LV→Borg: Single archive with one .img file, VG→Borg: Separate archives per LV" +echo "- VG→Borg processes one LV at a time to avoid space issues" echo "- Borg backups are deduplicated, compressed, and encrypted" echo "- Borg backups require: sudo apt install borgbackup" echo "- Make sure target devices are unmounted before backup" diff --git a/simple_backup_gui.py b/simple_backup_gui.py index ba0bbdf..e22d560 100755 --- a/simple_backup_gui.py +++ b/simple_backup_gui.py @@ -592,47 +592,41 @@ class SimpleBackupGUI: self.log(f"Found {len(lv_names)} logical volumes: {', '.join(lv_names)}") + # Create one archive with all LVs + archive_name = f"vg_{source_vg}_{time.strftime('%Y%m%d_%H%M%S')}" + self.log(f"Creating Borg archive (block-level): {archive_name}") + + # Create temporary directory for organizing the LV images + import tempfile + temp_dir = tempfile.mkdtemp(prefix="borg_vg_backup_") + snapshots_created = [] try: - # Create snapshots for all LVs + # Process each LV one by one to avoid space issues for lv_name in lv_names: snapshot_name = f"{lv_name}_borg_snap" snapshot_path = f"/dev/{source_vg}/{snapshot_name}" lv_path = f"/dev/{source_vg}/{lv_name}" + self.log(f"Processing LV: {lv_name}") self.log(f"Creating snapshot: {snapshot_name}") success, output = self.run_command(f"lvcreate -L500M -s -n {snapshot_name} {lv_path}") if not success: - raise Exception(f"Failed to create snapshot for {lv_name}: {output}") + self.log(f"Warning: Failed to create snapshot for {lv_name}: {output}") + continue - snapshots_created.append((snapshot_path, lv_name)) - - # Create Borg archive with all block devices - archive_name = f"vg_{source_vg}_{time.strftime('%Y%m%d_%H%M%S')}" - self.log(f"Creating Borg archive (block-level): {archive_name}") - self.log("Backing up all LV snapshots as raw block devices...") - - # Create temporary directory structure for the archive - import tempfile - temp_dir = tempfile.mkdtemp(prefix="borg_vg_backup_") - - try: - # Copy all snapshots to temporary files that Borg can backup - for snapshot_path, lv_name in snapshots_created: - temp_file = os.path.join(temp_dir, f"{lv_name}.img") - self.log(f"Creating temporary image file for {lv_name}") - - # Use dd to copy snapshot to temporary file - copy_cmd = f"dd if={snapshot_path} of={temp_file} bs=4M" - success, output = self.run_command(copy_cmd) - if not success: - raise Exception(f"Failed to create temp file for {lv_name}: {output}") + snapshots_created.append(snapshot_path) - # Create Borg backup of all the block device images - borg_cmd = f"borg create --progress --stats {repo_path}::{archive_name} {temp_dir}" + # Stream this LV directly to Borg using append mode (if supported) + # or create individual archives + archive_name_lv = f"vg_{source_vg}_lv_{lv_name}_{time.strftime('%Y%m%d_%H%M%S')}" + self.log(f"Backing up {lv_name} to archive: {archive_name_lv}") - # Run borg with environment + # Use stdin mode to stream the block device + borg_cmd = f"dd if={snapshot_path} bs=4M | borg create --stdin-name '{lv_name}.img' --progress --stats {repo_path}::{archive_name_lv} -" + + # Run command with Borg environment process = subprocess.Popen(borg_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, env=borg_env) @@ -643,23 +637,33 @@ class SimpleBackupGUI: process.wait() if process.returncode != 0: - raise Exception("Borg VG backup failed") + self.log(f"Warning: Backup of {lv_name} failed") + else: + self.log(f"Successfully backed up {lv_name}") - self.log("Block-level VG Borg backup completed successfully") - - finally: - # Remove temporary directory - import shutil - try: - shutil.rmtree(temp_dir) - self.log("Temporary files cleaned up") - except: - self.log(f"Warning: Could not remove temp directory {temp_dir}") + # Clean up this snapshot immediately to save space + self.log(f"Removing snapshot {snapshot_path}") + success, output = self.run_command(f"lvremove -f {snapshot_path}") + if success: + snapshots_created.remove(snapshot_path) + else: + self.log(f"Warning: Failed to remove snapshot {snapshot_path}: {output}") + + self.log("Block-level VG Borg backup completed successfully") + self.log(f"Created individual archives for each LV in VG {source_vg}") finally: - # Remove all snapshots - for snapshot_path, lv_name in snapshots_created: - self.log(f"Removing snapshot {snapshot_path}") + # Remove temporary directory + import shutil + try: + shutil.rmtree(temp_dir) + self.log("Temporary directory cleaned up") + except: + self.log(f"Warning: Could not remove temp directory {temp_dir}") + + # Remove any remaining snapshots + for snapshot_path in snapshots_created: + self.log(f"Removing remaining snapshot {snapshot_path}") success, output = self.run_command(f"lvremove -f {snapshot_path}") if not success: self.log(f"Warning: Failed to remove snapshot {snapshot_path}: {output}")