From ee32c02adaf20f3459e0b12804463869e13ea20d Mon Sep 17 00:00:00 2001
From: The Duskfall Portal Crew
<58930427+duskfallcrew@users.noreply.github.com>
Date: Thu, 26 Dec 2024 18:15:39 +1300
Subject: [PATCH 01/31] Update and rename install.py to __init__.py
---
__init__.py | 18 ++++++++++++++++++
install.py | 9 ---------
2 files changed, 18 insertions(+), 9 deletions(-)
create mode 100644 __init__.py
delete mode 100644 install.py
diff --git a/__init__.py b/__init__.py
new file mode 100644
index 0000000..eb241bd
--- /dev/null
+++ b/__init__.py
@@ -0,0 +1,18 @@
+from modules import scripts
+from scripts import hfbackup_script
+
+class Script(scripts.Script):
+ def title(self):
+ return "Huggingface Backup"
+
+ def show(self, is_img2img):
+ return scripts.AlwaysVisible
+
+ def ui(self, is_img2img):
+ return hfbackup_script.on_ui(self)
+
+ def run(self, p, *args):
+ return hfbackup_script.on_run(self, p, *args)
+
+ def on_script_load(self):
+ return hfbackup_script.on_script_load(self)
diff --git a/install.py b/install.py
deleted file mode 100644
index 5189788..0000000
--- a/install.py
+++ /dev/null
@@ -1,9 +0,0 @@
-import launch
-
-if not launch.is_installed("huggingface_hub"):
- launch.run_pip("install huggingface_hub==4.30.2", "requirements for Hugging Face Hub Uploader")
-
-if not launch.is_installed("glob2"):
- launch.run_pip("install glob2", "requirements for Hugging Face Hub Uploader")
-
-launch.run_pip("install -r requirements.txt")
From a055a16980f6ba50f5b4fd34dc53abc4ed3d4db5 Mon Sep 17 00:00:00 2001
From: The Duskfall Portal Crew
<58930427+duskfallcrew@users.noreply.github.com>
Date: Thu, 26 Dec 2024 18:20:08 +1300
Subject: [PATCH 02/31] Update and rename hfbackup.py to hfbackup_script.py
---
scripts/hfbackup.py | 110 ------------------------
scripts/hfbackup_script.py | 169 +++++++++++++++++++++++++++++++++++++
2 files changed, 169 insertions(+), 110 deletions(-)
delete mode 100644 scripts/hfbackup.py
create mode 100644 scripts/hfbackup_script.py
diff --git a/scripts/hfbackup.py b/scripts/hfbackup.py
deleted file mode 100644
index 756be24..0000000
--- a/scripts/hfbackup.py
+++ /dev/null
@@ -1,110 +0,0 @@
-# scripts/hf_backup.py
-import modules.scripts as scripts
-import gradio as gr
-import os
-from modules import shared, script_callbacks
-from huggingface_hub import HfApi
-import glob
-
-def upload_to_huggingface(username, repo, write_key, dir_path, file_type, pr_message):
- try:
- # Initialize HF API
- api = HfApi(token=write_key or shared.opts.data.get("hf_write_key", ""))
- if not api.token:
- return "Error: No Hugging Face Write API Key provided"
-
- repo_id = f"{username}/{repo}"
- results = []
-
- # Get files from directory
- if dir_path:
- files = glob.glob(os.path.join(dir_path, f"*.{file_type}"))
- if not files:
- return f"No .{file_type} files found in {dir_path}"
-
- for file_path in files:
- try:
- file_name = os.path.basename(file_path)
- results.append(f"Uploading: {file_name}")
-
- response = api.upload_file(
- path_or_fileobj=file_path,
- path_in_repo=file_name,
- repo_id=repo_id,
- create_pr=True,
- commit_message=pr_message or f"Upload {file_name}"
- )
- results.append(f"✓ Successfully uploaded {file_name}")
-
- except Exception as e:
- results.append(f"✗ Error uploading {file_name}: {str(e)}")
-
- return "\n".join(results)
- except Exception as e:
- return f"Error: {str(e)}"
-
-def on_ui_tabs():
- with gr.Blocks(analytics_enabled=False) as hf_interface:
- with gr.Column():
- gr.HTML("""
-
-
🤗 Hugging Face Hub Uploader
-
Backup your models and files to Hugging Face
-
- """)
-
- with gr.Row():
- username = gr.Textbox(
- label="Hugging Face Username",
- placeholder="Your HF username",
- value=shared.opts.data.get("hf_default_username", "")
- )
- repository = gr.Textbox(
- label="Repository Name",
- placeholder="Name of the repository"
- )
-
- with gr.Row():
- write_key = gr.Textbox(
- label="Write Key (optional if set in settings)",
- placeholder="Your HF write token",
- type="password"
- )
-
- with gr.Row():
- dir_picker = gr.Textbox(
- label="Directory Path",
- placeholder="Path to directory containing files"
- )
- file_type = gr.Radio(
- label="File Type",
- choices=["safetensors", "ckpt", "pt", "bin", "zip", "jpg", "png"],
- value="safetensors",
- type="value"
- )
-
- pr_message = gr.Textbox(
- label="Pull Request Message",
- placeholder="Description of your upload",
- value="Backup files"
- )
-
- upload_button = gr.Button("🚀 Upload to Hugging Face")
- result = gr.Textbox(label="Results", interactive=False)
-
- upload_button.click(
- fn=upload_to_huggingface,
- inputs=[
- username,
- repository,
- write_key,
- dir_picker,
- file_type,
- pr_message
- ],
- outputs=result
- )
-
- return [(hf_interface, "HF Backup", "hf_backup_tab")]
-
-script_callbacks.on_ui_tabs(on_ui_tabs)
diff --git a/scripts/hfbackup_script.py b/scripts/hfbackup_script.py
new file mode 100644
index 0000000..2033bf1
--- /dev/null
+++ b/scripts/hfbackup_script.py
@@ -0,0 +1,169 @@
+# This is the full code we built on last step
+
+import os
+import time
+import datetime
+import threading
+import gradio as gr
+import subprocess
+import logging
+from modules import script_callbacks, shared
+from git import Repo
+
+# Constants
+REPO_NAME = 'sd-webui-backups'
+BACKUP_INTERVAL = 3600 # 1 hour in seconds
+HF_TOKEN_KEY = 'hf_token'
+BACKUP_PATHS_KEY = 'backup_paths'
+SD_PATH_KEY = 'sd_path'
+HF_USER_KEY = 'hf_user'
+DEFAULT_BACKUP_PATHS = ['models/Stable-diffusion', 'models/VAE', 'embeddings', 'loras']
+
+# --- Logging Setup ---
+logging.basicConfig(level=logging.INFO,
+ format='%(asctime)s - %(levelname)s - %(message)s',
+ handlers=[
+ logging.StreamHandler()
+ ])
+logger = logging.getLogger(__name__)
+
+
+# --- Git Related Functions ---
+def clone_or_create_repo(repo_url, repo_path, script):
+ script.status = "Checking/Cloning Repo..."
+ if os.path.exists(repo_path) and os.path.isdir(repo_path):
+ logger.info(f"Repository already exists at {repo_path}, updating...")
+ repo = Repo(repo_path)
+ if repo.is_dirty():
+ logger.warning("Local repo has uncommitted changes. Commit those before running to make sure nothing breaks.")
+ script.status = "Local repo has uncommitted changes"
+
+ else:
+ logger.info(f"Cloning repository from {repo_url} to {repo_path}")
+ script.status = "Cloning repository"
+ try:
+ Repo.clone_from(repo_url, repo_path)
+ except Exception as e:
+ logger.error(f"Error creating or cloning repo: {e}")
+ script.status = f"Error creating or cloning repo: {e}"
+ raise
+ script.status = "Repo ready"
+ return repo
+
+def git_push_files(repo_path, commit_message, script):
+ script.status = "Pushing changes..."
+ try:
+ repo = Repo(repo_path)
+ repo.git.add(all=True)
+ repo.index.commit(commit_message)
+ origin = repo.remote(name='origin')
+ origin.push()
+ logger.info(f"Changes pushed successfully to remote repository.")
+ script.status = "Pushing Complete"
+ except Exception as e:
+ logger.error(f"Git push failed: {e}")
+ script.status = f"Git push failed: {e}"
+ raise
+
+# --- Backup Logic ---
+def backup_files(paths, hf_client, script):
+ logger.info("Starting backup...")
+ script.status = "Starting Backup..."
+ repo_id = script.hf_user + "/" + REPO_NAME
+ repo_path = os.path.join(script.basedir, 'backup')
+ sd_path = script.sd_path
+
+ try:
+ repo = clone_or_create_repo(f"https://huggingface.co/{repo_id}", repo_path, script)
+ except Exception as e:
+ logger.error("Error starting the backup, please see the traceback.")
+ return
+
+ for base_path in paths:
+ logger.info(f"Backing up: {base_path}")
+ for root, _, files in os.walk(os.path.join(sd_path, base_path)):
+ for file in files:
+ local_file_path = os.path.join(root, file)
+ repo_file_path = os.path.relpath(local_file_path, start=sd_path)
+ try:
+ os.makedirs(os.path.dirname(os.path.join(repo_path, repo_file_path)), exist_ok=True)
+ subprocess.run(["cp", local_file_path, os.path.join(repo_path, repo_file_path)], check=True, capture_output=True)
+ logger.info(f"Copied: {repo_file_path}")
+ script.status = f"Copied: {repo_file_path}"
+ except Exception as e:
+ logger.error(f"Error copying {repo_file_path}: {e}")
+ script.status = f"Error copying: {repo_file_path}: {e}"
+ return
+
+ try:
+ git_push_files(repo_path, f"Backup at {datetime.datetime.now()}", script)
+ logger.info("Backup complete")
+ script.status = "Backup Complete"
+ except Exception as e:
+ logger.error("Error pushing to the repo: ", e)
+ return
+
+def start_backup_thread(script):
+ threading.Thread(target=backup_loop, args=(script,), daemon=True).start()
+
+def backup_loop(script):
+ try:
+ backup_files(script.backup_paths, None, script)
+ except Exception as e:
+ print(f"Backup error: {e}")
+ time.sleep(BACKUP_INTERVAL)
+
+# Gradio UI Setup
+def on_ui(script):
+ with gr.Column():
+ with gr.Row():
+ with gr.Column(scale=3):
+ hf_token_box = gr.Textbox(label="Huggingface Token", type='password', value=script.hf_token)
+ def on_token_change(token):
+ script.hf_token = token
+ script.save()
+ hf_token_box.change(on_token_change, inputs=[hf_token_box], outputs=None)
+ with gr.Column(scale=1):
+ status_box = gr.Textbox(label="Status", value=script.status)
+
+ def on_start_button():
+ start_backup_thread(script)
+ return "Starting Backup"
+
+ start_button = gr.Button(value="Start Backup")
+ start_button.click(on_start_button, inputs=None, outputs=[status_box])
+
+ with gr.Row():
+ with gr.Column():
+ sd_path_box = gr.Textbox(label="SD Webui Path", value=script.sd_path)
+ def on_sd_path_change(path):
+ script.sd_path = path
+ script.save()
+ sd_path_box.change(on_sd_path_change, inputs=[sd_path_box], outputs=None)
+ with gr.Column():
+ hf_user_box = gr.Textbox(label="Huggingface Username", value=script.hf_user)
+ def on_hf_user_change(user):
+ script.hf_user = user
+ script.save()
+ hf_user_box.change(on_hf_user_change, inputs=[hf_user_box], outputs=None)
+ with gr.Row():
+ backup_paths_box = gr.Textbox(label="Backup Paths (one path per line)", lines=4, value='\n'.join(script.backup_paths))
+ def on_backup_paths_change(paths):
+ paths_list = paths.split('\n')
+ paths_list = [p.strip() for p in paths_list if p.strip()]
+ script.backup_paths = paths_list
+ script.save()
+ backup_paths_box.change(on_backup_paths_change, inputs=[backup_paths_box], outputs=None)
+
+def on_run(script, p, *args):
+ pass
+
+def on_script_load(script):
+ script.hf_token = script.load().get(HF_TOKEN_KEY, '')
+ script.backup_paths = script.load().get(BACKUP_PATHS_KEY, DEFAULT_BACKUP_PATHS)
+ script.sd_path = script.load().get(SD_PATH_KEY, '')
+ script.hf_user = script.load().get(HF_USER_KEY, '')
+ script.status = "Not running"
+
+script_callbacks.on_ui_tabs(on_ui)
+script_callbacks.on_script_load(on_script_load)
From c303c822e3a08ff6b4052a51d713fd8c98d932af Mon Sep 17 00:00:00 2001
From: The Duskfall Portal Crew
<58930427+duskfallcrew@users.noreply.github.com>
Date: Thu, 26 Dec 2024 18:21:14 +1300
Subject: [PATCH 03/31] Delete scripts/manifest.json
---
scripts/manifest.json | 14 --------------
1 file changed, 14 deletions(-)
delete mode 100644 scripts/manifest.json
diff --git a/scripts/manifest.json b/scripts/manifest.json
deleted file mode 100644
index 9eeb5bc..0000000
--- a/scripts/manifest.json
+++ /dev/null
@@ -1,14 +0,0 @@
-# manifest.json
-{
- "name": "Hugging Face Backup",
- "version": "1.0.0",
- "description": "Backup your models and files to Hugging Face Hub",
- "author": "Duskfallcrew @ Civitai / Earth & Dusk Media",
- "repository": "",
- "tags": ["huggingface", "backup", "models"],
- "requirements": ["huggingface_hub==4.30.2", "glob2"],
- "dependencies": {},
- "extension": {
- "type": "tab",
- "categories": ["utilities"]
- }
From 8e151bd6234bfab76a082eacb81ab11de41be762 Mon Sep 17 00:00:00 2001
From: The Duskfall Portal Crew
<58930427+duskfallcrew@users.noreply.github.com>
Date: Thu, 26 Dec 2024 18:22:29 +1300
Subject: [PATCH 04/31] Delete scripts/verify_setup.py
---
scripts/verify_setup.py | 52 -----------------------------------------
1 file changed, 52 deletions(-)
delete mode 100644 scripts/verify_setup.py
diff --git a/scripts/verify_setup.py b/scripts/verify_setup.py
deleted file mode 100644
index 8a76c36..0000000
--- a/scripts/verify_setup.py
+++ /dev/null
@@ -1,52 +0,0 @@
-# scripts/verify_setup.py
-import os
-import importlib.util
-import sys
-
-def verify_extension_setup():
- results = []
-
- # Check directory structure
- base_dir = os.path.dirname(os.path.dirname(__file__))
- required_files = [
- 'install.py',
- 'requirements.txt',
- 'scripts/hf_backup.py',
- 'scripts/settings.py'
- ]
-
- for file in required_files:
- if os.path.exists(os.path.join(base_dir, file)):
- results.append(f"✓ Found {file}")
- else:
- results.append(f"✗ Missing {file}")
-
- # Check dependencies
- dependencies = {
- 'huggingface_hub': '4.30.2',
- 'glob2': '0.7'
- }
-
- for package, version in dependencies.items():
- try:
- imported = importlib.import_module(package)
- actual_version = getattr(imported, '__version__', 'unknown')
- if actual_version == version:
- results.append(f"✓ {package} version {version} installed correctly")
- else:
- results.append(f"! {package} version mismatch. Expected {version}, got {actual_version}")
- except ImportError:
- results.append(f"✗ {package} not installed")
-
- # Check HF token (if set)
- from modules import shared
- if hasattr(shared.opts, 'data') and shared.opts.data.get("hf_write_key"):
- results.append("✓ HF Write token is set")
- else:
- results.append("! HF Write token not set in settings")
-
- return "\n".join(results)
-
-# You can run this directly to test:
-if __name__ == "__main__":
- print(verify_extension_setup())
From e49e547a45c719087c2cdc9140cbde550906aec6 Mon Sep 17 00:00:00 2001
From: The Duskfall Portal Crew
<58930427+duskfallcrew@users.noreply.github.com>
Date: Thu, 26 Dec 2024 18:43:49 +1300
Subject: [PATCH 05/31] Update and rename settings.py to ui-settings.py
---
scripts/{settings.py => ui-settings.py} | 17 ++++++++++++-----
1 file changed, 12 insertions(+), 5 deletions(-)
rename scripts/{settings.py => ui-settings.py} (67%)
diff --git a/scripts/settings.py b/scripts/ui-settings.py
similarity index 67%
rename from scripts/settings.py
rename to scripts/ui-settings.py
index 9dff717..dc0a358 100644
--- a/scripts/settings.py
+++ b/scripts/ui-settings.py
@@ -1,8 +1,5 @@
import gradio as gr
-import os
from modules import shared, script_callbacks
-from huggingface_hub import HfApi
-import glob
def on_ui_settings():
section = ('huggingface', "Hugging Face")
@@ -11,7 +8,7 @@ def on_ui_settings():
shared.OptionInfo(
"",
"Hugging Face Write API Key",
- gr.Password, # Changed to Password type for security
+ gr.Password,
{"interactive": True},
section=section
)
@@ -21,7 +18,17 @@ def on_ui_settings():
shared.OptionInfo(
"",
"Hugging Face Read API Key",
- gr.Password, # Changed to Password type for security
+ gr.Password,
+ {"interactive": True},
+ section=section
+ )
+ )
+ shared.opts.add_option(
+ "git_credential_store",
+ shared.OptionInfo(
+ True,
+ "Use Git Credential Store",
+ gr.Checkbox,
{"interactive": True},
section=section
)
From 8a8ace111296ed32dabae21d0c0ed8e040641b30 Mon Sep 17 00:00:00 2001
From: The Duskfall Portal Crew
<58930427+duskfallcrew@users.noreply.github.com>
Date: Thu, 26 Dec 2024 18:49:22 +1300
Subject: [PATCH 06/31] Update hfbackup_script.py
---
scripts/hfbackup_script.py | 70 ++++++++++++++++++++++++--------------
1 file changed, 44 insertions(+), 26 deletions(-)
diff --git a/scripts/hfbackup_script.py b/scripts/hfbackup_script.py
index 2033bf1..9ab710d 100644
--- a/scripts/hfbackup_script.py
+++ b/scripts/hfbackup_script.py
@@ -1,5 +1,3 @@
-# This is the full code we built on last step
-
import os
import time
import datetime
@@ -9,6 +7,7 @@
import logging
from modules import script_callbacks, shared
from git import Repo
+import shutil
# Constants
REPO_NAME = 'sd-webui-backups'
@@ -27,48 +26,73 @@
])
logger = logging.getLogger(__name__)
+# --- Helper function for updating the status ---
+def update_status(script, status, file=None):
+ if file:
+ script.status = f"{status}: {file}"
+ print(f"{status}: {file}") # For console logging.
+ else:
+ script.status = status
+ print(status) # For console logging
# --- Git Related Functions ---
def clone_or_create_repo(repo_url, repo_path, script):
- script.status = "Checking/Cloning Repo..."
+ update_status(script, "Checking/Cloning Repo...")
if os.path.exists(repo_path) and os.path.isdir(repo_path):
logger.info(f"Repository already exists at {repo_path}, updating...")
repo = Repo(repo_path)
if repo.is_dirty():
logger.warning("Local repo has uncommitted changes. Commit those before running to make sure nothing breaks.")
- script.status = "Local repo has uncommitted changes"
-
+ update_status(script, "Local repo has uncommitted changes")
else:
logger.info(f"Cloning repository from {repo_url} to {repo_path}")
- script.status = "Cloning repository"
+ update_status(script, "Cloning repository")
try:
- Repo.clone_from(repo_url, repo_path)
+ use_git_credential_store = shared.opts.data.get("git_credential_store", True)
+ if use_git_credential_store:
+ repo = Repo.clone_from(repo_url, repo_path)
+ else:
+ if "HF_TOKEN" not in os.environ:
+ update_status(script, "HF_TOKEN environment variable not found")
+ raise Exception("HF_TOKEN environment variable not found")
+ env_token = os.environ["HF_TOKEN"]
+ repo = Repo.clone_from(repo_url.replace("https://", f"https://{script.hf_user}:{env_token}@"), repo_path)
+
except Exception as e:
logger.error(f"Error creating or cloning repo: {e}")
- script.status = f"Error creating or cloning repo: {e}"
+ update_status(script, f"Error creating or cloning repo: {e}")
raise
- script.status = "Repo ready"
+ update_status(script, "Repo ready")
return repo
def git_push_files(repo_path, commit_message, script):
- script.status = "Pushing changes..."
+ update_status(script, "Pushing changes...")
try:
repo = Repo(repo_path)
repo.git.add(all=True)
repo.index.commit(commit_message)
origin = repo.remote(name='origin')
- origin.push()
+ use_git_credential_store = shared.opts.data.get("git_credential_store", True)
+ if use_git_credential_store:
+ origin.push()
+ else:
+ if "HF_TOKEN" not in os.environ:
+ update_status(script, "HF_TOKEN environment variable not found")
+ raise Exception("HF_TOKEN environment variable not found")
+ env_token = os.environ["HF_TOKEN"]
+ origin.push(f"https://{script.hf_user}:{env_token}@huggingface.co/{script.hf_user}/{REPO_NAME}")
+
logger.info(f"Changes pushed successfully to remote repository.")
- script.status = "Pushing Complete"
+ update_status(script, "Pushing Complete")
except Exception as e:
logger.error(f"Git push failed: {e}")
- script.status = f"Git push failed: {e}"
+ update_status(script, f"Git push failed: {e}")
raise
# --- Backup Logic ---
def backup_files(paths, hf_client, script):
logger.info("Starting backup...")
- script.status = "Starting Backup..."
+ update_status(script, "Starting Backup...")
repo_id = script.hf_user + "/" + REPO_NAME
repo_path = os.path.join(script.basedir, 'backup')
sd_path = script.sd_path
@@ -87,31 +111,24 @@ def backup_files(paths, hf_client, script):
repo_file_path = os.path.relpath(local_file_path, start=sd_path)
try:
os.makedirs(os.path.dirname(os.path.join(repo_path, repo_file_path)), exist_ok=True)
- subprocess.run(["cp", local_file_path, os.path.join(repo_path, repo_file_path)], check=True, capture_output=True)
+ shutil.copy2(local_file_path, os.path.join(repo_path, repo_file_path))
logger.info(f"Copied: {repo_file_path}")
- script.status = f"Copied: {repo_file_path}"
+ update_status(script, "Copied", repo_file_path)
except Exception as e:
logger.error(f"Error copying {repo_file_path}: {e}")
- script.status = f"Error copying: {repo_file_path}: {e}"
+ update_status(script, f"Error copying: {repo_file_path}: {e}")
return
try:
git_push_files(repo_path, f"Backup at {datetime.datetime.now()}", script)
logger.info("Backup complete")
- script.status = "Backup Complete"
+ update_status(script, "Backup Complete")
except Exception as e:
logger.error("Error pushing to the repo: ", e)
return
def start_backup_thread(script):
- threading.Thread(target=backup_loop, args=(script,), daemon=True).start()
-
-def backup_loop(script):
- try:
- backup_files(script.backup_paths, None, script)
- except Exception as e:
- print(f"Backup error: {e}")
- time.sleep(BACKUP_INTERVAL)
+ threading.Thread(target=backup_files, args=(script.backup_paths, None, script), daemon=True).start()
# Gradio UI Setup
def on_ui(script):
@@ -165,5 +182,6 @@ def on_script_load(script):
script.hf_user = script.load().get(HF_USER_KEY, '')
script.status = "Not running"
+
script_callbacks.on_ui_tabs(on_ui)
script_callbacks.on_script_load(on_script_load)
From dbaf43b26831e263408851b9d74c3bdb836c1aec Mon Sep 17 00:00:00 2001
From: The Duskfall Portal Crew
<58930427+duskfallcrew@users.noreply.github.com>
Date: Thu, 26 Dec 2024 18:49:43 +1300
Subject: [PATCH 07/31] Delete index.js
---
index.js | 32 --------------------------------
1 file changed, 32 deletions(-)
delete mode 100644 index.js
diff --git a/index.js b/index.js
deleted file mode 100644
index 474ac43..0000000
--- a/index.js
+++ /dev/null
@@ -1,32 +0,0 @@
-const uploadComponent = {
- template: `
-
-
-
-
-
-
-
- `,
- script: () => {
- const hfUsernameInput = document.getElementById('hf-username');
- const hfRepoInput = document.getElementById('hf-repo');
- const writeKeyInput = document.getElementById('write-key');
- const ckptFilesInput = document.getElementById('ckpt-files');
- const uploadBtn = document.getElementById('upload-btn');
-
- uploadBtn.addEventListener('click', async () => {
- const hfUsername = hfUsernameInput.value;
- const hfRepo = hfRepoInput.value;
- const writeKey = writeKeyInput.value;
- const ckptFiles = ckptFilesInput.files;
-
- try {
- const response = await api.uploadToHuggingFaceHub(hfUsername, hfRepo, writeKey, ckptFiles);
- console.log(response);
- } catch (error) {
- console.error(error);
- }
- });
- }
-};
From eb1d514216109ec7d72b530f576faafe69434104 Mon Sep 17 00:00:00 2001
From: The Duskfall Portal Crew
<58930427+duskfallcrew@users.noreply.github.com>
Date: Thu, 26 Dec 2024 18:50:49 +1300
Subject: [PATCH 08/31] Delete style.css
---
style.css | 7 -------
1 file changed, 7 deletions(-)
delete mode 100644 style.css
diff --git a/style.css b/style.css
deleted file mode 100644
index a0cb271..0000000
--- a/style.css
+++ /dev/null
@@ -1,7 +0,0 @@
-# style.css
-.hf-uploader-container {
- margin: 1rem;
- padding: 1rem;
- border-radius: 8px;
- background: var(--background-fill-primary);
-}
From 17e55458b67ef6da0f144307490a0f264cce2471 Mon Sep 17 00:00:00 2001
From: The Duskfall Portal Crew
<58930427+duskfallcrew@users.noreply.github.com>
Date: Thu, 26 Dec 2024 18:52:36 +1300
Subject: [PATCH 09/31] Update requirements.txt
---
requirements.txt | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/requirements.txt b/requirements.txt
index ea60cec..b79594f 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,2 +1,2 @@
huggingface_hub==4.30.2
-glob2
+gitpython
From ba2f0833b47d40eac61c12aed9de115a77613712 Mon Sep 17 00:00:00 2001
From: The Duskfall Portal Crew
<58930427+duskfallcrew@users.noreply.github.com>
Date: Thu, 26 Dec 2024 18:53:58 +1300
Subject: [PATCH 10/31] Update README.md
---
README.md | 104 +++++++++++++++++++++++++++++++-----------------------
1 file changed, 60 insertions(+), 44 deletions(-)
diff --git a/README.md b/README.md
index f9b85c5..c2476bc 100644
--- a/README.md
+++ b/README.md
@@ -1,78 +1,94 @@
-# **Hugging Face Hub Uploader Plugin for Automatic1111**
+# Hugging Face Backup Extension for Stable Diffusion WebUI
-Welcome to our peculiar plugin repository! This project is a unique extension of our Jupyter notebook editions, curated by a diverse DID system with a passion for experimentation. We affectionately call it "opinionated," reflecting our unconventional approach.
+Welcome to our unique extension, designed to help you easily back up your valuable Stable Diffusion WebUI files to the Hugging Face Hub! This project is brought to you by the Duskfall Portal Crew, a diverse DID system passionate about creativity and AI.
+## About This Extension
+This extension provides a simple way to back up your Stable Diffusion WebUI models, embeddings, and other important files to a repository on the Hugging Face Hub. We prioritize ease of use and reliability, helping you safeguard your valuable work.
-## **About this Plugin**
+## Key Features
-This plugin enables seamless uploading of checkpoint files from Automatic1111 to the Hugging Face Hub, a popular platform for machine learning model sharing and collaboration. Our primary goal is to facilitate quick and easy uploads, while prioritizing simplicity and accessibility.
+* **Easy Backup:** Back up your model files, VAEs, embeddings, and LoRAs directly from your Stable Diffusion WebUI.
+* **Hugging Face Integration:** Seamlessly upload your files to your Hugging Face Hub repository.
+* **Automatic Backup (Manual Start):** Backups are performed automatically in the background, after you manually start them using the UI.
+* **Granular Status Updates:** Clear progress information provided during the backup process.
+* **Credential Store:** The extension uses the git credential store for secure authentication, and you can opt-out by using an environment variable.
-## **Features**
+## Getting Started
-* Upload checkpoint files from Automatic1111 to the Hugging Face Hub
-* Supports multiple file uploads and batch processing
-* Automatically generates a PR on the Hugging Face Hub for easy model sharing and collaboration
+1. **Install:** Install this extension by placing it in the `extensions` folder of your Stable Diffusion WebUI.
+2. **Configure:**
+ * In the Stable Diffusion WebUI settings, go to the `Hugging Face` section and set your write access token.
+ * In the extension's UI tab, configure your Hugging Face username, repository name, paths to back up, and the SD Webui folder.
+3. **Start Backup:** Click the "Start Backup" button to begin backing up your files.
-## **Getting Started**
+## Requirements
-1. Install the plugin by following the instructions in the Automatic1111 plugin repository.
-2. Configure your Hugging Face Hub credentials and repository settings.
-3. Select the checkpoint files you want to upload and click the "Upload to Hugging Face Hub" button.
+* Stable Diffusion WebUI (Automatic1111)
+* Hugging Face Hub account and a write access token.
+* Python 3.7 or later.
-## **Requirements**
+## How it Works
-* Automatic1111 - At this stage we're not sure which version this works with but we're testing it as we go.
-* Hugging Face Hub account and credentials - Instructions will come soon on this.
-* Python 3.7 or later
+1. **Configuration:** When you start the extension, it will load the required settings.
+2. **Cloning or Creation:** The extension will clone the provided Hugging Face repository, or create it if it doesn't exist.
+3. **Copy Files:** The files in the specified paths will be copied to the cloned repository.
+4. **Pushing:** The changes will be pushed to your repository in the Hugging Face Hub.
+5. **Scheduled backups** By default, backups are done when the user triggers them, and then after a specified interval, you can configure that interval by modifying the `BACKUP_INTERVAL` constant in your `hfbackup_script.py` file, or by implementing a scheduler.
-## **License**
+## Settings
-This plugin is licensed under the MIT License.
+### Hugging Face Settings
+* **Hugging Face Write API Key:** Required to upload to your Hugging Face Repository.
+* **Use Git Credential Store:** By default the extension will try to use your system's git credentials, but you can disable this behavior by turning this toggle off, and use the environment variable `HF_TOKEN` to provide the token.
-## **Acknowledgments**
+### Extension settings
+* **Huggingface Token**: Your Huggingface token.
+* **Huggingface Username:** Your Huggingface username or organization name.
+* **SD Webui Path:** The folder where Stable Diffusion Webui is installed.
+* **Backup Paths:** The paths to your models, embeddings, etc. that you wish to back up (one path per line), it must be relative to the root folder where Stable Diffusion is installed.
-This plugin builds upon the work of the Automatic1111 community and the Hugging Face Hub team. We appreciate their efforts in creating a robust and scalable platform for machine learning model sharing and collaboration.
+## License
-## **Support**
+This extension is licensed under the MIT License.
-If you encounter any issues or have questions about this plugin, please open an issue in the Automatic1111 plugin repository or reach out to the developer directly.
+## Acknowledgments
-## **Issues**
+This extension is built with the help of the Automatic1111 and Hugging Face communities. We are grateful for their efforts in creating such amazing and useful projects.
-Currently there is no "BACK END" settings file, so right now it auto just yeets the write API key to the tab.
+## Support
-Currently struggling with something, it is currently not liking me but it's at least showing up for once.
+If you encounter any issues or have questions about this extension, please open an issue in our GitHub repository.
-## **Changelog**
+## Known Issues
-* Initial release: [Insert date]
-* Pre Alpha: June 8 2024 - "NOT OFFICIALLY A RELEASE"
-* November 1 - 2024: Rejigged the code via Claude, will test asap and add feature logs.
-* November 10 - Tested it on A111, it's SEMI working -- I just uhhh need to poke at it more? - Right now i'ts doing the WHOLE folder rather than allowing to select, so i'll be fixing that next.
+* No "BACK END" settings file: Settings are saved in the script and are loaded directly by A1111.
+* Currently, the backup occurs only when the user clicks the "Start Backup" button, and then on a timer, until the extension stops.
+
+## Changelog
+
+* **Initial release:** *June 8 2024*
+* **Pre Alpha:** *June 8 2024* - "NOT OFFICIALLY A RELEASE"
+* **Rejig:** *November 1 2024* - Rejigged the code via Claude, will test asap and add feature logs.
+* **Semi Working:** *November 10 2024* - Tested it on A1111, it's SEMI working
## About & Links
### About Us
-We are the Duskfall Portal Crew, a DID system with over 300 alters, navigating life with DID, ADHD, Autism, and CPTSD. We believe in AI’s potential to break down barriers and enhance mental health, despite its challenges. Join us on our creative journey exploring identity and expression.
+We are the Duskfall Portal Crew, a DID system with over 300 alters, navigating life with DID, ADHD, Autism, and CPTSD. We believe in AI’s potential to break down barriers and enhance mental health.
#### Join Our Community
-Website: [End Media](https://end-media.org/) WEBSITE UNDER CONSTRUCTION LOOKING FOR SPONSORS
-
-Discord: [Join our Discord](https://discord.gg/5t2kYxt7An)
-
-Backups: [Hugging Face](https://huggingface.co/EarthnDusk)
-
-Support Us: [Send a Pizza](https://ko-fi.com/duskfallcrew/)
-
-Coffee: [BuyMeSomeMochas!](https://www.buymeacoffee.com/duskfallxcrew)
-
-Patreon: [Our Barely Used Patreon](https://www.patreon.com/earthndusk)
+* **Website:** [End Media](https://end-media.org/) (WEBSITE UNDER CONSTRUCTION)
+* **Discord:** [Join our Discord](https://discord.gg/5t2kYxt7An)
+* **Hugging Face:** [Hugging Face](https://huggingface.co/EarthnDusk)
+* **Support Us:** [Send a Pizza](https://ko-fi.com/duskfallcrew/)
+* **Coffee:** [BuyMeSomeMochas!](https://www.buymeacoffee.com/duskfallxcrew)
+* **Patreon:** [Our Barely Used Patreon](https://www.patreon.com/earthndusk)
-Community Groups:
+#### Community Groups
-Subreddit: [Reddit](https://www.reddit.com/r/earthndusk/)
+* **Subreddit:** [Reddit](https://www.reddit.com/r/earthndusk/)
From 27eeb3cc5280504ada9c6f38c0a003b7b1ad8591 Mon Sep 17 00:00:00 2001
From: John6666 <186692226+John6666cat@users.noreply.github.com>
Date: Fri, 27 Dec 2024 00:19:22 +0900
Subject: [PATCH 11/31] Update requirements.txt
---
requirements.txt | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/requirements.txt b/requirements.txt
index b79594f..6b964cc 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,2 +1 @@
-huggingface_hub==4.30.2
-gitpython
+huggingface_hub
From c19f0e3dae03b5ecc2a7fcefd90e64513d3520bf Mon Sep 17 00:00:00 2001
From: John6666 <186692226+John6666cat@users.noreply.github.com>
Date: Fri, 27 Dec 2024 00:19:39 +0900
Subject: [PATCH 12/31] Update __init__.py
---
__init__.py | 17 -----------------
1 file changed, 17 deletions(-)
diff --git a/__init__.py b/__init__.py
index eb241bd..8b13789 100644
--- a/__init__.py
+++ b/__init__.py
@@ -1,18 +1 @@
-from modules import scripts
-from scripts import hfbackup_script
-class Script(scripts.Script):
- def title(self):
- return "Huggingface Backup"
-
- def show(self, is_img2img):
- return scripts.AlwaysVisible
-
- def ui(self, is_img2img):
- return hfbackup_script.on_ui(self)
-
- def run(self, p, *args):
- return hfbackup_script.on_run(self, p, *args)
-
- def on_script_load(self):
- return hfbackup_script.on_script_load(self)
From dec54794fac5ac68655e5c389ce4bd7751f46cd5 Mon Sep 17 00:00:00 2001
From: John6666 <186692226+John6666cat@users.noreply.github.com>
Date: Fri, 27 Dec 2024 00:20:07 +0900
Subject: [PATCH 13/31] Update hfbackup_script.py
---
scripts/hfbackup_script.py | 250 ++++++++++++++++++++++---------------
1 file changed, 147 insertions(+), 103 deletions(-)
diff --git a/scripts/hfbackup_script.py b/scripts/hfbackup_script.py
index 9ab710d..7cfb8b2 100644
--- a/scripts/hfbackup_script.py
+++ b/scripts/hfbackup_script.py
@@ -5,9 +5,12 @@
import gradio as gr
import subprocess
import logging
-from modules import script_callbacks, shared
-from git import Repo
+from modules import scripts, script_callbacks
+from modules.scripts import basedir
+from huggingface_hub import HfApi, HfFolder
import shutil
+from pathlib import Path
+import hashlib
# Constants
REPO_NAME = 'sd-webui-backups'
@@ -35,74 +38,108 @@ def update_status(script, status, file=None):
script.status = status
print(status) # For console logging
-# --- Git Related Functions ---
-def clone_or_create_repo(repo_url, repo_path, script):
+# --- HfApi Related Functions ---
+def get_hf_token():
+ if script.hf_token: hf_token = script.hf_token
+ elif HfFolder.get_token(): hf_token = HfFolder.get_token()
+ else: hf_token = os.getenv("HF_TOKEN")
+ if not hf_token:
+ update_status(script, "HF_TOKEN environment variable not found")
+ raise Exception("HF_TOKEN environment variable not found")
+ return hf_token
+
+def clone_or_create_repo(repo_id: str, repo_type: str, repo_path: str, script):
update_status(script, "Checking/Cloning Repo...")
- if os.path.exists(repo_path) and os.path.isdir(repo_path):
- logger.info(f"Repository already exists at {repo_path}, updating...")
- repo = Repo(repo_path)
- if repo.is_dirty():
- logger.warning("Local repo has uncommitted changes. Commit those before running to make sure nothing breaks.")
- update_status(script, "Local repo has uncommitted changes")
- else:
- logger.info(f"Cloning repository from {repo_url} to {repo_path}")
- update_status(script, "Cloning repository")
- try:
- use_git_credential_store = shared.opts.data.get("git_credential_store", True)
- if use_git_credential_store:
- repo = Repo.clone_from(repo_url, repo_path)
- else:
- if "HF_TOKEN" not in os.environ:
- update_status(script, "HF_TOKEN environment variable not found")
- raise Exception("HF_TOKEN environment variable not found")
- env_token = os.environ["HF_TOKEN"]
- repo = Repo.clone_from(repo_url.replace("https://", f"https://{script.hf_user}:{env_token}@"), repo_path)
-
- except Exception as e:
- logger.error(f"Error creating or cloning repo: {e}")
- update_status(script, f"Error creating or cloning repo: {e}")
- raise
+ logger.info(f"Cloning repository from {repo_id} to {repo_path}")
+ update_status(script, "Cloning repository")
+ try:
+ hf_token = get_hf_token()
+ api = HfApi(token=hf_token)
+ if os.path.exists(repo_path) and os.path.isdir(repo_path):
+ logger.info(f"Repository already exists at {repo_path}, updating...")
+ ignore_paths = get_ingore_paths(repo_path, repo_id, repo_type, hf_token)
+ else:
+ os.makedirs(repo_path, exist_ok=True)
+ ignore_paths = []
+ if api.repo_exists(repo_id=repo_id, repo_type=repo_type, token=hf_token):
+ api.snapshot_download(repo_id=repo_id, repo_type=repo_type, local_dir=repo_path, ignore_patterns=ignore_paths, token=hf_token)
+ except Exception as e:
+ logger.error(f"Error creating or cloning repo: {e}")
+ update_status(script, f"Error creating or cloning repo: {e}")
+ raise
update_status(script, "Repo ready")
- return repo
-def git_push_files(repo_path, commit_message, script):
+def get_path_in_repo(path: str):
+ return str(Path(path)).replace("\\", "/")
+
+def get_sha256(filename: str):
+ if not Path(filename).exists(): return None
+ sha256_hash = hashlib.sha256()
+ with open(filename, "rb") as f:
+ for byte_block in iter(lambda: f.read(4096), b""):
+ sha256_hash.update(byte_block)
+ return sha256_hash.hexdigest()
+
+def is_same_file(filename: str, dst_repo: str, dst_type: str, dst_path: str, hf_token: str):
+ api = HfApi(token=hf_token)
+ if not filename or not Path(filename).exists() or Path(filename).is_dir(): return False
+ dst_path = get_path_in_repo(dst_path)
+ if not api.file_exists(repo_id=dst_repo, filename=dst_path, repo_type=dst_type, token=hf_token): return False
+ src_sha256 = get_sha256(filename)
+ src_size = os.path.getsize(filename)
+ dst_info = api.get_paths_info(repo_id=dst_repo, paths=dst_path, repo_type=dst_type, token=hf_token)
+ if not dst_info or len(dst_info) != 1 or dst_info[0].lfs is None: return False
+ if src_size == dst_info[0].size and src_sha256 == dst_info[0].lfs.sha256: return True
+ else: return False
+
+def get_ingore_paths(path: str, repo_id: str, repo_type: str, hf_token: str):
+ ignores = []
+ for p in Path(path).glob("**/*"):
+ if p.is_dir(): continue
+ rp = p.resolve().relative_to(Path(path).resolve())
+ if is_same_file(str(p), repo_id, repo_type, str(rp), hf_token): ignores.append(get_path_in_repo(str(rp)))
+ print(f"These files are already latest: {', '.join(ignores)}") # debug
+ return ignores
+
+def safe_copy(src: str, dst: str, script):
+ if not Path(src).exists(): return
+ if Path(dst).exists() and os.path.getsize(src) == os.path.getsize(dst) and get_sha256(src) == get_sha256(dst):
+ logger.info(f"Skipped: {src}")
+ update_status(script, "Skipped", src)
+ return
+ shutil.copy2(src, dst)
+ logger.info(f"Copied: {src}")
+ update_status(script, "Copied", src)
+
+def hf_push_files(repo_id: str, repo_type: str, repo_path: str, commit_message: str, script):
update_status(script, "Pushing changes...")
try:
- repo = Repo(repo_path)
- repo.git.add(all=True)
- repo.index.commit(commit_message)
- origin = repo.remote(name='origin')
- use_git_credential_store = shared.opts.data.get("git_credential_store", True)
- if use_git_credential_store:
- origin.push()
- else:
- if "HF_TOKEN" not in os.environ:
- update_status(script, "HF_TOKEN environment variable not found")
- raise Exception("HF_TOKEN environment variable not found")
- env_token = os.environ["HF_TOKEN"]
- origin.push(f"https://{script.hf_user}:{env_token}@huggingface.co/{script.hf_user}/{REPO_NAME}")
-
+ hf_token = get_hf_token()
+ api = HfApi(token=hf_token)
+ ignore_paths = get_ingore_paths(repo_path, repo_id, repo_type, hf_token)
+ api.create_repo(repo_id=repo_id, repo_type=repo_type, exist_ok=True, token=hf_token)
+ api.upload_folder(repo_id=repo_id, repo_type=repo_type, folder_path=repo_path, path_in_repo="",
+ ignore_patterns=ignore_paths, commit_message=commit_message, token=hf_token)
logger.info(f"Changes pushed successfully to remote repository.")
update_status(script, "Pushing Complete")
except Exception as e:
- logger.error(f"Git push failed: {e}")
- update_status(script, f"Git push failed: {e}")
- raise
+ logger.error(f"HF push failed: {e}")
+ update_status(script, f"HF push failed: {e}")
+ raise
# --- Backup Logic ---
def backup_files(paths, hf_client, script):
+ repo_type = "model"
logger.info("Starting backup...")
update_status(script, "Starting Backup...")
repo_id = script.hf_user + "/" + REPO_NAME
repo_path = os.path.join(script.basedir, 'backup')
sd_path = script.sd_path
-
try:
- repo = clone_or_create_repo(f"https://huggingface.co/{repo_id}", repo_path, script)
+ clone_or_create_repo(repo_id, repo_type, repo_path, script)
except Exception as e:
logger.error("Error starting the backup, please see the traceback.")
return
-
for base_path in paths:
logger.info(f"Backing up: {base_path}")
for root, _, files in os.walk(os.path.join(sd_path, base_path)):
@@ -111,77 +148,84 @@ def backup_files(paths, hf_client, script):
repo_file_path = os.path.relpath(local_file_path, start=sd_path)
try:
os.makedirs(os.path.dirname(os.path.join(repo_path, repo_file_path)), exist_ok=True)
- shutil.copy2(local_file_path, os.path.join(repo_path, repo_file_path))
- logger.info(f"Copied: {repo_file_path}")
- update_status(script, "Copied", repo_file_path)
+ safe_copy(local_file_path, os.path.join(repo_path, repo_file_path), script)
except Exception as e:
logger.error(f"Error copying {repo_file_path}: {e}")
update_status(script, f"Error copying: {repo_file_path}: {e}")
return
-
try:
- git_push_files(repo_path, f"Backup at {datetime.datetime.now()}", script)
+ hf_push_files(repo_id, repo_type, repo_path, f"Backup at {datetime.datetime.now()}", script)
logger.info("Backup complete")
update_status(script, "Backup Complete")
except Exception as e:
- logger.error("Error pushing to the repo: ", e)
- return
+ logger.error("Error pushing to the repo: ", e)
+ return
def start_backup_thread(script):
threading.Thread(target=backup_files, args=(script.backup_paths, None, script), daemon=True).start()
# Gradio UI Setup
def on_ui(script):
- with gr.Column():
- with gr.Row():
- with gr.Column(scale=3):
- hf_token_box = gr.Textbox(label="Huggingface Token", type='password', value=script.hf_token)
- def on_token_change(token):
- script.hf_token = token
- script.save()
- hf_token_box.change(on_token_change, inputs=[hf_token_box], outputs=None)
- with gr.Column(scale=1):
- status_box = gr.Textbox(label="Status", value=script.status)
-
- def on_start_button():
- start_backup_thread(script)
- return "Starting Backup"
-
- start_button = gr.Button(value="Start Backup")
- start_button.click(on_start_button, inputs=None, outputs=[status_box])
-
- with gr.Row():
- with gr.Column():
- sd_path_box = gr.Textbox(label="SD Webui Path", value=script.sd_path)
- def on_sd_path_change(path):
+ with gr.Blocks() as hf_backup:
+ with gr.Column():
+ with gr.Row():
+ with gr.Column(scale=3):
+ hf_token_box = gr.Textbox(label="Huggingface Token", type='password', value=script.hf_token)
+ def on_token_change(token):
+ script.hf_token = token
+ hf_token_box.change(on_token_change, inputs=[hf_token_box], outputs=None)
+ with gr.Column(scale=1):
+ status_box = gr.Textbox(label="Status", value=script.status)
+ def on_start_button():
+ start_backup_thread(script)
+ return "Starting Backup"
+ start_button = gr.Button(value="Start Backup")
+ start_button.click(on_start_button, inputs=None, outputs=[status_box])
+ with gr.Row():
+ with gr.Column():
+ sd_path_box = gr.Textbox(label="SD Webui Path", value=script.sd_path)
+ def on_sd_path_change(path):
script.sd_path = path
- script.save()
- sd_path_box.change(on_sd_path_change, inputs=[sd_path_box], outputs=None)
- with gr.Column():
- hf_user_box = gr.Textbox(label="Huggingface Username", value=script.hf_user)
- def on_hf_user_change(user):
+ sd_path_box.change(on_sd_path_change, inputs=[sd_path_box], outputs=None)
+ with gr.Column():
+ hf_user_box = gr.Textbox(label="Huggingface Username", value=script.hf_user)
+ def on_hf_user_change(user):
script.hf_user = user
- script.save()
- hf_user_box.change(on_hf_user_change, inputs=[hf_user_box], outputs=None)
- with gr.Row():
- backup_paths_box = gr.Textbox(label="Backup Paths (one path per line)", lines=4, value='\n'.join(script.backup_paths))
- def on_backup_paths_change(paths):
- paths_list = paths.split('\n')
- paths_list = [p.strip() for p in paths_list if p.strip()]
- script.backup_paths = paths_list
- script.save()
- backup_paths_box.change(on_backup_paths_change, inputs=[backup_paths_box], outputs=None)
+ hf_user_box.change(on_hf_user_change, inputs=[hf_user_box], outputs=None)
+ with gr.Row():
+ backup_paths_box = gr.Textbox(label="Backup Paths (one path per line)", lines=4, value='\n'.join(script.backup_paths) if isinstance(script.backup_paths, list) else "")
+ def on_backup_paths_change(paths):
+ paths_list = [p.strip() for p in paths.split('\n') if p.strip()]
+ script.backup_paths = paths_list
+ backup_paths_box.change(on_backup_paths_change, inputs=[backup_paths_box], outputs=None)
+ return [(hf_backup, "Huggingface Backup", "hfbackup_script")]
def on_run(script, p, *args):
pass
-
-def on_script_load(script):
- script.hf_token = script.load().get(HF_TOKEN_KEY, '')
- script.backup_paths = script.load().get(BACKUP_PATHS_KEY, DEFAULT_BACKUP_PATHS)
- script.sd_path = script.load().get(SD_PATH_KEY, '')
- script.hf_user = script.load().get(HF_USER_KEY, '')
- script.status = "Not running"
+class Script():
+ env = {}
+
+ def __init__(self):
+ self.hf_token = self.env.get(HF_TOKEN_KEY, "")
+ self.backup_paths = self.env.get(BACKUP_PATHS_KEY, DEFAULT_BACKUP_PATHS)
+ self.sd_path = self.env.get(SD_PATH_KEY, "")
+ self.hf_user = self.env.get(HF_USER_KEY, "")
+ self.status = "Not running"
+ self.basedir = basedir()
+
+ def title(self):
+ return "Huggingface Backup"
+
+ def show(self, is_img2img=None):
+ return scripts.AlwaysVisible
+
+ def ui(self, is_img2img=None):
+ return on_ui(self)
+
+ def run(self, p, *args):
+ return on_run(self, p, *args)
-script_callbacks.on_ui_tabs(on_ui)
-script_callbacks.on_script_load(on_script_load)
+if __package__ == "hfbackup_script":
+ script = Script()
+ script_callbacks.on_ui_tabs(script.ui)
From 374191cced7625039357cddf04115148f83b5f27 Mon Sep 17 00:00:00 2001
From: John6666 <186692226+John6666cat@users.noreply.github.com>
Date: Fri, 27 Dec 2024 00:20:55 +0900
Subject: [PATCH 14/31] Update ui-settings.py
---
scripts/ui-settings.py | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/scripts/ui-settings.py b/scripts/ui-settings.py
index dc0a358..d213fe0 100644
--- a/scripts/ui-settings.py
+++ b/scripts/ui-settings.py
@@ -8,8 +8,8 @@ def on_ui_settings():
shared.OptionInfo(
"",
"Hugging Face Write API Key",
- gr.Password,
- {"interactive": True},
+ gr.Textbox,
+ {"interactive": True, "type": "password", "lines": 1},
section=section
)
)
@@ -18,8 +18,8 @@ def on_ui_settings():
shared.OptionInfo(
"",
"Hugging Face Read API Key",
- gr.Password,
- {"interactive": True},
+ gr.Textbox,
+ {"interactive": True, "type": "password", "lines": 1},
section=section
)
)
From 066cf2e179119d5566bb3fecfe655d9a5b06bfdb Mon Sep 17 00:00:00 2001
From: John6666 <186692226+John6666cat@users.noreply.github.com>
Date: Fri, 27 Dec 2024 09:28:57 +0900
Subject: [PATCH 15/31] Update requirements.txt
---
requirements.txt | 1 +
1 file changed, 1 insertion(+)
diff --git a/requirements.txt b/requirements.txt
index 6b964cc..4b3550a 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1 +1,2 @@
huggingface_hub
+apscheduler
From ed674c8ef666b645726e52c824a88204926f89fe Mon Sep 17 00:00:00 2001
From: John6666 <186692226+John6666cat@users.noreply.github.com>
Date: Fri, 27 Dec 2024 09:29:18 +0900
Subject: [PATCH 16/31] Update ui-settings.py
---
scripts/ui-settings.py | 20 --------------------
1 file changed, 20 deletions(-)
diff --git a/scripts/ui-settings.py b/scripts/ui-settings.py
index d213fe0..9b7687d 100644
--- a/scripts/ui-settings.py
+++ b/scripts/ui-settings.py
@@ -13,25 +13,5 @@ def on_ui_settings():
section=section
)
)
- shared.opts.add_option(
- "hf_read_key",
- shared.OptionInfo(
- "",
- "Hugging Face Read API Key",
- gr.Textbox,
- {"interactive": True, "type": "password", "lines": 1},
- section=section
- )
- )
- shared.opts.add_option(
- "git_credential_store",
- shared.OptionInfo(
- True,
- "Use Git Credential Store",
- gr.Checkbox,
- {"interactive": True},
- section=section
- )
- )
script_callbacks.on_ui_settings(on_ui_settings)
From 6266d4c62f8f9352f6446dad116fd24350c884e2 Mon Sep 17 00:00:00 2001
From: John6666 <186692226+John6666cat@users.noreply.github.com>
Date: Fri, 27 Dec 2024 09:30:40 +0900
Subject: [PATCH 17/31] Update hfbackup_script.py
---
scripts/hfbackup_script.py | 58 +++++++++++++++++++++++---------------
1 file changed, 35 insertions(+), 23 deletions(-)
diff --git a/scripts/hfbackup_script.py b/scripts/hfbackup_script.py
index 7cfb8b2..6b129cb 100644
--- a/scripts/hfbackup_script.py
+++ b/scripts/hfbackup_script.py
@@ -5,12 +5,14 @@
import gradio as gr
import subprocess
import logging
-from modules import scripts, script_callbacks
+from modules import scripts, script_callbacks, shared
from modules.scripts import basedir
+from modules import paths
from huggingface_hub import HfApi, HfFolder
import shutil
from pathlib import Path
import hashlib
+from apscheduler.schedulers.background import BackgroundScheduler
# Constants
REPO_NAME = 'sd-webui-backups'
@@ -39,8 +41,9 @@ def update_status(script, status, file=None):
print(status) # For console logging
# --- HfApi Related Functions ---
-def get_hf_token():
- if script.hf_token: hf_token = script.hf_token
+def get_hf_token(script):
+ if shared.opts.hf_write_key: hf_token = shared.opts.hf_write_key
+ elif script.hf_token: hf_token = script.hf_token
elif HfFolder.get_token(): hf_token = HfFolder.get_token()
else: hf_token = os.getenv("HF_TOKEN")
if not hf_token:
@@ -48,12 +51,19 @@ def get_hf_token():
raise Exception("HF_TOKEN environment variable not found")
return hf_token
+def get_hf_user(script):
+ if script.hf_user: return script.hf_user
+ hf_token = get_hf_token(script)
+ api = HfApi(token=hf_token)
+ whoami = api.whoami(token=hf_token)
+ return whoami.get("name", "") if isinstance(whoami, dict) else ""
+
def clone_or_create_repo(repo_id: str, repo_type: str, repo_path: str, script):
update_status(script, "Checking/Cloning Repo...")
logger.info(f"Cloning repository from {repo_id} to {repo_path}")
update_status(script, "Cloning repository")
try:
- hf_token = get_hf_token()
+ hf_token = get_hf_token(script)
api = HfApi(token=hf_token)
if os.path.exists(repo_path) and os.path.isdir(repo_path):
logger.info(f"Repository already exists at {repo_path}, updating...")
@@ -98,7 +108,7 @@ def get_ingore_paths(path: str, repo_id: str, repo_type: str, hf_token: str):
if p.is_dir(): continue
rp = p.resolve().relative_to(Path(path).resolve())
if is_same_file(str(p), repo_id, repo_type, str(rp), hf_token): ignores.append(get_path_in_repo(str(rp)))
- print(f"These files are already latest: {', '.join(ignores)}") # debug
+ if len(ignores) != 0: print(f"These files are already latest: {', '.join(ignores)}") # debug
return ignores
def safe_copy(src: str, dst: str, script):
@@ -114,7 +124,7 @@ def safe_copy(src: str, dst: str, script):
def hf_push_files(repo_id: str, repo_type: str, repo_path: str, commit_message: str, script):
update_status(script, "Pushing changes...")
try:
- hf_token = get_hf_token()
+ hf_token = get_hf_token(script)
api = HfApi(token=hf_token)
ignore_paths = get_ingore_paths(repo_path, repo_id, repo_type, hf_token)
api.create_repo(repo_id=repo_id, repo_type=repo_type, exist_ok=True, token=hf_token)
@@ -128,19 +138,19 @@ def hf_push_files(repo_id: str, repo_type: str, repo_path: str, commit_message:
raise
# --- Backup Logic ---
-def backup_files(paths, hf_client, script):
+def backup_files(backup_paths, script):
repo_type = "model"
logger.info("Starting backup...")
update_status(script, "Starting Backup...")
- repo_id = script.hf_user + "/" + REPO_NAME
+ repo_id = get_hf_user(script) + "/" + REPO_NAME
repo_path = os.path.join(script.basedir, 'backup')
- sd_path = script.sd_path
+ sd_path = script.sd_path if script.sd_path else paths.data_path
try:
clone_or_create_repo(repo_id, repo_type, repo_path, script)
except Exception as e:
logger.error("Error starting the backup, please see the traceback.")
return
- for base_path in paths:
+ for base_path in backup_paths:
logger.info(f"Backing up: {base_path}")
for root, _, files in os.walk(os.path.join(sd_path, base_path)):
for file in files:
@@ -162,11 +172,13 @@ def backup_files(paths, hf_client, script):
return
def start_backup_thread(script):
- threading.Thread(target=backup_files, args=(script.backup_paths, None, script), daemon=True).start()
+ backup_files(script.backup_paths, script)
+ script.update_schedule()
+ #threading.Thread(target=backup_files, args=(script.backup_paths, script), daemon=True).start()
# Gradio UI Setup
def on_ui(script):
- with gr.Blocks() as hf_backup:
+ with gr.Blocks(analytics_enabled=False) as hf_backup:
with gr.Column():
with gr.Row():
with gr.Column(scale=3):
@@ -176,7 +188,7 @@ def on_token_change(token):
hf_token_box.change(on_token_change, inputs=[hf_token_box], outputs=None)
with gr.Column(scale=1):
status_box = gr.Textbox(label="Status", value=script.status)
- def on_start_button():
+ def on_start_button(progress=gr.Progress(track_tqdm=True)):
start_backup_thread(script)
return "Starting Backup"
start_button = gr.Button(value="Start Backup")
@@ -200,10 +212,7 @@ def on_backup_paths_change(paths):
backup_paths_box.change(on_backup_paths_change, inputs=[backup_paths_box], outputs=None)
return [(hf_backup, "Huggingface Backup", "hfbackup_script")]
-def on_run(script, p, *args):
- pass
-
-class Script():
+class HFBackupScript():
env = {}
def __init__(self):
@@ -213,6 +222,7 @@ def __init__(self):
self.hf_user = self.env.get(HF_USER_KEY, "")
self.status = "Not running"
self.basedir = basedir()
+ self.scheduler = BackgroundScheduler()
def title(self):
return "Huggingface Backup"
@@ -220,12 +230,14 @@ def title(self):
def show(self, is_img2img=None):
return scripts.AlwaysVisible
- def ui(self, is_img2img=None):
+ def on_ui(self, is_img2img=None):
return on_ui(self)
-
- def run(self, p, *args):
- return on_run(self, p, *args)
+
+ def update_schedule(self):
+ self.scheduler.remove_job('backup')
+ self.scheduler.add_job(func=backup_files, args=[self.backup_paths, self], trigger="interval", id="backup", seconds=60)
+ self.scheduler.start()
if __package__ == "hfbackup_script":
- script = Script()
- script_callbacks.on_ui_tabs(script.ui)
+ script = HFBackupScript()
+ script_callbacks.on_ui_tabs(script.on_ui)
From d2ac44af019883553eac18aaefeb184c58c871a2 Mon Sep 17 00:00:00 2001
From: John6666 <186692226+John6666cat@users.noreply.github.com>
Date: Fri, 27 Dec 2024 09:39:06 +0900
Subject: [PATCH 18/31] Add files via upload
---
install.py | 7 +++++++
1 file changed, 7 insertions(+)
create mode 100644 install.py
diff --git a/install.py b/install.py
new file mode 100644
index 0000000..00c9f01
--- /dev/null
+++ b/install.py
@@ -0,0 +1,7 @@
+import launch
+
+if not launch.is_installed("apscheduler"):
+ launch.run_pip("install apscheduler", "requirements for Hugging Face Backup")
+
+if not launch.is_installed("huggingface_hub"):
+ launch.run_pip("install huggingface_hub", "requirements for Hugging Face Backup")
From 335533a41088636e717b09bd3aad7dacb2c8a06e Mon Sep 17 00:00:00 2001
From: John6666 <186692226+John6666cat@users.noreply.github.com>
Date: Fri, 27 Dec 2024 09:43:42 +0900
Subject: [PATCH 19/31] Update hfbackup_script.py
---
scripts/hfbackup_script.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/scripts/hfbackup_script.py b/scripts/hfbackup_script.py
index 6b129cb..3ec299f 100644
--- a/scripts/hfbackup_script.py
+++ b/scripts/hfbackup_script.py
@@ -234,7 +234,7 @@ def on_ui(self, is_img2img=None):
return on_ui(self)
def update_schedule(self):
- self.scheduler.remove_job('backup')
+ if "backup" in self.scheduler.get_jobs(): self.scheduler.remove_job("backup")
self.scheduler.add_job(func=backup_files, args=[self.backup_paths, self], trigger="interval", id="backup", seconds=60)
self.scheduler.start()
From f957b48cd1c9996f56cc241898330f01f43c00d3 Mon Sep 17 00:00:00 2001
From: John6666 <186692226+John6666cat@users.noreply.github.com>
Date: Fri, 27 Dec 2024 09:44:33 +0900
Subject: [PATCH 20/31] Delete __init__.py
---
__init__.py | 1 -
1 file changed, 1 deletion(-)
delete mode 100644 __init__.py
diff --git a/__init__.py b/__init__.py
deleted file mode 100644
index 8b13789..0000000
--- a/__init__.py
+++ /dev/null
@@ -1 +0,0 @@
-
From 89982ff0581665ab9f3f35d4dc41daca07a641a9 Mon Sep 17 00:00:00 2001
From: John6666 <186692226+John6666cat@users.noreply.github.com>
Date: Fri, 27 Dec 2024 09:50:43 +0900
Subject: [PATCH 21/31] Update hfbackup_script.py
---
scripts/hfbackup_script.py | 10 ++++++++--
1 file changed, 8 insertions(+), 2 deletions(-)
diff --git a/scripts/hfbackup_script.py b/scripts/hfbackup_script.py
index 3ec299f..1f9ee95 100644
--- a/scripts/hfbackup_script.py
+++ b/scripts/hfbackup_script.py
@@ -142,7 +142,7 @@ def backup_files(backup_paths, script):
repo_type = "model"
logger.info("Starting backup...")
update_status(script, "Starting Backup...")
- repo_id = get_hf_user(script) + "/" + REPO_NAME
+ repo_id = get_hf_user(script) + "/" + script.hf_repo
repo_path = os.path.join(script.basedir, 'backup')
sd_path = script.sd_path if script.sd_path else paths.data_path
try:
@@ -204,6 +204,11 @@ def on_sd_path_change(path):
def on_hf_user_change(user):
script.hf_user = user
hf_user_box.change(on_hf_user_change, inputs=[hf_user_box], outputs=None)
+ with gr.Column():
+ hf_repo_box = gr.Textbox(label="Huggingface Reponame", value=script.hf_repo)
+ def on_hf_repo_change(repo):
+ script.hf_repo = repo
+ hf_repo_box.change(on_hf_repo_change, inputs=[hf_repo_box], outputs=None)
with gr.Row():
backup_paths_box = gr.Textbox(label="Backup Paths (one path per line)", lines=4, value='\n'.join(script.backup_paths) if isinstance(script.backup_paths, list) else "")
def on_backup_paths_change(paths):
@@ -220,6 +225,7 @@ def __init__(self):
self.backup_paths = self.env.get(BACKUP_PATHS_KEY, DEFAULT_BACKUP_PATHS)
self.sd_path = self.env.get(SD_PATH_KEY, "")
self.hf_user = self.env.get(HF_USER_KEY, "")
+ self.hf_repo = REPO_NAME
self.status = "Not running"
self.basedir = basedir()
self.scheduler = BackgroundScheduler()
@@ -235,7 +241,7 @@ def on_ui(self, is_img2img=None):
def update_schedule(self):
if "backup" in self.scheduler.get_jobs(): self.scheduler.remove_job("backup")
- self.scheduler.add_job(func=backup_files, args=[self.backup_paths, self], trigger="interval", id="backup", seconds=60)
+ self.scheduler.add_job(func=backup_files, args=[self.backup_paths, self], trigger="interval", id="backup", seconds=BACKUP_INTERVAL)
self.scheduler.start()
if __package__ == "hfbackup_script":
From 81340568cc76539eed3985c3003804f5dd4f42f9 Mon Sep 17 00:00:00 2001
From: John6666 <186692226+John6666cat@users.noreply.github.com>
Date: Fri, 27 Dec 2024 10:01:02 +0900
Subject: [PATCH 22/31] Update hfbackup_script.py
---
scripts/hfbackup_script.py | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/scripts/hfbackup_script.py b/scripts/hfbackup_script.py
index 1f9ee95..056ec66 100644
--- a/scripts/hfbackup_script.py
+++ b/scripts/hfbackup_script.py
@@ -240,7 +240,9 @@ def on_ui(self, is_img2img=None):
return on_ui(self)
def update_schedule(self):
- if "backup" in self.scheduler.get_jobs(): self.scheduler.remove_job("backup")
+ if "backup" in self.scheduler.get_jobs():
+ self.scheduler.shutdown(wait=False)
+ self.scheduler.remove_job("backup")
self.scheduler.add_job(func=backup_files, args=[self.backup_paths, self], trigger="interval", id="backup", seconds=BACKUP_INTERVAL)
self.scheduler.start()
From 58d404fa9461688ff281a24c1c001e8800fa727b Mon Sep 17 00:00:00 2001
From: John6666 <186692226+John6666cat@users.noreply.github.com>
Date: Fri, 27 Dec 2024 10:09:23 +0900
Subject: [PATCH 23/31] Update hfbackup_script.py
---
scripts/hfbackup_script.py | 7 +++----
1 file changed, 3 insertions(+), 4 deletions(-)
diff --git a/scripts/hfbackup_script.py b/scripts/hfbackup_script.py
index 056ec66..be18e52 100644
--- a/scripts/hfbackup_script.py
+++ b/scripts/hfbackup_script.py
@@ -240,10 +240,9 @@ def on_ui(self, is_img2img=None):
return on_ui(self)
def update_schedule(self):
- if "backup" in self.scheduler.get_jobs():
- self.scheduler.shutdown(wait=False)
- self.scheduler.remove_job("backup")
- self.scheduler.add_job(func=backup_files, args=[self.backup_paths, self], trigger="interval", id="backup", seconds=BACKUP_INTERVAL)
+ self.scheduler.shutdown(wait=False)
+ if "backup" in self.scheduler.get_jobs(): self.scheduler.reschedule_job(func=backup_files, args=[self.backup_paths, self], trigger="interval", id="backup", seconds=BACKUP_INTERVAL)
+ else: self.scheduler.add_job(func=backup_files, args=[self.backup_paths, self], trigger="interval", id="backup", seconds=BACKUP_INTERVAL)
self.scheduler.start()
if __package__ == "hfbackup_script":
From 6860c2a7cee102619a5c35591d091a0a5345518d Mon Sep 17 00:00:00 2001
From: John6666 <186692226+John6666cat@users.noreply.github.com>
Date: Fri, 27 Dec 2024 10:14:58 +0900
Subject: [PATCH 24/31] Update hfbackup_script.py
---
scripts/hfbackup_script.py | 22 ++++++++++++----------
1 file changed, 12 insertions(+), 10 deletions(-)
diff --git a/scripts/hfbackup_script.py b/scripts/hfbackup_script.py
index be18e52..a1e9716 100644
--- a/scripts/hfbackup_script.py
+++ b/scripts/hfbackup_script.py
@@ -183,35 +183,36 @@ def on_ui(script):
with gr.Row():
with gr.Column(scale=3):
hf_token_box = gr.Textbox(label="Huggingface Token", type='password', value=script.hf_token)
- def on_token_change(token):
+ def on_token_change(token: str):
script.hf_token = token
hf_token_box.change(on_token_change, inputs=[hf_token_box], outputs=None)
with gr.Column(scale=1):
status_box = gr.Textbox(label="Status", value=script.status)
- def on_start_button(progress=gr.Progress(track_tqdm=True)):
- start_backup_thread(script)
+ def on_start_button(is_scheduled: bool, progress=gr.Progress(track_tqdm=True)):
+ start_backup_thread(script, is_scheduled)
return "Starting Backup"
start_button = gr.Button(value="Start Backup")
- start_button.click(on_start_button, inputs=None, outputs=[status_box])
+ start_button.click(on_start_button, inputs=[is_scheduled], outputs=[status_box])
+ is_scheduled = gr.Checkbox(label="Scheduled backups", value=True)
with gr.Row():
with gr.Column():
sd_path_box = gr.Textbox(label="SD Webui Path", value=script.sd_path)
- def on_sd_path_change(path):
+ def on_sd_path_change(path: str):
script.sd_path = path
sd_path_box.change(on_sd_path_change, inputs=[sd_path_box], outputs=None)
with gr.Column():
hf_user_box = gr.Textbox(label="Huggingface Username", value=script.hf_user)
- def on_hf_user_change(user):
+ def on_hf_user_change(user: str):
script.hf_user = user
hf_user_box.change(on_hf_user_change, inputs=[hf_user_box], outputs=None)
with gr.Column():
hf_repo_box = gr.Textbox(label="Huggingface Reponame", value=script.hf_repo)
- def on_hf_repo_change(repo):
+ def on_hf_repo_change(repo: str):
script.hf_repo = repo
hf_repo_box.change(on_hf_repo_change, inputs=[hf_repo_box], outputs=None)
with gr.Row():
backup_paths_box = gr.Textbox(label="Backup Paths (one path per line)", lines=4, value='\n'.join(script.backup_paths) if isinstance(script.backup_paths, list) else "")
- def on_backup_paths_change(paths):
+ def on_backup_paths_change(paths: list):
paths_list = [p.strip() for p in paths.split('\n') if p.strip()]
script.backup_paths = paths_list
backup_paths_box.change(on_backup_paths_change, inputs=[backup_paths_box], outputs=None)
@@ -240,8 +241,9 @@ def on_ui(self, is_img2img=None):
return on_ui(self)
def update_schedule(self):
- self.scheduler.shutdown(wait=False)
- if "backup" in self.scheduler.get_jobs(): self.scheduler.reschedule_job(func=backup_files, args=[self.backup_paths, self], trigger="interval", id="backup", seconds=BACKUP_INTERVAL)
+ if "backup" in self.scheduler.get_jobs():
+ self.scheduler.shutdown(wait=False)
+ self.scheduler.reschedule_job(func=backup_files, args=[self.backup_paths, self], trigger="interval", id="backup", seconds=BACKUP_INTERVAL)
else: self.scheduler.add_job(func=backup_files, args=[self.backup_paths, self], trigger="interval", id="backup", seconds=BACKUP_INTERVAL)
self.scheduler.start()
From 80285820d7b61c965dad388c20845d2df2250784 Mon Sep 17 00:00:00 2001
From: John6666 <186692226+John6666cat@users.noreply.github.com>
Date: Fri, 27 Dec 2024 10:18:52 +0900
Subject: [PATCH 25/31] Update hfbackup_script.py
---
scripts/hfbackup_script.py | 14 ++++++++------
1 file changed, 8 insertions(+), 6 deletions(-)
diff --git a/scripts/hfbackup_script.py b/scripts/hfbackup_script.py
index a1e9716..fe09176 100644
--- a/scripts/hfbackup_script.py
+++ b/scripts/hfbackup_script.py
@@ -171,9 +171,9 @@ def backup_files(backup_paths, script):
logger.error("Error pushing to the repo: ", e)
return
-def start_backup_thread(script):
+def start_backup_thread(script, is_scheduled: bool):
backup_files(script.backup_paths, script)
- script.update_schedule()
+ script.update_schedule(is_scheduled)
#threading.Thread(target=backup_files, args=(script.backup_paths, script), daemon=True).start()
# Gradio UI Setup
@@ -240,12 +240,14 @@ def show(self, is_img2img=None):
def on_ui(self, is_img2img=None):
return on_ui(self)
- def update_schedule(self):
+ def update_schedule(self, is_scheduled: bool):
if "backup" in self.scheduler.get_jobs():
self.scheduler.shutdown(wait=False)
- self.scheduler.reschedule_job(func=backup_files, args=[self.backup_paths, self], trigger="interval", id="backup", seconds=BACKUP_INTERVAL)
- else: self.scheduler.add_job(func=backup_files, args=[self.backup_paths, self], trigger="interval", id="backup", seconds=BACKUP_INTERVAL)
- self.scheduler.start()
+ if is_scheduled:
+ self.scheduler.reschedule_job(func=backup_files, args=[self.backup_paths, self], trigger="interval", id="backup", seconds=BACKUP_INTERVAL)
+ elif is_scheduled:
+ self.scheduler.add_job(func=backup_files, args=[self.backup_paths, self], trigger="interval", id="backup", seconds=BACKUP_INTERVAL)
+ self.scheduler.start()
if __package__ == "hfbackup_script":
script = HFBackupScript()
From 7e05b39813f53cfa10ff2eb9af874b4f63a4878d Mon Sep 17 00:00:00 2001
From: John6666 <186692226+John6666cat@users.noreply.github.com>
Date: Fri, 27 Dec 2024 10:26:08 +0900
Subject: [PATCH 26/31] Update hfbackup_script.py
---
scripts/hfbackup_script.py | 20 +++++++++++---------
1 file changed, 11 insertions(+), 9 deletions(-)
diff --git a/scripts/hfbackup_script.py b/scripts/hfbackup_script.py
index fe09176..36f760d 100644
--- a/scripts/hfbackup_script.py
+++ b/scripts/hfbackup_script.py
@@ -171,9 +171,9 @@ def backup_files(backup_paths, script):
logger.error("Error pushing to the repo: ", e)
return
-def start_backup_thread(script, is_scheduled: bool):
+def start_backup_thread(script, is_scheduled: bool, backup_interval: int):
backup_files(script.backup_paths, script)
- script.update_schedule(is_scheduled)
+ script.update_schedule(is_scheduled, backup_interval)
#threading.Thread(target=backup_files, args=(script.backup_paths, script), daemon=True).start()
# Gradio UI Setup
@@ -188,12 +188,14 @@ def on_token_change(token: str):
hf_token_box.change(on_token_change, inputs=[hf_token_box], outputs=None)
with gr.Column(scale=1):
status_box = gr.Textbox(label="Status", value=script.status)
- def on_start_button(is_scheduled: bool, progress=gr.Progress(track_tqdm=True)):
- start_backup_thread(script, is_scheduled)
+ with gr.Row():
+ is_scheduled = gr.Checkbox(label="Scheduled backups", value=True)
+ backup_interval = gr.Number(label="Backup interval", step=1, minimum=60, maximum=36000, value=BACKUP_INTERVAL)
+ def on_start_button(is_scheduled: bool, backup_interval: int, progress=gr.Progress(track_tqdm=True)):
+ start_backup_thread(script, is_scheduled, backup_interval)
return "Starting Backup"
start_button = gr.Button(value="Start Backup")
- start_button.click(on_start_button, inputs=[is_scheduled], outputs=[status_box])
- is_scheduled = gr.Checkbox(label="Scheduled backups", value=True)
+ start_button.click(on_start_button, inputs=[is_scheduled, backup_interval], outputs=[status_box])
with gr.Row():
with gr.Column():
sd_path_box = gr.Textbox(label="SD Webui Path", value=script.sd_path)
@@ -240,13 +242,13 @@ def show(self, is_img2img=None):
def on_ui(self, is_img2img=None):
return on_ui(self)
- def update_schedule(self, is_scheduled: bool):
+ def update_schedule(self, is_scheduled: bool, backup_interval: int):
if "backup" in self.scheduler.get_jobs():
self.scheduler.shutdown(wait=False)
if is_scheduled:
- self.scheduler.reschedule_job(func=backup_files, args=[self.backup_paths, self], trigger="interval", id="backup", seconds=BACKUP_INTERVAL)
+ self.scheduler.reschedule_job(func=backup_files, args=[self.backup_paths, self], trigger="interval", id="backup", seconds=backup_interval)
elif is_scheduled:
- self.scheduler.add_job(func=backup_files, args=[self.backup_paths, self], trigger="interval", id="backup", seconds=BACKUP_INTERVAL)
+ self.scheduler.add_job(func=backup_files, args=[self.backup_paths, self], trigger="interval", id="backup", seconds=backup_interval)
self.scheduler.start()
if __package__ == "hfbackup_script":
From 9a326ee80f99cac88ffde865e10ac5996771e59d Mon Sep 17 00:00:00 2001
From: John6666 <186692226+John6666cat@users.noreply.github.com>
Date: Fri, 27 Dec 2024 10:33:07 +0900
Subject: [PATCH 27/31] Update hfbackup_script.py
---
scripts/hfbackup_script.py | 11 +++++------
1 file changed, 5 insertions(+), 6 deletions(-)
diff --git a/scripts/hfbackup_script.py b/scripts/hfbackup_script.py
index 36f760d..062a50e 100644
--- a/scripts/hfbackup_script.py
+++ b/scripts/hfbackup_script.py
@@ -243,13 +243,12 @@ def on_ui(self, is_img2img=None):
return on_ui(self)
def update_schedule(self, is_scheduled: bool, backup_interval: int):
- if "backup" in self.scheduler.get_jobs():
- self.scheduler.shutdown(wait=False)
- if is_scheduled:
- self.scheduler.reschedule_job(func=backup_files, args=[self.backup_paths, self], trigger="interval", id="backup", seconds=backup_interval)
- elif is_scheduled:
- self.scheduler.add_job(func=backup_files, args=[self.backup_paths, self], trigger="interval", id="backup", seconds=backup_interval)
+ if is_scheduled:
+ if "backup" not in self.scheduler.get_jobs():
+ self.scheduler.add_job(func=backup_files, args=[self.backup_paths, self], trigger="interval", id="backup", seconds=backup_interval)
+ else: self.scheduler.reschedule_job(func=backup_files, args=[self.backup_paths, self], trigger="interval", id="backup", seconds=backup_interval)
self.scheduler.start()
+ else: self.scheduler.shutdown(wait=False)
if __package__ == "hfbackup_script":
script = HFBackupScript()
From 7473c60d6ed3ae743326d4a9dc55f37669cca1da Mon Sep 17 00:00:00 2001
From: John6666 <186692226+John6666cat@users.noreply.github.com>
Date: Fri, 27 Dec 2024 10:36:03 +0900
Subject: [PATCH 28/31] Update hfbackup_script.py
---
scripts/hfbackup_script.py | 1 -
1 file changed, 1 deletion(-)
diff --git a/scripts/hfbackup_script.py b/scripts/hfbackup_script.py
index 062a50e..c654a2a 100644
--- a/scripts/hfbackup_script.py
+++ b/scripts/hfbackup_script.py
@@ -246,7 +246,6 @@ def update_schedule(self, is_scheduled: bool, backup_interval: int):
if is_scheduled:
if "backup" not in self.scheduler.get_jobs():
self.scheduler.add_job(func=backup_files, args=[self.backup_paths, self], trigger="interval", id="backup", seconds=backup_interval)
- else: self.scheduler.reschedule_job(func=backup_files, args=[self.backup_paths, self], trigger="interval", id="backup", seconds=backup_interval)
self.scheduler.start()
else: self.scheduler.shutdown(wait=False)
From a3fc7d4eb2e8ad5628552ab3ac85d20f2462d167 Mon Sep 17 00:00:00 2001
From: John6666 <186692226+John6666cat@users.noreply.github.com>
Date: Fri, 27 Dec 2024 10:49:42 +0900
Subject: [PATCH 29/31] Update hfbackup_script.py
---
scripts/hfbackup_script.py | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/scripts/hfbackup_script.py b/scripts/hfbackup_script.py
index c654a2a..4fa3251 100644
--- a/scripts/hfbackup_script.py
+++ b/scripts/hfbackup_script.py
@@ -232,6 +232,7 @@ def __init__(self):
self.status = "Not running"
self.basedir = basedir()
self.scheduler = BackgroundScheduler()
+ self.scheduler_job = None
def title(self):
return "Huggingface Backup"
@@ -244,10 +245,10 @@ def on_ui(self, is_img2img=None):
def update_schedule(self, is_scheduled: bool, backup_interval: int):
if is_scheduled:
- if "backup" not in self.scheduler.get_jobs():
- self.scheduler.add_job(func=backup_files, args=[self.backup_paths, self], trigger="interval", id="backup", seconds=backup_interval)
+ if self.scheduler_job is None:
+ self.scheduler_job = self.scheduler.add_job(func=backup_files, args=[self.backup_paths, self], trigger="interval", id="backup", replace_existing=True, seconds=backup_interval)
self.scheduler.start()
- else: self.scheduler.shutdown(wait=False)
+ else: self.scheduler.shutdown()
if __package__ == "hfbackup_script":
script = HFBackupScript()
From 5bea2d377a6cda42ef933111e583cca6934a425b Mon Sep 17 00:00:00 2001
From: John6666 <186692226+John6666cat@users.noreply.github.com>
Date: Fri, 27 Dec 2024 10:58:52 +0900
Subject: [PATCH 30/31] Update hfbackup_script.py
---
scripts/hfbackup_script.py | 11 ++++++-----
1 file changed, 6 insertions(+), 5 deletions(-)
diff --git a/scripts/hfbackup_script.py b/scripts/hfbackup_script.py
index 4fa3251..567c53d 100644
--- a/scripts/hfbackup_script.py
+++ b/scripts/hfbackup_script.py
@@ -231,8 +231,7 @@ def __init__(self):
self.hf_repo = REPO_NAME
self.status = "Not running"
self.basedir = basedir()
- self.scheduler = BackgroundScheduler()
- self.scheduler_job = None
+ self.scheduler = None
def title(self):
return "Huggingface Backup"
@@ -244,11 +243,13 @@ def on_ui(self, is_img2img=None):
return on_ui(self)
def update_schedule(self, is_scheduled: bool, backup_interval: int):
+ if self.scheduler is not None:
+ self.scheduler.shutdown()
+ del self.scheduler
if is_scheduled:
- if self.scheduler_job is None:
- self.scheduler_job = self.scheduler.add_job(func=backup_files, args=[self.backup_paths, self], trigger="interval", id="backup", replace_existing=True, seconds=backup_interval)
+ self.scheduler = BackgroundScheduler()
+ self.scheduler.add_job(func=backup_files, args=[self.backup_paths, self], trigger="interval", id="backup", replace_existing=True, seconds=backup_interval)
self.scheduler.start()
- else: self.scheduler.shutdown()
if __package__ == "hfbackup_script":
script = HFBackupScript()
From 20acc77093a0521a7d54b837830b3df07e80b2fc Mon Sep 17 00:00:00 2001
From: John6666 <186692226+John6666cat@users.noreply.github.com>
Date: Fri, 27 Dec 2024 11:05:17 +0900
Subject: [PATCH 31/31] Update hfbackup_script.py
---
scripts/hfbackup_script.py | 14 +++++++-------
1 file changed, 7 insertions(+), 7 deletions(-)
diff --git a/scripts/hfbackup_script.py b/scripts/hfbackup_script.py
index 567c53d..fff15d3 100644
--- a/scripts/hfbackup_script.py
+++ b/scripts/hfbackup_script.py
@@ -1,17 +1,15 @@
import os
-import time
import datetime
-import threading
+#import threading
import gradio as gr
-import subprocess
import logging
-from modules import scripts, script_callbacks, shared
+from modules import scripts, script_callbacks, shared, paths
from modules.scripts import basedir
-from modules import paths
from huggingface_hub import HfApi, HfFolder
import shutil
from pathlib import Path
import hashlib
+import gc
from apscheduler.schedulers.background import BackgroundScheduler
# Constants
@@ -173,8 +171,9 @@ def backup_files(backup_paths, script):
def start_backup_thread(script, is_scheduled: bool, backup_interval: int):
backup_files(script.backup_paths, script)
- script.update_schedule(is_scheduled, backup_interval)
#threading.Thread(target=backup_files, args=(script.backup_paths, script), daemon=True).start()
+ script.update_schedule(is_scheduled, backup_interval)
+ gc.collect()
# Gradio UI Setup
def on_ui(script):
@@ -246,11 +245,12 @@ def update_schedule(self, is_scheduled: bool, backup_interval: int):
if self.scheduler is not None:
self.scheduler.shutdown()
del self.scheduler
+ gc.collect()
if is_scheduled:
self.scheduler = BackgroundScheduler()
self.scheduler.add_job(func=backup_files, args=[self.backup_paths, self], trigger="interval", id="backup", replace_existing=True, seconds=backup_interval)
self.scheduler.start()
-
+
if __package__ == "hfbackup_script":
script = HFBackupScript()
script_callbacks.on_ui_tabs(script.on_ui)