diff --git a/README.md b/README.md index 9e84be6..b345e5b 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,10 @@ A comprehensive web-based solution for backing up GitHub repositories with sched

+- **Seamless Backup Experience**: + - Non-blocking backups without page refreshes + - Stay in place while operations run in the background + - Quick repository bulk import via "Add by Username" feature - **User Settings**: Change username and password functionality - **Docker Ready**: Fully containerized with health checks and proper user permissions @@ -89,12 +93,12 @@ docker run -d \ ### Environment Variables -| Variable | Description | Default | -|----------|-------------|---------| -| `SECRET_KEY` | Flask secret key for sessions | `dev-secret-key-change-in-production` | -| `DATABASE_URL` | SQLite database file path | `sqlite:////app/data/github_backup.db` | -| `PUID` | User ID for file permissions | `1000` | -| `PGID` | Group ID for file permissions | `1000` | +| Variable | Description | Default | +| -------------- | ----------------------------- | -------------------------------------- | +| `SECRET_KEY` | Flask secret key for sessions | `dev-secret-key-change-in-production` | +| `DATABASE_URL` | SQLite database file path | `sqlite:////app/data/github_backup.db` | +| `PUID` | User ID for file permissions | `1000` | +| `PGID` | Group ID for file permissions | `1000` | ### GitHub Token Setup diff --git a/app.py b/app.py index 0dbb620..43c82c2 100644 --- a/app.py +++ b/app.py @@ -446,7 +446,33 @@ def reset_password(): @login_required def repositories(): repos = Repository.query.filter_by(user_id=current_user.id).all() - return render_template('repositories.html', repositories=repos) + + # Get backup job status + running_jobs = BackupJob.query.filter_by(user_id=current_user.id, status='running').all() + pending_jobs = BackupJob.query.filter_by(user_id=current_user.id, status='pending').all() + completed_jobs = BackupJob.query.filter_by(user_id=current_user.id, status='completed').all() + failed_jobs = BackupJob.query.filter_by(user_id=current_user.id, status='failed').all() + + # Calculate status + total_repos = len(repos) + running_count = len(running_jobs) + pending_count = len(pending_jobs) + + # Status percentage (based on active backups vs total) + active_backups = running_count + pending_count + + backup_status = { + 'running': running_count, + 'pending': pending_count, + 'completed': len(completed_jobs), + 'failed': len(failed_jobs), + 'total_repos': total_repos, + 'active': active_backups > 0, + 'running_jobs': running_jobs, + 'pending_jobs': pending_jobs + } + + return render_template('repositories.html', repositories=repos, backup_status=backup_status) @app.route('/repositories/add', methods=['GET', 'POST']) @login_required @@ -525,6 +551,154 @@ def add_repository(): return render_template('add_repository.html') +@app.route('/repositories/add-by-username', methods=['GET', 'POST']) +@login_required +def add_repositories_by_username(): + """Add all repositories from a GitHub user""" + if request.method == 'POST': + github_username = request.form.get('github_username', '').strip() + github_token = request.form.get('github_token', '').strip() + backup_format = request.form.get('backup_format', 'folder') + schedule_type = request.form.get('schedule_type', 'daily') + retention_count = int(request.form.get('retention_count', 5)) + + if not github_username: + flash('Please provide a GitHub username', 'error') + return render_template('add_by_username.html') + + try: + from github import Github, GithubException + + # Initialize GitHub API + if github_token: + g = Github(github_token) + else: + g = Github() # Unauthenticated (limited rate limit) + + # Fetch the user + try: + user = g.get_user(github_username) + except GithubException as e: + flash(f'GitHub user "{github_username}" not found or API error: {str(e)}', 'error') + logger.warning(f"Failed to fetch GitHub user {github_username}: {str(e)}") + return render_template('add_by_username.html') + + # Get all repositories + try: + repos = user.get_repos(type='all') # all, public, private + repos_list = list(repos) + except GithubException as e: + flash(f'Error fetching repositories: {str(e)}', 'error') + logger.warning(f"Failed to fetch repos for {github_username}: {str(e)}") + return render_template('add_by_username.html') + + if not repos_list: + flash(f'No repositories found for user "{github_username}"', 'info') + return redirect(url_for('repositories')) + + added_count = 0 + skipped_count = 0 + failed_repos = [] + + for repo in repos_list: + try: + # Skip if repo is a fork (optional - change if you want to include forks) + if repo.fork: + logger.info(f"Skipping forked repository: {repo.name}") + skipped_count += 1 + continue + + repo_name = repo.name + repo_url = repo.clone_url # Uses HTTPS URL + + # Check if this repository already exists for this user + existing = Repository.query.filter_by( + user_id=current_user.id, + name=repo_name, + url=repo_url + ).first() + + if existing: + logger.info(f"Repository {repo_name} already exists for user, skipping") + skipped_count += 1 + continue + + # Create new repository record + new_repo = Repository( + user_id=current_user.id, + name=repo_name, + url=repo_url, + github_token=github_token if repo.private else '', # Only store token for private repos + backup_format=backup_format, + schedule_type=schedule_type, + retention_count=retention_count, + is_active=True + ) + + db.session.add(new_repo) + added_count += 1 + logger.info(f"Added repository: {repo_name}") + + except Exception as e: + failed_repos.append((repo.name, str(e))) + logger.error(f"Failed to add repository {repo.name}: {str(e)}") + continue + + # Commit all new repositories + if added_count > 0: + try: + db.session.commit() + logger.info(f"Committed {added_count} new repositories for user {current_user.id}") + + # Now schedule backup jobs for newly added repositories + new_repos = Repository.query.filter_by( + user_id=current_user.id, + name=repo_name # This will get the last one, but we'll schedule all active ones + ).filter(Repository.schedule_type != 'manual').all() + + # Actually, let's schedule all added repos from this batch + # Get repos added in last few seconds + cutoff_time = datetime.utcnow() - timedelta(seconds=5) + recently_added = Repository.query.filter_by( + user_id=current_user.id, + is_active=True + ).filter(Repository.created_at > cutoff_time).all() + + for repo in recently_added: + if repo.schedule_type != 'manual': + try: + schedule_backup_job(repo) + logger.info(f"Scheduled backup for {repo.name}") + except Exception as e: + logger.error(f"Failed to schedule backup for {repo.name}: {e}") + + except Exception as e: + db.session.rollback() + flash(f'Error saving repositories: {str(e)}', 'error') + logger.error(f"Failed to commit repositories: {str(e)}") + return render_template('add_by_username.html') + + # Build success message + message = f'Successfully added {added_count} repositories' + if skipped_count > 0: + message += f' ({skipped_count} skipped - already exist or are forks)' + if failed_repos: + message += f' ({len(failed_repos)} failed)' + + flash(message, 'success') + + if failed_repos: + logger.warning(f"Failed to add {len(failed_repos)} repositories: {failed_repos}") + + return redirect(url_for('repositories')) + + except Exception as e: + flash(f'Unexpected error: {str(e)}', 'error') + logger.error(f"Unexpected error in add_repositories_by_username: {str(e)}", exc_info=True) + return render_template('add_by_username.html') + + return render_template('add_by_username.html') + @app.route('/repositories//edit', methods=['GET', 'POST']) @login_required def edit_repository(repo_id): @@ -621,6 +795,43 @@ def delete_repository(repo_id): flash('Repository deleted successfully', 'success') return redirect(url_for('repositories')) +@app.route('/repositories/delete-all', methods=['POST']) +@login_required +def delete_all_repositories(): + """Delete all repositories for the current user""" + repositories = Repository.query.filter_by(user_id=current_user.id).all() + + if not repositories: + flash('No repositories to delete', 'info') + return redirect(url_for('repositories')) + + deleted_count = 0 + + for repository in repositories: + try: + # Remove scheduled job + try: + scheduler.remove_job(f'backup_{repository.id}') + logger.info(f"Removed scheduled job for repository {repository.id}") + except: + pass + + db.session.delete(repository) + deleted_count += 1 + logger.info(f"Deleted repository: {repository.name}") + except Exception as e: + logger.error(f"Failed to delete repository {repository.id}: {str(e)}") + continue + + if deleted_count > 0: + db.session.commit() + flash(f'Deleted {deleted_count} repository/repositories successfully', 'success') + logger.info(f"Deleted {deleted_count} repositories for user {current_user.id}") + else: + flash('Failed to delete repositories', 'error') + + return redirect(url_for('repositories')) + @app.route('/repositories//backup', methods=['POST']) @login_required def manual_backup(repo_id): @@ -636,6 +847,35 @@ def manual_backup(repo_id): return redirect(url_for('repositories')) +@app.route('/repositories/backup-all', methods=['POST']) +@login_required +def backup_all_repositories(): + """Trigger backups for all active repositories""" + repositories = Repository.query.filter_by(user_id=current_user.id, is_active=True).all() + + if not repositories: + flash('No active repositories to backup', 'info') + return redirect(url_for('repositories')) + + backup_count = 0 + error_count = 0 + + for repository in repositories: + try: + backup_service.backup_repository(repository) + backup_count += 1 + logger.info(f"Triggered backup for repository: {repository.name}") + except Exception as e: + error_count += 1 + logger.error(f"Failed to trigger backup for {repository.name}: {str(e)}") + + if error_count == 0: + flash(f'Started backup for {backup_count} repositories', 'success') + else: + flash(f'Started backup for {backup_count} repositories ({error_count} failed)', 'warning') + + return redirect(url_for('repositories')) + @app.route('/jobs') @login_required def backup_jobs(): diff --git a/backup_service.py b/backup_service.py index 4a1cbf0..7e8cca2 100644 --- a/backup_service.py +++ b/backup_service.py @@ -17,6 +17,29 @@ def __init__(self): self.backup_base_dir = Path('/app/backups') self.backup_base_dir.mkdir(exist_ok=True) + def _extract_github_username(self, repo_url): + """Extract GitHub username from repository URL + + Handles both formats: + - https://github.com/username/repo + - git@github.com:username/repo.git + """ + try: + # Parse the URL + if repo_url.startswith('git@'): + # git@github.com:username/repo.git + parts = repo_url.split(':')[1].split('/') + username = parts[0] + else: + # https://github.com/username/repo + parts = repo_url.rstrip('/').split('/') + username = parts[-2] # Second to last part + + return username.strip() + except (IndexError, AttributeError): + logger.warning(f"Could not extract username from URL: {repo_url}, using 'unknown'") + return 'unknown' + def backup_repository(self, repository): """Backup a repository according to its settings""" logger.info(f"Starting backup for repository: {repository.name}") @@ -44,7 +67,8 @@ def backup_repository(self, repository): return # Auto-cleanup: Check for and clean up any orphaned temp directories - user_backup_dir = self.backup_base_dir / f"user_{repository.user_id}" + github_username = self._extract_github_username(repository.url) + user_backup_dir = self.backup_base_dir / github_username repo_backup_dir = user_backup_dir / repository.name if repo_backup_dir.exists(): self._cleanup_temp_directories(repo_backup_dir) @@ -69,8 +93,11 @@ def backup_repository(self, repository): temp_clone_dir = None try: - # Create user-specific backup directory - user_backup_dir = self.backup_base_dir / f"user_{repository.user_id}" + # Extract GitHub username from repository URL + github_username = self._extract_github_username(repository.url) + + # Create GitHub-username-specific backup directory + user_backup_dir = self.backup_base_dir / github_username user_backup_dir.mkdir(exist_ok=True) # Create repository-specific backup directory diff --git a/templates/add_by_username.html b/templates/add_by_username.html new file mode 100644 index 0000000..aa0189b --- /dev/null +++ b/templates/add_by_username.html @@ -0,0 +1,190 @@ +{% extends "base.html" %} + +{% block page_title %}Add Repositories by Username{% endblock %} + +{% block content %} +
+
+
+
+

+ Add All Repositories from a GitHub User +

+
+
+ + + +
+
+ + +
+ Enter the GitHub username. All their public repositories will be discovered and added. +
+
+ +
+ + +
+ + Provide a token to: +
    +
  • Access private repositories (if you have access)
    • +
  • Increase GitHub API rate limits
    • +
  • Create a Personal Access Token in GitHub Settings
    • +
+
+
+ +
+
+ + +
+ Default format for all added repositories (can be changed per-repo later) +
+
+ +
+ + +
+ Default schedule for all added repositories (can be changed per-repo later) +
+
+
+ +
+ + +
+ Default number of backup versions to keep for each repository (can be changed per-repo later) +
+
+ +
+ + How this works: +
    +
  • Searches for all repositories owned by the specified GitHub user
    • +
  • Automatically skips repositories that are already added
    • +
  • Skips forked repositories (optional - can be modified)
    • +
  • Applies the selected defaults to all repositories
    • +
  • You can customize each repository's settings individually after adding
    • +
  • If a token is provided, it will be used for private repositories
    • +
+
+ +
+ + Back + + +
+
+
+
+
+
+ +
+
+
+
+
+ Tips +
+
+
+
    +
  • Public Repositories: Can be added without a token, but you'll have limited API rate limits (60 requests/hour)
    • +
  • Private Repositories: Requires a GitHub Personal Access Token with repository read access
    • +
  • Rate Limiting: GitHub API limits unauthenticated requests. Use a token for higher limits (5000 requests/hour)
    • +
  • Forks: Currently skipped, but you can add them manually if needed
    • +
  • Large Accounts: Accounts with many repositories may take longer to process
    • +
+
+
+
+
+{% endblock %} + +{% block scripts %} + +{% endblock %} diff --git a/templates/dashboard.html b/templates/dashboard.html index 93ca62e..0239471 100644 --- a/templates/dashboard.html +++ b/templates/dashboard.html @@ -78,9 +78,14 @@

{{ recent_jobs|selectattr('status', 'equalto', 'failed')|list|length }}

Repositories
- - Add Repository - +
+ + Add Repository + + + Add by Username + +
{% if repositories %} diff --git a/templates/repositories.html b/templates/repositories.html index e981a39..01797c2 100644 --- a/templates/repositories.html +++ b/templates/repositories.html @@ -3,11 +3,136 @@ {% block page_title %}Repositories{% endblock %} {% block content %} + + +

Repositories

- - Add Repository - +
+
+ +
+ + Add Repository + + + Add by Username + + {% if repositories %} + + {% endif %} +
{% if repositories %} @@ -70,7 +195,7 @@
{% endif %}
-
+ @@ -109,6 +234,41 @@
Confirm Delete
{% endfor %}
+ + + + {% else %}
@@ -123,4 +283,51 @@

No Repositories

{% endif %} + + + +{% endblock %} + +{% block scripts %} + {% endblock %}