mirror of
https://github.com/go-i2p/gitlab-to-gitea.git
synced 2025-07-01 09:06:22 -04:00
check in forkfix and johnconnor
This commit is contained in:
eyedeekay
@ -4,7 +4,7 @@ Go-based tool for migrating GitLab repositories, users, groups, issues and relat
|
||||
|
||||
More-or-less a port of [gitlab-to-gitea](https://git.autonomic.zone/kawaiipunk/gitlab-to-gitea) from python to Go because *fixing* python appears to be a thing I just can't get my mind around, but *rewriting* it? I'm actually OK at that.
|
||||
|
||||
Also includes: `cmd/forkfix`, for fixing fork relationships between migrated repositories by manipulating the gitea mysql database and `cmd/unmigrate` to delete everything from a gitea instance except for the admin users.
|
||||
Also includes: `cmd/forkfix`, for fixing fork relationships between migrated repositories by manipulating the gitea mysql database, `cmd/unmigrate` to delete everything from a gitea instance except for the admin users, and `cmd/johnconnor` which is a super-dangerous script for eliminating spam accounts from gitlab instances.
|
||||
|
||||
## Core Functionality
|
||||
|
||||
@ -18,7 +18,7 @@ Also includes: `cmd/forkfix`, for fixing fork relationships between migrated rep
|
||||
|
||||
- Modular package structure instead of monolithic script
|
||||
- Configuration via environment variables rather than hardcoded values
|
||||
- Added utility tools (`forkfix` and `unmigrate`)
|
||||
- Added utility tools (`forkfix`, `unmigrate`, `johnconnor`)
|
||||
- Database connectivity for commit action imports
|
||||
- Improved error handling with recovery mechanisms
|
||||
- Separation of API client code from migration logic
|
||||
|
||||
224
cmd/forkfix/main.go
Normal file
224
cmd/forkfix/main.go
Normal file
@ -0,0 +1,224 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"flag"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
|
||||
_ "github.com/mattn/go-sqlite3"
|
||||
)
|
||||
|
||||
// Repository represents a Gitea repository record
|
||||
type Repository struct {
|
||||
ID int64
|
||||
Name string
|
||||
OwnerName string
|
||||
IsFork bool
|
||||
ForkID sql.NullInt64
|
||||
}
|
||||
|
||||
func main() {
|
||||
// Define command line flags
|
||||
dbPath := flag.String("db", "", "Path to gitea.db (required)")
|
||||
forkID := flag.Int64("fork", 0, "ID of the repository to mark as fork (required)")
|
||||
originalID := flag.Int64("original", 0, "ID of the original repository (required)")
|
||||
unsetFork := flag.Bool("unset", false, "Unset fork relationship instead of setting it")
|
||||
listRepos := flag.Bool("list", false, "List all repositories in the database")
|
||||
backup := flag.Bool("backup", true, "Create a backup of the database before making changes")
|
||||
help := flag.Bool("help", false, "Show help")
|
||||
|
||||
flag.Parse()
|
||||
|
||||
// Display help if requested or if no arguments provided
|
||||
if *help || flag.NFlag() == 0 {
|
||||
printUsage()
|
||||
return
|
||||
}
|
||||
|
||||
// List repositories if requested
|
||||
if *listRepos {
|
||||
if *dbPath == "" {
|
||||
log.Fatal("Database path (-db) is required")
|
||||
}
|
||||
listRepositories(*dbPath)
|
||||
return
|
||||
}
|
||||
|
||||
// Validate required arguments
|
||||
if *dbPath == "" {
|
||||
log.Fatal("Database path (-db) is required")
|
||||
}
|
||||
|
||||
if !*unsetFork && (*forkID == 0 || *originalID == 0) {
|
||||
log.Fatal("Both fork ID (-fork) and original repository ID (-original) are required")
|
||||
}
|
||||
|
||||
if *unsetFork && *forkID == 0 {
|
||||
log.Fatal("Fork ID (-fork) is required to unset a fork relationship")
|
||||
}
|
||||
|
||||
// Create backup if requested
|
||||
if *backup {
|
||||
backupFile := backupDatabase(*dbPath)
|
||||
fmt.Printf("Created backup at: %s\n", backupFile)
|
||||
}
|
||||
|
||||
// Open database connection
|
||||
db, err := sql.Open("sqlite3", *dbPath)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to open database: %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
// Verify the database connection
|
||||
if err := db.Ping(); err != nil {
|
||||
log.Fatalf("Failed to connect to database: %v", err)
|
||||
}
|
||||
|
||||
// Verify repositories exist before making changes
|
||||
if !*unsetFork {
|
||||
verifyRepository(db, *forkID, "fork")
|
||||
verifyRepository(db, *originalID, "original")
|
||||
} else {
|
||||
verifyRepository(db, *forkID, "fork")
|
||||
}
|
||||
|
||||
// Execute the operation
|
||||
if !*unsetFork {
|
||||
// Set fork relationship
|
||||
updateForkRelationship(db, *forkID, *originalID)
|
||||
fmt.Printf("Repository ID %d is now marked as a fork of repository ID %d\n", *forkID, *originalID)
|
||||
} else {
|
||||
// Unset fork relationship
|
||||
unsetForkRelationship(db, *forkID)
|
||||
fmt.Printf("Repository ID %d is no longer marked as a fork\n", *forkID)
|
||||
}
|
||||
}
|
||||
|
||||
// printUsage displays the help information
|
||||
func printUsage() {
|
||||
fmt.Println("Gitea Repository Fork Relationship Fixer")
|
||||
fmt.Println("\nThis tool helps set, update, or remove fork relationships between repositories in a Gitea instance using SQLite.")
|
||||
fmt.Println("\nUsage:")
|
||||
fmt.Println(" Set a fork relationship:")
|
||||
fmt.Println(" gitea-fork-fixer -db /path/to/gitea.db -fork 123 -original 456")
|
||||
fmt.Println("\n Remove a fork relationship:")
|
||||
fmt.Println(" gitea-fork-fixer -db /path/to/gitea.db -fork 123 -unset")
|
||||
fmt.Println("\n List all repositories:")
|
||||
fmt.Println(" gitea-fork-fixer -db /path/to/gitea.db -list")
|
||||
fmt.Println("\nOptions:")
|
||||
flag.PrintDefaults()
|
||||
fmt.Println("\nNOTE: Always restart Gitea after making changes for them to take effect.")
|
||||
}
|
||||
|
||||
// backupDatabase creates a backup of the database file
|
||||
func backupDatabase(dbPath string) string {
|
||||
backupPath := dbPath + ".backup-" + fmt.Sprintf("%d", os.Getpid())
|
||||
|
||||
// Read the original database file
|
||||
data, err := os.ReadFile(dbPath)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to read database for backup: %v", err)
|
||||
}
|
||||
|
||||
// Write to the backup file
|
||||
err = os.WriteFile(backupPath, data, 0o644)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to create backup: %v", err)
|
||||
}
|
||||
|
||||
return backupPath
|
||||
}
|
||||
|
||||
// verifyRepository checks if a repository exists in the database
|
||||
func verifyRepository(db *sql.DB, repoID int64, repoType string) {
|
||||
var count int
|
||||
err := db.QueryRow("SELECT COUNT(*) FROM repository WHERE id = ?", repoID).Scan(&count)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to verify %s repository: %v", repoType, err)
|
||||
}
|
||||
|
||||
if count == 0 {
|
||||
log.Fatalf("%s repository with ID %d does not exist", repoType, repoID)
|
||||
}
|
||||
}
|
||||
|
||||
// updateForkRelationship sets a repository as a fork of another
|
||||
func updateForkRelationship(db *sql.DB, forkID, originalID int64) {
|
||||
tx, err := db.Begin()
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to begin transaction: %v", err)
|
||||
}
|
||||
|
||||
_, err = tx.Exec("UPDATE repository SET fork_id = ?, is_fork = 1 WHERE id = ?", originalID, forkID)
|
||||
if err != nil {
|
||||
tx.Rollback()
|
||||
log.Fatalf("Failed to update fork relationship: %v", err)
|
||||
}
|
||||
|
||||
if err := tx.Commit(); err != nil {
|
||||
log.Fatalf("Failed to commit changes: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// unsetForkRelationship removes a fork relationship
|
||||
func unsetForkRelationship(db *sql.DB, forkID int64) {
|
||||
tx, err := db.Begin()
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to begin transaction: %v", err)
|
||||
}
|
||||
|
||||
_, err = tx.Exec("UPDATE repository SET fork_id = NULL, is_fork = 0 WHERE id = ?", forkID)
|
||||
if err != nil {
|
||||
tx.Rollback()
|
||||
log.Fatalf("Failed to unset fork relationship: %v", err)
|
||||
}
|
||||
|
||||
if err := tx.Commit(); err != nil {
|
||||
log.Fatalf("Failed to commit changes: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// listRepositories displays all repositories in the database
|
||||
func listRepositories(dbPath string) {
|
||||
db, err := sql.Open("sqlite3", dbPath)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to open database: %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
rows, err := db.Query(`
|
||||
SELECT r.id, r.name, u.name as owner_name, r.is_fork, r.fork_id
|
||||
FROM repository r
|
||||
JOIN user u ON r.owner_id = u.id
|
||||
ORDER BY r.id
|
||||
`)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to query repositories: %v", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
fmt.Println("ID\tOwner/Name\tIs Fork\tFork of ID")
|
||||
fmt.Println("--\t----------\t-------\t----------")
|
||||
|
||||
for rows.Next() {
|
||||
var repo Repository
|
||||
if err := rows.Scan(&repo.ID, &repo.Name, &repo.OwnerName, &repo.IsFork, &repo.ForkID); err != nil {
|
||||
log.Fatalf("Failed to scan row: %v", err)
|
||||
}
|
||||
|
||||
forkID := "NULL"
|
||||
if repo.ForkID.Valid {
|
||||
forkID = fmt.Sprintf("%d", repo.ForkID.Int64)
|
||||
}
|
||||
|
||||
fmt.Printf("%d\t%s/%s\t%t\t%s\n",
|
||||
repo.ID, repo.OwnerName, repo.Name, repo.IsFork, forkID)
|
||||
}
|
||||
|
||||
if err := rows.Err(); err != nil {
|
||||
log.Fatalf("Error iterating through results: %v", err)
|
||||
}
|
||||
}
|
||||
275
cmd/johnconner/main.go
Normal file
275
cmd/johnconner/main.go
Normal file
@ -0,0 +1,275 @@
|
||||
// This is different, and worse, than everything else in this repo.
|
||||
// It is a script to delete users and groups from a GitLab instance.
|
||||
// It still contains I2P-specific behavior.
|
||||
// It does not use the official gitlab API.
|
||||
|
||||
// I SERIOUSLY DON'T THINK THIS IS A GOOD IDEA.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"regexp"
|
||||
)
|
||||
|
||||
const (
|
||||
disclaimerMessage = `
|
||||
# This is a gitlab anti-bot script for selfhosted instances.
|
||||
# It was used on i2pgit.org after a bot attack.
|
||||
# You **WILL** need to modify it for other instances.
|
||||
# You **WILL** need to set up an API key with very dangerous perms to use it.
|
||||
# It **WILL** break your shit in terrible, unfixable ways if you aren't very very careful.
|
||||
# You **SHOULD NEVER RUN THIS SCRIPT WITHOUT READING IT TWICE FIRST**
|
||||
# FOR REAL this is like a BIG RED BUTTON UNDER A BULLETPROOF GLASS CASE WITH TWO KEYS.
|
||||
# IN CASE OF BOT ARMY consider NOT using this thing but maybe if you really have to, then ask for help, then use it.
|
||||
# IT'S WEEDEATING WITH A FLAMETHROWER. IT'S CHAINSAW SURGERY. I CANNOT EMPHASIZE THAT ENOUGH.
|
||||
# I HAVE NO DIRECT KNOWLEDGE OF IT KILLING ANY KITTENS BUT I CANNOT RULE IT OUT.
|
||||
# That said, it did take care of the bot problem.
|
||||
`
|
||||
)
|
||||
|
||||
var (
|
||||
gitlabAPIURL = "https://gitlab.example.com/api/v4" // Modify to your GitLab instance
|
||||
gitlabToken = "" // Set your API token here or via env var
|
||||
)
|
||||
|
||||
type User struct {
|
||||
ID int `json:"id"`
|
||||
LastActivityOn interface{} `json:"last_activity_on"` // Could be null
|
||||
IsFollowed bool `json:"is_followed"`
|
||||
CanCreateProject bool `json:"can_create_project"`
|
||||
}
|
||||
|
||||
type Membership struct {
|
||||
SourceType string `json:"source_type"`
|
||||
SourceName string `json:"source_name"`
|
||||
SourceID int `json:"source_id"`
|
||||
}
|
||||
|
||||
func init() {
|
||||
// Check for token in env var
|
||||
if envToken := os.Getenv("GITLAB_API_TOKEN"); envToken != "" {
|
||||
gitlabToken = envToken
|
||||
}
|
||||
|
||||
if gitlabToken == "" {
|
||||
fmt.Println("ERROR: GITLAB_API_TOKEN not set. Please set it as an environment variable or in the code.")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// Display disclaimer
|
||||
fmt.Println(disclaimerMessage)
|
||||
fmt.Println("Are you ABSOLUTELY SURE you want to continue? This can cause IRREVERSIBLE DAMAGE.")
|
||||
fmt.Print("Type 'YES I UNDERSTAND THE CONSEQUENCES' to continue: ")
|
||||
|
||||
var confirmation string
|
||||
fmt.Scanln(&confirmation)
|
||||
if confirmation != "YES I UNDERSTAND THE CONSEQUENCES" {
|
||||
fmt.Println("Aborted.")
|
||||
os.Exit(0)
|
||||
}
|
||||
}
|
||||
|
||||
func main() {
|
||||
users, err := getAllUsers()
|
||||
if err != nil {
|
||||
fmt.Printf("Error getting users: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
for _, user := range users {
|
||||
fmt.Println("begin loop")
|
||||
fmt.Printf("User ID: %d\n", user.ID)
|
||||
|
||||
if user.IsFollowed {
|
||||
fmt.Println("User is followed, skipping")
|
||||
continue
|
||||
}
|
||||
|
||||
if user.LastActivityOn == nil {
|
||||
fmt.Printf("User %d has no activity, deleting\n", user.ID)
|
||||
deleteUser(user.ID)
|
||||
continue
|
||||
}
|
||||
|
||||
if !user.CanCreateProject {
|
||||
fmt.Printf("User %d cannot create projects, deleting\n", user.ID)
|
||||
deleteUser(user.ID)
|
||||
continue
|
||||
}
|
||||
|
||||
memberships, err := getUserMemberships(user.ID)
|
||||
if err != nil {
|
||||
fmt.Printf("Error getting memberships for user %d: %v\n", user.ID, err)
|
||||
continue
|
||||
}
|
||||
|
||||
if len(memberships) == 0 {
|
||||
fmt.Printf("User %d has no memberships, deleting\n", user.ID)
|
||||
deleteUser(user.ID)
|
||||
continue
|
||||
}
|
||||
|
||||
hasI2PProject := false
|
||||
for _, membership := range memberships {
|
||||
fmt.Printf("Project is: %s\n", membership.SourceType)
|
||||
|
||||
if membership.SourceType == "Project" {
|
||||
fmt.Printf("Project name is: %s\n", membership.SourceName)
|
||||
|
||||
if isI2PRelated(membership.SourceName) {
|
||||
fmt.Printf("%s contained an I2P related term\n", membership.SourceName)
|
||||
hasI2PProject = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if membership.SourceType == "Namespace" {
|
||||
fmt.Printf("Group name is: %s\n", membership.SourceName)
|
||||
|
||||
if !isI2PRelated(membership.SourceName) {
|
||||
fmt.Printf("Deleting non-I2P related group %d\n", membership.SourceID)
|
||||
deleteGroup(membership.SourceID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if hasI2PProject {
|
||||
fmt.Printf("User %d has I2P related project.\n", user.ID)
|
||||
} else {
|
||||
fmt.Printf("User %d has no I2P related projects, deleting\n", user.ID)
|
||||
deleteUser(user.ID)
|
||||
}
|
||||
|
||||
fmt.Println("")
|
||||
}
|
||||
}
|
||||
|
||||
func isI2PRelated(name string) bool {
|
||||
return regexp.MustCompile(`(?i)i2p`).MatchString(name)
|
||||
}
|
||||
|
||||
func getAllUsers() ([]User, error) {
|
||||
var allUsers []User
|
||||
page := 1
|
||||
|
||||
for {
|
||||
url := fmt.Sprintf("%s/users?page=%d&per_page=100", gitlabAPIURL, page)
|
||||
req, err := http.NewRequest("GET", url, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
req.Header.Set("PRIVATE-TOKEN", gitlabToken)
|
||||
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("API returned status code %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
var users []User
|
||||
if err := json.NewDecoder(resp.Body).Decode(&users); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if len(users) == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
allUsers = append(allUsers, users...)
|
||||
page++
|
||||
}
|
||||
|
||||
return allUsers, nil
|
||||
}
|
||||
|
||||
func getUserMemberships(userID int) ([]Membership, error) {
|
||||
url := fmt.Sprintf("%s/users/%d/memberships", gitlabAPIURL, userID)
|
||||
req, err := http.NewRequest("GET", url, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
req.Header.Set("PRIVATE-TOKEN", gitlabToken)
|
||||
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
return nil, fmt.Errorf("API returned status code %d: %s", resp.StatusCode, string(body))
|
||||
}
|
||||
|
||||
var memberships []Membership
|
||||
if err := json.NewDecoder(resp.Body).Decode(&memberships); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return memberships, nil
|
||||
}
|
||||
|
||||
func deleteUser(userID int) {
|
||||
url := fmt.Sprintf("%s/users/%d", gitlabAPIURL, userID)
|
||||
req, err := http.NewRequest("DELETE", url, nil)
|
||||
if err != nil {
|
||||
fmt.Printf("Error creating delete request for user %d: %v\n", userID, err)
|
||||
return
|
||||
}
|
||||
|
||||
req.Header.Set("PRIVATE-TOKEN", gitlabToken)
|
||||
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
fmt.Printf("Error deleting user %d: %v\n", userID, err)
|
||||
return
|
||||
}
|
||||
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode >= 200 && resp.StatusCode < 300 {
|
||||
fmt.Printf("Successfully deleted user %d\n", userID)
|
||||
} else {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
fmt.Printf("Failed to delete user %d. Status: %d, Response: %s\n", userID, resp.StatusCode, string(body))
|
||||
}
|
||||
}
|
||||
|
||||
func deleteGroup(groupID int) {
|
||||
url := fmt.Sprintf("%s/groups/%d", gitlabAPIURL, groupID)
|
||||
req, err := http.NewRequest("DELETE", url, nil)
|
||||
if err != nil {
|
||||
fmt.Printf("Error creating delete request for group %d: %v\n", groupID, err)
|
||||
return
|
||||
}
|
||||
|
||||
req.Header.Set("PRIVATE-TOKEN", gitlabToken)
|
||||
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
fmt.Printf("Error deleting group %d: %v\n", groupID, err)
|
||||
return
|
||||
}
|
||||
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode >= 200 && resp.StatusCode < 300 {
|
||||
fmt.Printf("Successfully deleted group %d\n", groupID)
|
||||
} else {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
fmt.Printf("Failed to delete group %d. Status: %d, Response: %s\n", groupID, resp.StatusCode, string(body))
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user