From 525f0b616f3d09405b571f5adbc1ee14d50ac89b Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Mon, 31 Dec 2018 09:37:04 +0000 Subject: [PATCH 01/19] Use git plumbing for upload: #5621 repo_editor.go: UploadRepoFile --- models/repo_editor.go | 307 +++++++++++++++++++++++++++++++++++------- 1 file changed, 261 insertions(+), 46 deletions(-) diff --git a/models/repo_editor.go b/models/repo_editor.go index 1adaa2c9552b..8d0edac6b706 100644 --- a/models/repo_editor.go +++ b/models/repo_editor.go @@ -5,6 +5,8 @@ package models import ( + "bytes" + "context" "fmt" "io" "io/ioutil" @@ -13,6 +15,7 @@ import ( "os/exec" "path" "path/filepath" + "strings" "time" "github.com/Unknwon/com" @@ -84,74 +87,286 @@ type UpdateRepoFileOptions struct { IsNewFile bool } -// UpdateRepoFile adds or updates a file in repository. -func (repo *Repository) UpdateRepoFile(doer *User, opts UpdateRepoFileOptions) (err error) { - repoWorkingPool.CheckIn(com.ToStr(repo.ID)) - defer repoWorkingPool.CheckOut(com.ToStr(repo.ID)) +func (repo *Repository) bareClone(repoPath string, branch string) (err error) { + if _, stderr, err := process.GetManager().ExecTimeout(5*time.Minute, + fmt.Sprintf("bareClone (git clone -s --bare): %s", repoPath), + "git", "clone", "-s", "--bare", "-b", branch, repo.RepoPath(), repoPath); err != nil { + return fmt.Errorf("bareClone: %v %s", err, stderr) + } + return nil +} - if err = repo.DiscardLocalRepoBranchChanges(opts.OldBranch); err != nil { - return fmt.Errorf("DiscardLocalRepoBranchChanges [branch: %s]: %v", opts.OldBranch, err) - } else if err = repo.UpdateLocalCopyBranch(opts.OldBranch); err != nil { - return fmt.Errorf("UpdateLocalCopyBranch [branch: %s]: %v", opts.OldBranch, err) +func (repo *Repository) setDefaultIndex(repoPath string) (err error) { + if _, stderr, err := process.GetManager().ExecDir(5*time.Minute, + repoPath, + fmt.Sprintf("setDefaultIndex (git read-tree HEAD): %s", repoPath), + "git", "read-tree", "HEAD"); err != nil { + return fmt.Errorf("setDefaultIndex: %v %s", err, stderr) } + return nil +} - if opts.OldBranch != opts.NewBranch { - if err := repo.CheckoutNewBranch(opts.OldBranch, opts.NewBranch); err != nil { - return fmt.Errorf("CheckoutNewBranch [old_branch: %s, new_branch: %s]: %v", opts.OldBranch, opts.NewBranch, err) +// FIXME: We should probably return the mode too +func (repo *Repository) lsFiles(repoPath string, args ...string) ([]string, error) { + stdOut := new(bytes.Buffer) + stdErr := new(bytes.Buffer) + + timeout := 5 * time.Minute + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + cmdArgs := []string{"ls-files", "-z", "--"} + for _, arg := range args { + if arg != "" { + cmdArgs = append(cmdArgs, arg) } } - localPath := repo.LocalCopyPath() - oldFilePath := path.Join(localPath, opts.OldTreeName) - filePath := path.Join(localPath, opts.NewTreeName) - dir := path.Dir(filePath) + cmd := exec.CommandContext(ctx, "git", cmdArgs...) + desc := fmt.Sprintf("lsFiles: (git ls-files) %v", cmdArgs) + cmd.Dir = repoPath + cmd.Stdout = stdOut + cmd.Stderr = stdErr - if err := os.MkdirAll(dir, os.ModePerm); err != nil { - return fmt.Errorf("Failed to create dir %s: %v", dir, err) + if err := cmd.Start(); err != nil { + return nil, fmt.Errorf("exec(%s) failed: %v(%v)", desc, err, ctx.Err()) + } + + pid := process.GetManager().Add(desc, cmd) + err := cmd.Wait() + process.GetManager().Remove(pid) + + if err != nil { + err = fmt.Errorf("exec(%d:%s) failed: %v(%v) stdout: %v stderr: %v", pid, desc, err, ctx.Err(), stdOut, stdErr) + return nil, err + } + + filelist := make([]string, len(args)) + for _, line := range bytes.Split(stdOut.Bytes(), []byte{'\000'}) { + filelist = append(filelist, string(line)) + } + + return filelist, err +} + +func (repo *Repository) removeFilesFromIndex(repoPath string, args ...string) error { + stdOut := new(bytes.Buffer) + stdErr := new(bytes.Buffer) + stdIn := new(bytes.Buffer) + for _, file := range args { + if file != "" { + stdIn.WriteString("0 0000000000000000000000000000000000000000\t") + stdIn.WriteString(file) + stdIn.WriteByte('\000') + } + } + + timeout := 5 * time.Minute + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + cmdArgs := []string{"update-index", "--remove", "-z", "--index-info"} + cmd := exec.CommandContext(ctx, "git", cmdArgs...) + desc := fmt.Sprintf("removeFilesFromIndex: (git update-index) %v", args) + cmd.Dir = repoPath + cmd.Stdout = stdOut + cmd.Stderr = stdErr + cmd.Stdin = bytes.NewReader(stdIn.Bytes()) + + if err := cmd.Start(); err != nil { + return fmt.Errorf("exec(%s) failed: %v(%v)", desc, err, ctx.Err()) + } + + pid := process.GetManager().Add(desc, cmd) + err := cmd.Wait() + process.GetManager().Remove(pid) + + if err != nil { + err = fmt.Errorf("exec(%d:%s) failed: %v(%v) stdout: %v stderr: %v", pid, desc, err, ctx.Err(), stdOut, stdErr) + } + + return err +} + +func (repo *Repository) hashObject(repoPath string, content io.Reader) (string, error) { + timeout := 5 * time.Minute + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + hashCmd := exec.CommandContext(ctx, "git", "hash-object", "-w", "--stdin") + hashCmd.Dir = repoPath + hashCmd.Stdin = content + stdOutBuffer := new(bytes.Buffer) + stdErrBuffer := new(bytes.Buffer) + hashCmd.Stdout = stdOutBuffer + hashCmd.Stderr = stdErrBuffer + desc := fmt.Sprintf("hashObject: (git hash-object)") + if err := hashCmd.Start(); err != nil { + return "", fmt.Errorf("git hash-object: %s", err) + } + + pid := process.GetManager().Add(desc, hashCmd) + err := hashCmd.Wait() + process.GetManager().Remove(pid) + + if err != nil { + err = fmt.Errorf("exec(%d:%s) failed: %v(%v) stdout: %v stderr: %v", pid, desc, err, ctx.Err(), stdOutBuffer, stdErrBuffer) + return "", err + } + + return strings.TrimSpace(stdOutBuffer.String()), nil +} + +func (repo *Repository) addObjectToIndex(repoPath, mode, objectHash, objectPath string) error { + if _, stderr, err := process.GetManager().ExecDir(5*time.Minute, + repoPath, + fmt.Sprintf("addObjectToIndex (git update-index): %s", repoPath), + "git", "update-index", "--add", "--replace", "--cacheinfo", mode, objectHash, objectPath); err != nil { + return fmt.Errorf("git update-index: %s", stderr) + } + return nil +} + +func (repo *Repository) writeTree(repoPath string) (string, error) { + + treeHash, stderr, err := process.GetManager().ExecDir(5*time.Minute, + repoPath, + fmt.Sprintf("writeTree (git write-tree): %s", repoPath), + "git", "write-tree") + if err != nil { + return "", fmt.Errorf("git write-tree: %s", stderr) + } + return strings.TrimSpace(treeHash), nil +} + +func (repo *Repository) commitTree(repoPath string, doer *User, treeHash string, message string) (string, error) { + commitTimeStr := time.Now().Format(time.UnixDate) + + // FIXME: Should we add SSH_ORIGINAL_COMMAND to this + // Because this may call hooks we should pass in the environment + env := append(os.Environ(), + "GIT_AUTHOR_NAME="+doer.DisplayName(), + "GIT_AUTHOR_EMAIL="+doer.getEmail(), + "GIT_AUTHOR_DATE="+commitTimeStr, + "GIT_COMMITTER_NAME="+doer.DisplayName(), + "GIT_COMMITTER_EMAIL="+doer.getEmail(), + "GIT_COMMITTER_DATE="+commitTimeStr, + ) + commitHash, stderr, err := process.GetManager().ExecDirEnv(5*time.Minute, + repoPath, + fmt.Sprintf("commitTree (git commit-tree): %s", repoPath), + env, + "git", "commit-tree", treeHash, "-p", "HEAD", "-m", message) + if err != nil { + return "", fmt.Errorf("git commit-tree: %s", stderr) + } + return strings.TrimSpace(commitHash), nil +} + +func (repo *Repository) actuallyPush(repoPath string, doer *User, commitHash string, branch string) error { + isWiki := "false" + if strings.HasSuffix(repo.Name, ".wiki") { + isWiki = "true" + } + + // FIXME: Should we add SSH_ORIGINAL_COMMAND to this + // Because calls hooks we need to pass in the environment + env := append(os.Environ(), + "GIT_AUTHOR_NAME="+doer.DisplayName(), + "GIT_AUTHOR_EMAIL="+doer.getEmail(), + "GIT_COMMITTER_NAME="+doer.DisplayName(), + "GIT_COMMITTER_EMAIL="+doer.getEmail(), + EnvRepoName+"="+repo.Name, + EnvRepoUsername+"="+repo.OwnerName, + EnvRepoIsWiki+"="+isWiki, + EnvPusherName+"="+doer.Name, + EnvPusherID+"="+fmt.Sprintf("%d", doer.ID), + ProtectedBranchRepoID+"="+fmt.Sprintf("%d", repo.ID), + ) + + if _, stderr, err := process.GetManager().ExecDirEnv(5*time.Minute, + repoPath, + fmt.Sprintf("actuallyPush (git push): %s", repoPath), + env, + "git", "push", repo.RepoPath(), strings.TrimSpace(commitHash)+":refs/heads/"+strings.TrimSpace(branch)); err != nil { + return fmt.Errorf("git push: %s", stderr) + } + return nil +} + +// UpdateRepoFile adds or updates a file in the repository. +func (repo *Repository) UpdateRepoFile(doer *User, opts UpdateRepoFileOptions) (err error) { + timeStr := com.ToStr(time.Now().Nanosecond()) // SHOULD USE SOMETHING UNIQUE + tmpBasePath := path.Join(LocalCopyPath(), "upload-"+timeStr+".git") + if err := os.MkdirAll(path.Dir(tmpBasePath), os.ModePerm); err != nil { + return fmt.Errorf("Failed to create dir %s: %v", tmpBasePath, err) + } + + defer os.RemoveAll(path.Dir(tmpBasePath)) + + // Do a bare shared clone into tmpBasePath and + // make HEAD to point to the OldBranch tree + if err := repo.bareClone(tmpBasePath, opts.OldBranch); err != nil { + return fmt.Errorf("UpdateRepoFile: %v", err) + } + + // Set the default index + if err := repo.setDefaultIndex(tmpBasePath); err != nil { + return fmt.Errorf("UpdateRepoFile: %v", err) + } + + filesInIndex, err := repo.lsFiles(tmpBasePath, opts.NewTreeName, opts.OldTreeName) + + if err != nil { + return fmt.Errorf("UpdateRepoFile: %v", err) } - // If it's meant to be a new file, make sure it doesn't exist. if opts.IsNewFile { - if com.IsExist(filePath) { - return ErrRepoFileAlreadyExist{filePath} + for _, file := range filesInIndex { + if file == opts.NewTreeName { + return ErrRepoFileAlreadyExist{opts.NewTreeName} + } } } - // Ignore move step if it's a new file under a directory. - // Otherwise, move the file when name changed. - if com.IsFile(oldFilePath) && opts.OldTreeName != opts.NewTreeName { - if err = git.MoveFile(localPath, opts.OldTreeName, opts.NewTreeName); err != nil { - return fmt.Errorf("git mv %s %s: %v", opts.OldTreeName, opts.NewTreeName, err) + //var stdout string + if opts.OldTreeName != opts.NewTreeName && len(filesInIndex) > 0 { + for _, file := range filesInIndex { + if file == opts.OldTreeName { + if err := repo.removeFilesFromIndex(tmpBasePath, opts.OldTreeName); err != nil { + return err + } + } } + } - if err = ioutil.WriteFile(filePath, []byte(opts.Content), 0666); err != nil { - return fmt.Errorf("WriteFile: %v", err) + // Add the object to the database + objectHash, err := repo.hashObject(tmpBasePath, strings.NewReader(opts.Content)) + if err != nil { + return err } - if err = git.AddChanges(localPath, true); err != nil { - return fmt.Errorf("git add --all: %v", err) - } else if err = git.CommitChanges(localPath, git.CommitChangesOptions{ - Committer: doer.NewGitSig(), - Message: opts.Message, - }); err != nil { - return fmt.Errorf("CommitChanges: %v", err) - } else if err = git.Push(localPath, git.PushOptions{ - Remote: "origin", - Branch: opts.NewBranch, - }); err != nil { - return fmt.Errorf("git push origin %s: %v", opts.NewBranch, err) + // Add the object to the index + if err := repo.addObjectToIndex(tmpBasePath, "100666", objectHash, opts.NewTreeName); err != nil { + return err } - gitRepo, err := git.OpenRepository(repo.RepoPath()) + // Now write the tree + treeHash, err := repo.writeTree(tmpBasePath) if err != nil { - log.Error(4, "OpenRepository: %v", err) - return nil + return err } - commit, err := gitRepo.GetBranchCommit(opts.NewBranch) + + // Now commit the tree + commitHash, err := repo.commitTree(tmpBasePath, doer, treeHash, opts.Message) if err != nil { - log.Error(4, "GetBranchCommit [branch: %s]: %v", opts.NewBranch, err) - return nil + return err + } + + // Then push this tree to NewBranch + if err := repo.actuallyPush(tmpBasePath, doer, commitHash, opts.NewBranch); err != nil { + return err } // Simulate push event. @@ -172,7 +387,7 @@ func (repo *Repository) UpdateRepoFile(doer *User, opts UpdateRepoFileOptions) ( RepoName: repo.Name, RefFullName: git.BranchPrefix + opts.NewBranch, OldCommitID: oldCommitID, - NewCommitID: commit.ID.String(), + NewCommitID: commitHash, }, ) if err != nil { From ba01dc0182169466fe4fbc4012f8db34f3df2028 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Wed, 2 Jan 2019 16:09:06 +0000 Subject: [PATCH 02/19] Use git plumbing for upload: #5621 repo_editor.go: GetDiffPreview --- models/repo_editor.go | 75 ++++++++++++++++++++++++++----------------- 1 file changed, 46 insertions(+), 29 deletions(-) diff --git a/models/repo_editor.go b/models/repo_editor.go index 8d0edac6b706..3c2a5ef9de77 100644 --- a/models/repo_editor.go +++ b/models/repo_editor.go @@ -9,12 +9,10 @@ import ( "context" "fmt" "io" - "io/ioutil" "mime/multipart" "os" "os/exec" "path" - "path/filepath" "strings" "time" @@ -398,43 +396,27 @@ func (repo *Repository) UpdateRepoFile(doer *User, opts UpdateRepoFileOptions) ( return nil } -// GetDiffPreview produces and returns diff result of a file which is not yet committed. -func (repo *Repository) GetDiffPreview(branch, treePath, content string) (diff *Diff, err error) { - repoWorkingPool.CheckIn(com.ToStr(repo.ID)) - defer repoWorkingPool.CheckOut(com.ToStr(repo.ID)) - - if err = repo.DiscardLocalRepoBranchChanges(branch); err != nil { - return nil, fmt.Errorf("DiscardLocalRepoBranchChanges [branch: %s]: %v", branch, err) - } else if err = repo.UpdateLocalCopyBranch(branch); err != nil { - return nil, fmt.Errorf("UpdateLocalCopyBranch [branch: %s]: %v", branch, err) - } - - localPath := repo.LocalCopyPath() - filePath := path.Join(localPath, treePath) - dir := filepath.Dir(filePath) - - if err := os.MkdirAll(dir, os.ModePerm); err != nil { - return nil, fmt.Errorf("Failed to create dir %s: %v", dir, err) - } +func (repo *Repository) diffIndex(repoPath string) (diff *Diff, err error) { + timeout := 5 * time.Minute + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() - if err = ioutil.WriteFile(filePath, []byte(content), 0666); err != nil { - return nil, fmt.Errorf("WriteFile: %v", err) - } + stdErr := new(bytes.Buffer) - cmd := exec.Command("git", "diff", treePath) - cmd.Dir = localPath - cmd.Stderr = os.Stderr + cmd := exec.CommandContext(ctx, "git", "diff-index", "--cached", "-p", "HEAD") + cmd.Dir = repoPath + cmd.Stderr = stdErr stdout, err := cmd.StdoutPipe() if err != nil { - return nil, fmt.Errorf("StdoutPipe: %v", err) + return nil, fmt.Errorf("StdoutPipe: %v stderr %s", err, stdErr.String()) } if err = cmd.Start(); err != nil { - return nil, fmt.Errorf("Start: %v", err) + return nil, fmt.Errorf("Start: %v stderr %s", err, stdErr.String()) } - pid := process.GetManager().Add(fmt.Sprintf("GetDiffPreview [repo_path: %s]", repo.RepoPath()), cmd) + pid := process.GetManager().Add(fmt.Sprintf("diffIndex [repo_path: %s]", repo.RepoPath()), cmd) defer process.GetManager().Remove(pid) diff, err = ParsePatch(setting.Git.MaxGitDiffLines, setting.Git.MaxGitDiffLineCharacters, setting.Git.MaxGitDiffFiles, stdout) @@ -449,6 +431,41 @@ func (repo *Repository) GetDiffPreview(branch, treePath, content string) (diff * return diff, nil } +// GetDiffPreview produces and returns diff result of a file which is not yet committed. +func (repo *Repository) GetDiffPreview(branch, treePath, content string) (diff *Diff, err error) { + timeStr := com.ToStr(time.Now().Nanosecond()) // SHOULD USE SOMETHING UNIQUE + tmpBasePath := path.Join(LocalCopyPath(), "upload-"+timeStr+".git") + if err := os.MkdirAll(path.Dir(tmpBasePath), os.ModePerm); err != nil { + return nil, fmt.Errorf("Failed to create dir %s: %v", tmpBasePath, err) + } + + defer os.RemoveAll(path.Dir(tmpBasePath)) + + // Do a bare shared clone into tmpBasePath and + // make HEAD to point to the branch tree + if err := repo.bareClone(tmpBasePath, branch); err != nil { + return nil, fmt.Errorf("GetDiffPreview: %v", err) + } + + // Set the default index + if err := repo.setDefaultIndex(tmpBasePath); err != nil { + return nil, fmt.Errorf("GetDiffPreview: %v", err) + } + + // Add the object to the database + objectHash, err := repo.hashObject(tmpBasePath, strings.NewReader(content)) + if err != nil { + return nil, fmt.Errorf("GetDiffPreview: %v", err) + } + + // Add the object to the index + if err := repo.addObjectToIndex(tmpBasePath, "100666", objectHash, treePath); err != nil { + return nil, fmt.Errorf("GetDiffPreview: %v", err) + } + + return repo.diffIndex(tmpBasePath) +} + // ________ .__ __ ___________.__.__ // \______ \ ____ | | _____/ |_ ____ \_ _____/|__| | ____ // | | \_/ __ \| | _/ __ \ __\/ __ \ | __) | | | _/ __ \ From ef4eea9e1ebc980c296de664e461106031110f8b Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Wed, 2 Jan 2019 16:53:38 +0000 Subject: [PATCH 03/19] Use git plumbing for upload: #5621 repo_editor.go: DeleteRepoFile --- models/repo_editor.go | 63 +++++++++++++++++++++---------------------- 1 file changed, 30 insertions(+), 33 deletions(-) diff --git a/models/repo_editor.go b/models/repo_editor.go index 3c2a5ef9de77..2714432f3ce5 100644 --- a/models/repo_editor.go +++ b/models/repo_editor.go @@ -485,49 +485,44 @@ type DeleteRepoFileOptions struct { // DeleteRepoFile deletes a repository file func (repo *Repository) DeleteRepoFile(doer *User, opts DeleteRepoFileOptions) (err error) { - repoWorkingPool.CheckIn(com.ToStr(repo.ID)) - defer repoWorkingPool.CheckOut(com.ToStr(repo.ID)) - - if err = repo.DiscardLocalRepoBranchChanges(opts.OldBranch); err != nil { - return fmt.Errorf("DiscardLocalRepoBranchChanges [branch: %s]: %v", opts.OldBranch, err) - } else if err = repo.UpdateLocalCopyBranch(opts.OldBranch); err != nil { - return fmt.Errorf("UpdateLocalCopyBranch [branch: %s]: %v", opts.OldBranch, err) + timeStr := com.ToStr(time.Now().Nanosecond()) // SHOULD USE SOMETHING UNIQUE + tmpBasePath := path.Join(LocalCopyPath(), "upload-"+timeStr+".git") + if err := os.MkdirAll(path.Dir(tmpBasePath), os.ModePerm); err != nil { + return fmt.Errorf("Failed to create dir %s: %v", tmpBasePath, err) } - if opts.OldBranch != opts.NewBranch { - if err := repo.CheckoutNewBranch(opts.OldBranch, opts.NewBranch); err != nil { - return fmt.Errorf("CheckoutNewBranch [old_branch: %s, new_branch: %s]: %v", opts.OldBranch, opts.NewBranch, err) - } + defer os.RemoveAll(path.Dir(tmpBasePath)) + + // Do a bare shared clone into tmpBasePath and + // make HEAD to point to the OldBranch tree + if err := repo.bareClone(tmpBasePath, opts.OldBranch); err != nil { + return fmt.Errorf("UpdateRepoFile: %s", err) } - localPath := repo.LocalCopyPath() - if err = os.Remove(path.Join(localPath, opts.TreePath)); err != nil { - return fmt.Errorf("Remove: %v", err) + // Set the default index + if err := repo.setDefaultIndex(tmpBasePath); err != nil { + return fmt.Errorf("UpdateRepoFile: %v", err) } - if err = git.AddChanges(localPath, true); err != nil { - return fmt.Errorf("git add --all: %v", err) - } else if err = git.CommitChanges(localPath, git.CommitChangesOptions{ - Committer: doer.NewGitSig(), - Message: opts.Message, - }); err != nil { - return fmt.Errorf("CommitChanges: %v", err) - } else if err = git.Push(localPath, git.PushOptions{ - Remote: "origin", - Branch: opts.NewBranch, - }); err != nil { - return fmt.Errorf("git push origin %s: %v", opts.NewBranch, err) + if err := repo.removeFilesFromIndex(tmpBasePath, opts.TreePath); err != nil { + return err } - gitRepo, err := git.OpenRepository(repo.RepoPath()) + // Now write the tree + treeHash, err := repo.writeTree(tmpBasePath) if err != nil { - log.Error(4, "OpenRepository: %v", err) - return nil + return err } - commit, err := gitRepo.GetBranchCommit(opts.NewBranch) + + // Now commit the tree + commitHash, err := repo.commitTree(tmpBasePath, doer, treeHash, opts.Message) if err != nil { - log.Error(4, "GetBranchCommit [branch: %s]: %v", opts.NewBranch, err) - return nil + return err + } + + // Then push this tree to NewBranch + if err := repo.actuallyPush(tmpBasePath, doer, commitHash, opts.NewBranch); err != nil { + return err } // Simulate push event. @@ -548,12 +543,14 @@ func (repo *Repository) DeleteRepoFile(doer *User, opts DeleteRepoFileOptions) ( RepoName: repo.Name, RefFullName: git.BranchPrefix + opts.NewBranch, OldCommitID: oldCommitID, - NewCommitID: commit.ID.String(), + NewCommitID: commitHash, }, ) if err != nil { return fmt.Errorf("PushUpdate: %v", err) } + + // FIXME: Should we UpdateRepoIndexer(repo) here? return nil } From 3e46acd8d345a4d183d8dd6767f476fe0548f40e Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Wed, 2 Jan 2019 19:50:46 +0000 Subject: [PATCH 04/19] Use git plumbing for upload: #5621 repo_editor.go: UploadRepoFiles --- models/repo_editor.go | 81 ++++++++++++++++++++----------------------- 1 file changed, 38 insertions(+), 43 deletions(-) diff --git a/models/repo_editor.go b/models/repo_editor.go index 2714432f3ce5..910290275370 100644 --- a/models/repo_editor.go +++ b/models/repo_editor.go @@ -21,7 +21,6 @@ import ( "code.gitea.io/git" - "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/process" "code.gitea.io/gitea/modules/setting" ) @@ -712,64 +711,59 @@ func (repo *Repository) UploadRepoFiles(doer *User, opts UploadRepoFileOptions) return fmt.Errorf("GetUploadsByUUIDs [uuids: %v]: %v", opts.Files, err) } - repoWorkingPool.CheckIn(com.ToStr(repo.ID)) - defer repoWorkingPool.CheckOut(com.ToStr(repo.ID)) - - if err = repo.DiscardLocalRepoBranchChanges(opts.OldBranch); err != nil { - return fmt.Errorf("DiscardLocalRepoBranchChanges [branch: %s]: %v", opts.OldBranch, err) - } else if err = repo.UpdateLocalCopyBranch(opts.OldBranch); err != nil { - return fmt.Errorf("UpdateLocalCopyBranch [branch: %s]: %v", opts.OldBranch, err) + timeStr := com.ToStr(time.Now().Nanosecond()) // SHOULD USE SOMETHING UNIQUE + tmpBasePath := path.Join(LocalCopyPath(), "upload-"+timeStr+".git") + if err := os.MkdirAll(path.Dir(tmpBasePath), os.ModePerm); err != nil { + return fmt.Errorf("Failed to create dir %s: %v", tmpBasePath, err) } - if opts.OldBranch != opts.NewBranch { - if err = repo.CheckoutNewBranch(opts.OldBranch, opts.NewBranch); err != nil { - return fmt.Errorf("CheckoutNewBranch [old_branch: %s, new_branch: %s]: %v", opts.OldBranch, opts.NewBranch, err) - } - } + defer os.RemoveAll(path.Dir(tmpBasePath)) - localPath := repo.LocalCopyPath() - dirPath := path.Join(localPath, opts.TreePath) + // Do a bare shared clone into tmpBasePath and + // make HEAD to point to the OldBranch tree + if err := repo.bareClone(tmpBasePath, opts.OldBranch); err != nil { + return fmt.Errorf("UpdateRepoFiles: %v", err) + } - if err := os.MkdirAll(dirPath, os.ModePerm); err != nil { - return fmt.Errorf("Failed to create dir %s: %v", dirPath, err) + // Set the default index + if err := repo.setDefaultIndex(tmpBasePath); err != nil { + return fmt.Errorf("UpdateRepoFiles: %v", err) } // Copy uploaded files into repository. for _, upload := range uploads { - tmpPath := upload.LocalPath() - targetPath := path.Join(dirPath, upload.Name) - if !com.IsFile(tmpPath) { - continue + file, err := os.Open(upload.LocalPath()) + if err != nil { + return err } + defer file.Close() - if err = com.Copy(tmpPath, targetPath); err != nil { - return fmt.Errorf("Copy: %v", err) + objectHash, err := repo.hashObject(tmpBasePath, file) + if err != nil { + return err } - } - if err = git.AddChanges(localPath, true); err != nil { - return fmt.Errorf("git add --all: %v", err) - } else if err = git.CommitChanges(localPath, git.CommitChangesOptions{ - Committer: doer.NewGitSig(), - Message: opts.Message, - }); err != nil { - return fmt.Errorf("CommitChanges: %v", err) - } else if err = git.Push(localPath, git.PushOptions{ - Remote: "origin", - Branch: opts.NewBranch, - }); err != nil { - return fmt.Errorf("git push origin %s: %v", opts.NewBranch, err) + // Add the object to the index + if err := repo.addObjectToIndex(tmpBasePath, "100666", objectHash, path.Join(opts.TreePath, upload.Name)); err != nil { + return err + } } - gitRepo, err := git.OpenRepository(repo.RepoPath()) + // Now write the tree + treeHash, err := repo.writeTree(tmpBasePath) if err != nil { - log.Error(4, "OpenRepository: %v", err) - return nil + return err } - commit, err := gitRepo.GetBranchCommit(opts.NewBranch) + + // Now commit the tree + commitHash, err := repo.commitTree(tmpBasePath, doer, treeHash, opts.Message) if err != nil { - log.Error(4, "GetBranchCommit [branch: %s]: %v", opts.NewBranch, err) - return nil + return err + } + + // Then push this tree to NewBranch + if err := repo.actuallyPush(tmpBasePath, doer, commitHash, opts.NewBranch); err != nil { + return err } // Simulate push event. @@ -790,12 +784,13 @@ func (repo *Repository) UploadRepoFiles(doer *User, opts UploadRepoFileOptions) RepoName: repo.Name, RefFullName: git.BranchPrefix + opts.NewBranch, OldCommitID: oldCommitID, - NewCommitID: commit.ID.String(), + NewCommitID: commitHash, }, ) if err != nil { return fmt.Errorf("PushUpdate: %v", err) } + // FIXME: Should we UpdateRepoIndexer(repo) here? return DeleteUploads(uploads...) } From 3203e986b1b3df6c2545da00f16616dc160382f3 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Wed, 2 Jan 2019 21:12:34 +0000 Subject: [PATCH 05/19] Move branch checkout functions out of repo_editor.go as they are no longer used there --- models/repo_branch.go | 40 ++++++++++++++++++++++++++++++++++++++++ models/repo_editor.go | 40 ---------------------------------------- 2 files changed, 40 insertions(+), 40 deletions(-) diff --git a/models/repo_branch.go b/models/repo_branch.go index cd12742ba1c9..88417cbd36ae 100644 --- a/models/repo_branch.go +++ b/models/repo_branch.go @@ -14,6 +14,46 @@ import ( "github.com/Unknwon/com" ) +// discardLocalRepoBranchChanges discards local commits/changes of +// given branch to make sure it is even to remote branch. +func discardLocalRepoBranchChanges(localPath, branch string) error { + if !com.IsExist(localPath) { + return nil + } + // No need to check if nothing in the repository. + if !git.IsBranchExist(localPath, branch) { + return nil + } + + refName := "origin/" + branch + if err := git.ResetHEAD(localPath, true, refName); err != nil { + return fmt.Errorf("git reset --hard %s: %v", refName, err) + } + return nil +} + +// DiscardLocalRepoBranchChanges discards the local repository branch changes +func (repo *Repository) DiscardLocalRepoBranchChanges(branch string) error { + return discardLocalRepoBranchChanges(repo.LocalCopyPath(), branch) +} + +// checkoutNewBranch checks out to a new branch from the a branch name. +func checkoutNewBranch(repoPath, localPath, oldBranch, newBranch string) error { + if err := git.Checkout(localPath, git.CheckoutOptions{ + Timeout: time.Duration(setting.Git.Timeout.Pull) * time.Second, + Branch: newBranch, + OldBranch: oldBranch, + }); err != nil { + return fmt.Errorf("git checkout -b %s %s: %v", newBranch, oldBranch, err) + } + return nil +} + +// CheckoutNewBranch checks out a new branch +func (repo *Repository) CheckoutNewBranch(oldBranch, newBranch string) error { + return checkoutNewBranch(repo.RepoPath(), repo.LocalCopyPath(), oldBranch, newBranch) +} + // Branch holds the branch information type Branch struct { Path string diff --git a/models/repo_editor.go b/models/repo_editor.go index 910290275370..375865be35d8 100644 --- a/models/repo_editor.go +++ b/models/repo_editor.go @@ -32,46 +32,6 @@ import ( // /_______ /\____ | |__||__| \___ / |__|____/\___ > // \/ \/ \/ \/ -// discardLocalRepoBranchChanges discards local commits/changes of -// given branch to make sure it is even to remote branch. -func discardLocalRepoBranchChanges(localPath, branch string) error { - if !com.IsExist(localPath) { - return nil - } - // No need to check if nothing in the repository. - if !git.IsBranchExist(localPath, branch) { - return nil - } - - refName := "origin/" + branch - if err := git.ResetHEAD(localPath, true, refName); err != nil { - return fmt.Errorf("git reset --hard %s: %v", refName, err) - } - return nil -} - -// DiscardLocalRepoBranchChanges discards the local repository branch changes -func (repo *Repository) DiscardLocalRepoBranchChanges(branch string) error { - return discardLocalRepoBranchChanges(repo.LocalCopyPath(), branch) -} - -// checkoutNewBranch checks out to a new branch from the a branch name. -func checkoutNewBranch(repoPath, localPath, oldBranch, newBranch string) error { - if err := git.Checkout(localPath, git.CheckoutOptions{ - Timeout: time.Duration(setting.Git.Timeout.Pull) * time.Second, - Branch: newBranch, - OldBranch: oldBranch, - }); err != nil { - return fmt.Errorf("git checkout -b %s %s: %v", newBranch, oldBranch, err) - } - return nil -} - -// CheckoutNewBranch checks out a new branch -func (repo *Repository) CheckoutNewBranch(oldBranch, newBranch string) error { - return checkoutNewBranch(repo.RepoPath(), repo.LocalCopyPath(), oldBranch, newBranch) -} - // UpdateRepoFileOptions holds the repository file update options type UpdateRepoFileOptions struct { LastCommitID string From a5067f1476f5ef87385cebb4feb5d756e7a5ac85 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Thu, 3 Jan 2019 16:19:27 +0000 Subject: [PATCH 06/19] BUGFIX: The default permissions should be 100644 This is a change from the previous code but is more in keeping with the default behaviour of git. Signed-off-by: Andrew Thornton --- models/repo_editor.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/models/repo_editor.go b/models/repo_editor.go index 375865be35d8..67ce33a740ef 100644 --- a/models/repo_editor.go +++ b/models/repo_editor.go @@ -305,7 +305,7 @@ func (repo *Repository) UpdateRepoFile(doer *User, opts UpdateRepoFileOptions) ( } // Add the object to the index - if err := repo.addObjectToIndex(tmpBasePath, "100666", objectHash, opts.NewTreeName); err != nil { + if err := repo.addObjectToIndex(tmpBasePath, "100644", objectHash, opts.NewTreeName); err != nil { return err } @@ -418,7 +418,7 @@ func (repo *Repository) GetDiffPreview(branch, treePath, content string) (diff * } // Add the object to the index - if err := repo.addObjectToIndex(tmpBasePath, "100666", objectHash, treePath); err != nil { + if err := repo.addObjectToIndex(tmpBasePath, "100644", objectHash, treePath); err != nil { return nil, fmt.Errorf("GetDiffPreview: %v", err) } @@ -704,7 +704,7 @@ func (repo *Repository) UploadRepoFiles(doer *User, opts UploadRepoFileOptions) } // Add the object to the index - if err := repo.addObjectToIndex(tmpBasePath, "100666", objectHash, path.Join(opts.TreePath, upload.Name)); err != nil { + if err := repo.addObjectToIndex(tmpBasePath, "100644", objectHash, path.Join(opts.TreePath, upload.Name)); err != nil { return err } } From 51c70aa04861676e7958576d51584a89585d864b Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Fri, 4 Jan 2019 15:14:31 +0000 Subject: [PATCH 07/19] Standardise cleanUploadFilename to more closely match git See verify_path in: https://github.com/git/git/blob/7f4e64169352e03476b0ea64e7e2973669e491a2/read-cache.c#L951 Signed-off-by: Andrew Thornton --- routers/repo/editor.go | 13 +++++++------ routers/repo/editor_test.go | 11 ++++++----- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/routers/repo/editor.go b/routers/repo/editor.go index 4e3557dbb2a5..c0eefe8143a8 100644 --- a/routers/repo/editor.go +++ b/routers/repo/editor.go @@ -576,12 +576,13 @@ func UploadFilePost(ctx *context.Context, form auth.UploadRepoFileForm) { } func cleanUploadFileName(name string) string { - name = strings.TrimLeft(name, "./\\") - name = strings.Replace(name, "../", "", -1) - name = strings.Replace(name, "..\\", "", -1) - name = strings.TrimPrefix(path.Clean(name), ".git/") - if name == ".git" { - return "" + // Rebase the filename + name = strings.Trim(path.Clean("/"+name), " /") + // Git disallows any filenames to have a .git directory in them. + for _, part := range strings.Split(name, "/") { + if strings.ToLower(part) == ".git" { + return "" + } } return name } diff --git a/routers/repo/editor_test.go b/routers/repo/editor_test.go index e5b9570205c3..63518caff28f 100644 --- a/routers/repo/editor_test.go +++ b/routers/repo/editor_test.go @@ -15,14 +15,15 @@ func TestCleanUploadName(t *testing.T) { models.PrepareTestEnv(t) var kases = map[string]string{ - ".git/refs/master": "git/refs/master", + ".git/refs/master": "", "/root/abc": "root/abc", "./../../abc": "abc", - "a/../.git": "a/.git", - "a/../../../abc": "a/abc", + "a/../.git": "", + "a/../../../abc": "abc", "../../../acd": "acd", - "../../.git/abc": "git/abc", - "..\\..\\.git/abc": "git/abc", + "../../.git/abc": "", + "..\\..\\.git/abc": "..\\..\\.git/abc", + "abc/../def": "def", } for k, v := range kases { assert.EqualValues(t, v, cleanUploadFileName(k)) From 5b4d74079e0170d1ece8bf9490b795d13d7fd5c9 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Fri, 4 Jan 2019 18:28:21 +0000 Subject: [PATCH 08/19] Redirect on bad paths Signed-off-by: Andrew Thornton --- models/repo_editor.go | 27 +++++++++++++++++++++------ routers/repo/editor.go | 37 ++++++++++++++++++++++++++++++------- 2 files changed, 51 insertions(+), 13 deletions(-) diff --git a/models/repo_editor.go b/models/repo_editor.go index 67ce33a740ef..84d9119b70c6 100644 --- a/models/repo_editor.go +++ b/models/repo_editor.go @@ -455,33 +455,48 @@ func (repo *Repository) DeleteRepoFile(doer *User, opts DeleteRepoFileOptions) ( // Do a bare shared clone into tmpBasePath and // make HEAD to point to the OldBranch tree if err := repo.bareClone(tmpBasePath, opts.OldBranch); err != nil { - return fmt.Errorf("UpdateRepoFile: %s", err) + return fmt.Errorf("DeleteRepoFile: %v", err) } // Set the default index if err := repo.setDefaultIndex(tmpBasePath); err != nil { - return fmt.Errorf("UpdateRepoFile: %v", err) + return fmt.Errorf("DeleteRepoFile: %v", err) + } + + filelist, err := repo.lsFiles(tmpBasePath, opts.TreePath) + if err != nil { + return fmt.Errorf("DeleteRepoFile: %v", err) + } + + inFilelist := false + for _, file := range filelist { + if file == opts.TreePath { + inFilelist = true + } + } + if !inFilelist { + return git.ErrNotExist{RelPath: opts.TreePath} } if err := repo.removeFilesFromIndex(tmpBasePath, opts.TreePath); err != nil { - return err + return fmt.Errorf("DeleteRepoFile: %v", err) } // Now write the tree treeHash, err := repo.writeTree(tmpBasePath) if err != nil { - return err + return fmt.Errorf("DeleteRepoFile: %v", err) } // Now commit the tree commitHash, err := repo.commitTree(tmpBasePath, doer, treeHash, opts.Message) if err != nil { - return err + return fmt.Errorf("DeleteRepoFile: %v", err) } // Then push this tree to NewBranch if err := repo.actuallyPush(tmpBasePath, doer, commitHash, opts.NewBranch); err != nil { - return err + return fmt.Errorf("DeleteRepoFile: %v", err) } // Simulate push event. diff --git a/routers/repo/editor.go b/routers/repo/editor.go index c0eefe8143a8..4a2eeebea965 100644 --- a/routers/repo/editor.go +++ b/routers/repo/editor.go @@ -62,6 +62,16 @@ func editFile(ctx *context.Context, isNewFile bool) { ctx.Data["RequireSimpleMDE"] = true canCommit := renderCommitRights(ctx) + treePath := cleanUploadFileName(ctx.Repo.TreePath) + if treePath != ctx.Repo.TreePath { + if isNewFile { + ctx.Redirect(path.Join(ctx.Repo.RepoLink, "_new", ctx.Repo.BranchName, treePath)) + } else { + ctx.Redirect(path.Join(ctx.Repo.RepoLink, "_edit", ctx.Repo.BranchName, treePath)) + } + return + } + treeNames, treePaths := getParentTreeFields(ctx.Repo.TreePath) if !isNewFile { @@ -155,7 +165,7 @@ func editFilePost(ctx *context.Context, form auth.EditRepoFileForm, isNewFile bo oldBranchName := ctx.Repo.BranchName branchName := oldBranchName - oldTreePath := ctx.Repo.TreePath + oldTreePath := cleanUploadFileName(ctx.Repo.TreePath) lastCommit := form.LastCommit form.LastCommit = ctx.Repo.Commit.ID.String() @@ -328,7 +338,11 @@ func NewFilePost(ctx *context.Context, form auth.EditRepoFileForm) { // DiffPreviewPost render preview diff page func DiffPreviewPost(ctx *context.Context, form auth.EditPreviewDiffForm) { - treePath := ctx.Repo.TreePath + treePath := cleanUploadFileName(ctx.Repo.TreePath) + if len(treePath) == 0 { + ctx.Error(500, "file name to diff is invalid") + return + } entry, err := ctx.Repo.Commit.GetTreeEntryByPath(treePath) if err != nil { @@ -358,7 +372,14 @@ func DiffPreviewPost(ctx *context.Context, form auth.EditPreviewDiffForm) { func DeleteFile(ctx *context.Context) { ctx.Data["PageIsDelete"] = true ctx.Data["BranchLink"] = ctx.Repo.RepoLink + "/src/" + ctx.Repo.BranchNameSubURL() - ctx.Data["TreePath"] = ctx.Repo.TreePath + treePath := cleanUploadFileName(ctx.Repo.TreePath) + + if treePath != ctx.Repo.TreePath { + ctx.Redirect(path.Join(ctx.Repo.RepoLink, "_delete", ctx.Repo.BranchName, treePath)) + return + } + + ctx.Data["TreePath"] = treePath canCommit := renderCommitRights(ctx) ctx.Data["commit_summary"] = "" @@ -453,6 +474,12 @@ func UploadFile(ctx *context.Context) { ctx.Data["PageIsUpload"] = true renderUploadSettings(ctx) canCommit := renderCommitRights(ctx) + treePath := cleanUploadFileName(ctx.Repo.TreePath) + if treePath != ctx.Repo.TreePath { + ctx.Redirect(path.Join(ctx.Repo.RepoLink, "_upload", ctx.Repo.BranchName, treePath)) + return + } + ctx.Repo.TreePath = treePath treeNames, treePaths := getParentTreeFields(ctx.Repo.TreePath) if len(treeNames) == 0 { @@ -489,10 +516,6 @@ func UploadFilePost(ctx *context.Context, form auth.UploadRepoFileForm) { } form.TreePath = cleanUploadFileName(form.TreePath) - if len(form.TreePath) == 0 { - ctx.Error(500, "Upload file name is invalid") - return - } treeNames, treePaths := getParentTreeFields(form.TreePath) if len(treeNames) == 0 { From eabbdca446a7c56b28f2644dd82933db291b813e Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Thu, 10 Jan 2019 15:12:41 +0000 Subject: [PATCH 09/19] Refactor to move the uploading functions out to a module Signed-off-by: Andrew Thornton --- models/repo_editor.go | 771 ------------------------------------- models/upload.go | 155 ++++++++ modules/uploader/delete.go | 100 +++++ modules/uploader/diff.go | 38 ++ modules/uploader/repo.go | 297 ++++++++++++++ modules/uploader/update.go | 121 ++++++ modules/uploader/upload.go | 112 ++++++ routers/repo/editor.go | 9 +- 8 files changed, 828 insertions(+), 775 deletions(-) delete mode 100644 models/repo_editor.go create mode 100644 models/upload.go create mode 100644 modules/uploader/delete.go create mode 100644 modules/uploader/diff.go create mode 100644 modules/uploader/repo.go create mode 100644 modules/uploader/update.go create mode 100644 modules/uploader/upload.go diff --git a/models/repo_editor.go b/models/repo_editor.go deleted file mode 100644 index 84d9119b70c6..000000000000 --- a/models/repo_editor.go +++ /dev/null @@ -1,771 +0,0 @@ -// Copyright 2016 The Gogs Authors. All rights reserved. -// Use of this source code is governed by a MIT-style -// license that can be found in the LICENSE file. - -package models - -import ( - "bytes" - "context" - "fmt" - "io" - "mime/multipart" - "os" - "os/exec" - "path" - "strings" - "time" - - "github.com/Unknwon/com" - gouuid "github.com/satori/go.uuid" - - "code.gitea.io/git" - - "code.gitea.io/gitea/modules/process" - "code.gitea.io/gitea/modules/setting" -) - -// ___________ .___.__ __ ___________.__.__ -// \_ _____/ __| _/|__|/ |_ \_ _____/|__| | ____ -// | __)_ / __ | | \ __\ | __) | | | _/ __ \ -// | \/ /_/ | | || | | \ | | |_\ ___/ -// /_______ /\____ | |__||__| \___ / |__|____/\___ > -// \/ \/ \/ \/ - -// UpdateRepoFileOptions holds the repository file update options -type UpdateRepoFileOptions struct { - LastCommitID string - OldBranch string - NewBranch string - OldTreeName string - NewTreeName string - Message string - Content string - IsNewFile bool -} - -func (repo *Repository) bareClone(repoPath string, branch string) (err error) { - if _, stderr, err := process.GetManager().ExecTimeout(5*time.Minute, - fmt.Sprintf("bareClone (git clone -s --bare): %s", repoPath), - "git", "clone", "-s", "--bare", "-b", branch, repo.RepoPath(), repoPath); err != nil { - return fmt.Errorf("bareClone: %v %s", err, stderr) - } - return nil -} - -func (repo *Repository) setDefaultIndex(repoPath string) (err error) { - if _, stderr, err := process.GetManager().ExecDir(5*time.Minute, - repoPath, - fmt.Sprintf("setDefaultIndex (git read-tree HEAD): %s", repoPath), - "git", "read-tree", "HEAD"); err != nil { - return fmt.Errorf("setDefaultIndex: %v %s", err, stderr) - } - return nil -} - -// FIXME: We should probably return the mode too -func (repo *Repository) lsFiles(repoPath string, args ...string) ([]string, error) { - stdOut := new(bytes.Buffer) - stdErr := new(bytes.Buffer) - - timeout := 5 * time.Minute - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() - - cmdArgs := []string{"ls-files", "-z", "--"} - for _, arg := range args { - if arg != "" { - cmdArgs = append(cmdArgs, arg) - } - } - - cmd := exec.CommandContext(ctx, "git", cmdArgs...) - desc := fmt.Sprintf("lsFiles: (git ls-files) %v", cmdArgs) - cmd.Dir = repoPath - cmd.Stdout = stdOut - cmd.Stderr = stdErr - - if err := cmd.Start(); err != nil { - return nil, fmt.Errorf("exec(%s) failed: %v(%v)", desc, err, ctx.Err()) - } - - pid := process.GetManager().Add(desc, cmd) - err := cmd.Wait() - process.GetManager().Remove(pid) - - if err != nil { - err = fmt.Errorf("exec(%d:%s) failed: %v(%v) stdout: %v stderr: %v", pid, desc, err, ctx.Err(), stdOut, stdErr) - return nil, err - } - - filelist := make([]string, len(args)) - for _, line := range bytes.Split(stdOut.Bytes(), []byte{'\000'}) { - filelist = append(filelist, string(line)) - } - - return filelist, err -} - -func (repo *Repository) removeFilesFromIndex(repoPath string, args ...string) error { - stdOut := new(bytes.Buffer) - stdErr := new(bytes.Buffer) - stdIn := new(bytes.Buffer) - for _, file := range args { - if file != "" { - stdIn.WriteString("0 0000000000000000000000000000000000000000\t") - stdIn.WriteString(file) - stdIn.WriteByte('\000') - } - } - - timeout := 5 * time.Minute - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() - - cmdArgs := []string{"update-index", "--remove", "-z", "--index-info"} - cmd := exec.CommandContext(ctx, "git", cmdArgs...) - desc := fmt.Sprintf("removeFilesFromIndex: (git update-index) %v", args) - cmd.Dir = repoPath - cmd.Stdout = stdOut - cmd.Stderr = stdErr - cmd.Stdin = bytes.NewReader(stdIn.Bytes()) - - if err := cmd.Start(); err != nil { - return fmt.Errorf("exec(%s) failed: %v(%v)", desc, err, ctx.Err()) - } - - pid := process.GetManager().Add(desc, cmd) - err := cmd.Wait() - process.GetManager().Remove(pid) - - if err != nil { - err = fmt.Errorf("exec(%d:%s) failed: %v(%v) stdout: %v stderr: %v", pid, desc, err, ctx.Err(), stdOut, stdErr) - } - - return err -} - -func (repo *Repository) hashObject(repoPath string, content io.Reader) (string, error) { - timeout := 5 * time.Minute - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() - - hashCmd := exec.CommandContext(ctx, "git", "hash-object", "-w", "--stdin") - hashCmd.Dir = repoPath - hashCmd.Stdin = content - stdOutBuffer := new(bytes.Buffer) - stdErrBuffer := new(bytes.Buffer) - hashCmd.Stdout = stdOutBuffer - hashCmd.Stderr = stdErrBuffer - desc := fmt.Sprintf("hashObject: (git hash-object)") - if err := hashCmd.Start(); err != nil { - return "", fmt.Errorf("git hash-object: %s", err) - } - - pid := process.GetManager().Add(desc, hashCmd) - err := hashCmd.Wait() - process.GetManager().Remove(pid) - - if err != nil { - err = fmt.Errorf("exec(%d:%s) failed: %v(%v) stdout: %v stderr: %v", pid, desc, err, ctx.Err(), stdOutBuffer, stdErrBuffer) - return "", err - } - - return strings.TrimSpace(stdOutBuffer.String()), nil -} - -func (repo *Repository) addObjectToIndex(repoPath, mode, objectHash, objectPath string) error { - if _, stderr, err := process.GetManager().ExecDir(5*time.Minute, - repoPath, - fmt.Sprintf("addObjectToIndex (git update-index): %s", repoPath), - "git", "update-index", "--add", "--replace", "--cacheinfo", mode, objectHash, objectPath); err != nil { - return fmt.Errorf("git update-index: %s", stderr) - } - return nil -} - -func (repo *Repository) writeTree(repoPath string) (string, error) { - - treeHash, stderr, err := process.GetManager().ExecDir(5*time.Minute, - repoPath, - fmt.Sprintf("writeTree (git write-tree): %s", repoPath), - "git", "write-tree") - if err != nil { - return "", fmt.Errorf("git write-tree: %s", stderr) - } - return strings.TrimSpace(treeHash), nil -} - -func (repo *Repository) commitTree(repoPath string, doer *User, treeHash string, message string) (string, error) { - commitTimeStr := time.Now().Format(time.UnixDate) - - // FIXME: Should we add SSH_ORIGINAL_COMMAND to this - // Because this may call hooks we should pass in the environment - env := append(os.Environ(), - "GIT_AUTHOR_NAME="+doer.DisplayName(), - "GIT_AUTHOR_EMAIL="+doer.getEmail(), - "GIT_AUTHOR_DATE="+commitTimeStr, - "GIT_COMMITTER_NAME="+doer.DisplayName(), - "GIT_COMMITTER_EMAIL="+doer.getEmail(), - "GIT_COMMITTER_DATE="+commitTimeStr, - ) - commitHash, stderr, err := process.GetManager().ExecDirEnv(5*time.Minute, - repoPath, - fmt.Sprintf("commitTree (git commit-tree): %s", repoPath), - env, - "git", "commit-tree", treeHash, "-p", "HEAD", "-m", message) - if err != nil { - return "", fmt.Errorf("git commit-tree: %s", stderr) - } - return strings.TrimSpace(commitHash), nil -} - -func (repo *Repository) actuallyPush(repoPath string, doer *User, commitHash string, branch string) error { - isWiki := "false" - if strings.HasSuffix(repo.Name, ".wiki") { - isWiki = "true" - } - - // FIXME: Should we add SSH_ORIGINAL_COMMAND to this - // Because calls hooks we need to pass in the environment - env := append(os.Environ(), - "GIT_AUTHOR_NAME="+doer.DisplayName(), - "GIT_AUTHOR_EMAIL="+doer.getEmail(), - "GIT_COMMITTER_NAME="+doer.DisplayName(), - "GIT_COMMITTER_EMAIL="+doer.getEmail(), - EnvRepoName+"="+repo.Name, - EnvRepoUsername+"="+repo.OwnerName, - EnvRepoIsWiki+"="+isWiki, - EnvPusherName+"="+doer.Name, - EnvPusherID+"="+fmt.Sprintf("%d", doer.ID), - ProtectedBranchRepoID+"="+fmt.Sprintf("%d", repo.ID), - ) - - if _, stderr, err := process.GetManager().ExecDirEnv(5*time.Minute, - repoPath, - fmt.Sprintf("actuallyPush (git push): %s", repoPath), - env, - "git", "push", repo.RepoPath(), strings.TrimSpace(commitHash)+":refs/heads/"+strings.TrimSpace(branch)); err != nil { - return fmt.Errorf("git push: %s", stderr) - } - return nil -} - -// UpdateRepoFile adds or updates a file in the repository. -func (repo *Repository) UpdateRepoFile(doer *User, opts UpdateRepoFileOptions) (err error) { - timeStr := com.ToStr(time.Now().Nanosecond()) // SHOULD USE SOMETHING UNIQUE - tmpBasePath := path.Join(LocalCopyPath(), "upload-"+timeStr+".git") - if err := os.MkdirAll(path.Dir(tmpBasePath), os.ModePerm); err != nil { - return fmt.Errorf("Failed to create dir %s: %v", tmpBasePath, err) - } - - defer os.RemoveAll(path.Dir(tmpBasePath)) - - // Do a bare shared clone into tmpBasePath and - // make HEAD to point to the OldBranch tree - if err := repo.bareClone(tmpBasePath, opts.OldBranch); err != nil { - return fmt.Errorf("UpdateRepoFile: %v", err) - } - - // Set the default index - if err := repo.setDefaultIndex(tmpBasePath); err != nil { - return fmt.Errorf("UpdateRepoFile: %v", err) - } - - filesInIndex, err := repo.lsFiles(tmpBasePath, opts.NewTreeName, opts.OldTreeName) - - if err != nil { - return fmt.Errorf("UpdateRepoFile: %v", err) - } - - if opts.IsNewFile { - for _, file := range filesInIndex { - if file == opts.NewTreeName { - return ErrRepoFileAlreadyExist{opts.NewTreeName} - } - } - } - - //var stdout string - if opts.OldTreeName != opts.NewTreeName && len(filesInIndex) > 0 { - for _, file := range filesInIndex { - if file == opts.OldTreeName { - if err := repo.removeFilesFromIndex(tmpBasePath, opts.OldTreeName); err != nil { - return err - } - } - } - - } - - // Add the object to the database - objectHash, err := repo.hashObject(tmpBasePath, strings.NewReader(opts.Content)) - if err != nil { - return err - } - - // Add the object to the index - if err := repo.addObjectToIndex(tmpBasePath, "100644", objectHash, opts.NewTreeName); err != nil { - return err - } - - // Now write the tree - treeHash, err := repo.writeTree(tmpBasePath) - if err != nil { - return err - } - - // Now commit the tree - commitHash, err := repo.commitTree(tmpBasePath, doer, treeHash, opts.Message) - if err != nil { - return err - } - - // Then push this tree to NewBranch - if err := repo.actuallyPush(tmpBasePath, doer, commitHash, opts.NewBranch); err != nil { - return err - } - - // Simulate push event. - oldCommitID := opts.LastCommitID - if opts.NewBranch != opts.OldBranch { - oldCommitID = git.EmptySHA - } - - if err = repo.GetOwner(); err != nil { - return fmt.Errorf("GetOwner: %v", err) - } - err = PushUpdate( - opts.NewBranch, - PushUpdateOptions{ - PusherID: doer.ID, - PusherName: doer.Name, - RepoUserName: repo.Owner.Name, - RepoName: repo.Name, - RefFullName: git.BranchPrefix + opts.NewBranch, - OldCommitID: oldCommitID, - NewCommitID: commitHash, - }, - ) - if err != nil { - return fmt.Errorf("PushUpdate: %v", err) - } - UpdateRepoIndexer(repo) - - return nil -} - -func (repo *Repository) diffIndex(repoPath string) (diff *Diff, err error) { - timeout := 5 * time.Minute - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() - - stdErr := new(bytes.Buffer) - - cmd := exec.CommandContext(ctx, "git", "diff-index", "--cached", "-p", "HEAD") - cmd.Dir = repoPath - cmd.Stderr = stdErr - - stdout, err := cmd.StdoutPipe() - if err != nil { - return nil, fmt.Errorf("StdoutPipe: %v stderr %s", err, stdErr.String()) - } - - if err = cmd.Start(); err != nil { - return nil, fmt.Errorf("Start: %v stderr %s", err, stdErr.String()) - } - - pid := process.GetManager().Add(fmt.Sprintf("diffIndex [repo_path: %s]", repo.RepoPath()), cmd) - defer process.GetManager().Remove(pid) - - diff, err = ParsePatch(setting.Git.MaxGitDiffLines, setting.Git.MaxGitDiffLineCharacters, setting.Git.MaxGitDiffFiles, stdout) - if err != nil { - return nil, fmt.Errorf("ParsePatch: %v", err) - } - - if err = cmd.Wait(); err != nil { - return nil, fmt.Errorf("Wait: %v", err) - } - - return diff, nil -} - -// GetDiffPreview produces and returns diff result of a file which is not yet committed. -func (repo *Repository) GetDiffPreview(branch, treePath, content string) (diff *Diff, err error) { - timeStr := com.ToStr(time.Now().Nanosecond()) // SHOULD USE SOMETHING UNIQUE - tmpBasePath := path.Join(LocalCopyPath(), "upload-"+timeStr+".git") - if err := os.MkdirAll(path.Dir(tmpBasePath), os.ModePerm); err != nil { - return nil, fmt.Errorf("Failed to create dir %s: %v", tmpBasePath, err) - } - - defer os.RemoveAll(path.Dir(tmpBasePath)) - - // Do a bare shared clone into tmpBasePath and - // make HEAD to point to the branch tree - if err := repo.bareClone(tmpBasePath, branch); err != nil { - return nil, fmt.Errorf("GetDiffPreview: %v", err) - } - - // Set the default index - if err := repo.setDefaultIndex(tmpBasePath); err != nil { - return nil, fmt.Errorf("GetDiffPreview: %v", err) - } - - // Add the object to the database - objectHash, err := repo.hashObject(tmpBasePath, strings.NewReader(content)) - if err != nil { - return nil, fmt.Errorf("GetDiffPreview: %v", err) - } - - // Add the object to the index - if err := repo.addObjectToIndex(tmpBasePath, "100644", objectHash, treePath); err != nil { - return nil, fmt.Errorf("GetDiffPreview: %v", err) - } - - return repo.diffIndex(tmpBasePath) -} - -// ________ .__ __ ___________.__.__ -// \______ \ ____ | | _____/ |_ ____ \_ _____/|__| | ____ -// | | \_/ __ \| | _/ __ \ __\/ __ \ | __) | | | _/ __ \ -// | ` \ ___/| |_\ ___/| | \ ___/ | \ | | |_\ ___/ -// /_______ /\___ >____/\___ >__| \___ > \___ / |__|____/\___ > -// \/ \/ \/ \/ \/ \/ -// - -// DeleteRepoFileOptions holds the repository delete file options -type DeleteRepoFileOptions struct { - LastCommitID string - OldBranch string - NewBranch string - TreePath string - Message string -} - -// DeleteRepoFile deletes a repository file -func (repo *Repository) DeleteRepoFile(doer *User, opts DeleteRepoFileOptions) (err error) { - timeStr := com.ToStr(time.Now().Nanosecond()) // SHOULD USE SOMETHING UNIQUE - tmpBasePath := path.Join(LocalCopyPath(), "upload-"+timeStr+".git") - if err := os.MkdirAll(path.Dir(tmpBasePath), os.ModePerm); err != nil { - return fmt.Errorf("Failed to create dir %s: %v", tmpBasePath, err) - } - - defer os.RemoveAll(path.Dir(tmpBasePath)) - - // Do a bare shared clone into tmpBasePath and - // make HEAD to point to the OldBranch tree - if err := repo.bareClone(tmpBasePath, opts.OldBranch); err != nil { - return fmt.Errorf("DeleteRepoFile: %v", err) - } - - // Set the default index - if err := repo.setDefaultIndex(tmpBasePath); err != nil { - return fmt.Errorf("DeleteRepoFile: %v", err) - } - - filelist, err := repo.lsFiles(tmpBasePath, opts.TreePath) - if err != nil { - return fmt.Errorf("DeleteRepoFile: %v", err) - } - - inFilelist := false - for _, file := range filelist { - if file == opts.TreePath { - inFilelist = true - } - } - if !inFilelist { - return git.ErrNotExist{RelPath: opts.TreePath} - } - - if err := repo.removeFilesFromIndex(tmpBasePath, opts.TreePath); err != nil { - return fmt.Errorf("DeleteRepoFile: %v", err) - } - - // Now write the tree - treeHash, err := repo.writeTree(tmpBasePath) - if err != nil { - return fmt.Errorf("DeleteRepoFile: %v", err) - } - - // Now commit the tree - commitHash, err := repo.commitTree(tmpBasePath, doer, treeHash, opts.Message) - if err != nil { - return fmt.Errorf("DeleteRepoFile: %v", err) - } - - // Then push this tree to NewBranch - if err := repo.actuallyPush(tmpBasePath, doer, commitHash, opts.NewBranch); err != nil { - return fmt.Errorf("DeleteRepoFile: %v", err) - } - - // Simulate push event. - oldCommitID := opts.LastCommitID - if opts.NewBranch != opts.OldBranch { - oldCommitID = git.EmptySHA - } - - if err = repo.GetOwner(); err != nil { - return fmt.Errorf("GetOwner: %v", err) - } - err = PushUpdate( - opts.NewBranch, - PushUpdateOptions{ - PusherID: doer.ID, - PusherName: doer.Name, - RepoUserName: repo.Owner.Name, - RepoName: repo.Name, - RefFullName: git.BranchPrefix + opts.NewBranch, - OldCommitID: oldCommitID, - NewCommitID: commitHash, - }, - ) - if err != nil { - return fmt.Errorf("PushUpdate: %v", err) - } - - // FIXME: Should we UpdateRepoIndexer(repo) here? - return nil -} - -// ____ ___ .__ .___ ___________.___.__ -// | | \______ | | _________ __| _/ \_ _____/| | | ____ ______ -// | | /\____ \| | / _ \__ \ / __ | | __) | | | _/ __ \ / ___/ -// | | / | |_> > |_( <_> ) __ \_/ /_/ | | \ | | |_\ ___/ \___ \ -// |______/ | __/|____/\____(____ /\____ | \___ / |___|____/\___ >____ > -// |__| \/ \/ \/ \/ \/ -// - -// Upload represent a uploaded file to a repo to be deleted when moved -type Upload struct { - ID int64 `xorm:"pk autoincr"` - UUID string `xorm:"uuid UNIQUE"` - Name string -} - -// UploadLocalPath returns where uploads is stored in local file system based on given UUID. -func UploadLocalPath(uuid string) string { - return path.Join(setting.Repository.Upload.TempPath, uuid[0:1], uuid[1:2], uuid) -} - -// LocalPath returns where uploads are temporarily stored in local file system. -func (upload *Upload) LocalPath() string { - return UploadLocalPath(upload.UUID) -} - -// NewUpload creates a new upload object. -func NewUpload(name string, buf []byte, file multipart.File) (_ *Upload, err error) { - upload := &Upload{ - UUID: gouuid.NewV4().String(), - Name: name, - } - - localPath := upload.LocalPath() - if err = os.MkdirAll(path.Dir(localPath), os.ModePerm); err != nil { - return nil, fmt.Errorf("MkdirAll: %v", err) - } - - fw, err := os.Create(localPath) - if err != nil { - return nil, fmt.Errorf("Create: %v", err) - } - defer fw.Close() - - if _, err = fw.Write(buf); err != nil { - return nil, fmt.Errorf("Write: %v", err) - } else if _, err = io.Copy(fw, file); err != nil { - return nil, fmt.Errorf("Copy: %v", err) - } - - if _, err := x.Insert(upload); err != nil { - return nil, err - } - - return upload, nil -} - -// GetUploadByUUID returns the Upload by UUID -func GetUploadByUUID(uuid string) (*Upload, error) { - upload := &Upload{UUID: uuid} - has, err := x.Get(upload) - if err != nil { - return nil, err - } else if !has { - return nil, ErrUploadNotExist{0, uuid} - } - return upload, nil -} - -// GetUploadsByUUIDs returns multiple uploads by UUIDS -func GetUploadsByUUIDs(uuids []string) ([]*Upload, error) { - if len(uuids) == 0 { - return []*Upload{}, nil - } - - // Silently drop invalid uuids. - uploads := make([]*Upload, 0, len(uuids)) - return uploads, x.In("uuid", uuids).Find(&uploads) -} - -// DeleteUploads deletes multiple uploads -func DeleteUploads(uploads ...*Upload) (err error) { - if len(uploads) == 0 { - return nil - } - - sess := x.NewSession() - defer sess.Close() - if err = sess.Begin(); err != nil { - return err - } - - ids := make([]int64, len(uploads)) - for i := 0; i < len(uploads); i++ { - ids[i] = uploads[i].ID - } - if _, err = sess. - In("id", ids). - Delete(new(Upload)); err != nil { - return fmt.Errorf("delete uploads: %v", err) - } - - for _, upload := range uploads { - localPath := upload.LocalPath() - if !com.IsFile(localPath) { - continue - } - - if err := os.Remove(localPath); err != nil { - return fmt.Errorf("remove upload: %v", err) - } - } - - return sess.Commit() -} - -// DeleteUpload delete a upload -func DeleteUpload(u *Upload) error { - return DeleteUploads(u) -} - -// DeleteUploadByUUID deletes a upload by UUID -func DeleteUploadByUUID(uuid string) error { - upload, err := GetUploadByUUID(uuid) - if err != nil { - if IsErrUploadNotExist(err) { - return nil - } - return fmt.Errorf("GetUploadByUUID: %v", err) - } - - if err := DeleteUpload(upload); err != nil { - return fmt.Errorf("DeleteUpload: %v", err) - } - - return nil -} - -// UploadRepoFileOptions contains the uploaded repository file options -type UploadRepoFileOptions struct { - LastCommitID string - OldBranch string - NewBranch string - TreePath string - Message string - Files []string // In UUID format. -} - -// UploadRepoFiles uploads files to a repository -func (repo *Repository) UploadRepoFiles(doer *User, opts UploadRepoFileOptions) (err error) { - if len(opts.Files) == 0 { - return nil - } - - uploads, err := GetUploadsByUUIDs(opts.Files) - if err != nil { - return fmt.Errorf("GetUploadsByUUIDs [uuids: %v]: %v", opts.Files, err) - } - - timeStr := com.ToStr(time.Now().Nanosecond()) // SHOULD USE SOMETHING UNIQUE - tmpBasePath := path.Join(LocalCopyPath(), "upload-"+timeStr+".git") - if err := os.MkdirAll(path.Dir(tmpBasePath), os.ModePerm); err != nil { - return fmt.Errorf("Failed to create dir %s: %v", tmpBasePath, err) - } - - defer os.RemoveAll(path.Dir(tmpBasePath)) - - // Do a bare shared clone into tmpBasePath and - // make HEAD to point to the OldBranch tree - if err := repo.bareClone(tmpBasePath, opts.OldBranch); err != nil { - return fmt.Errorf("UpdateRepoFiles: %v", err) - } - - // Set the default index - if err := repo.setDefaultIndex(tmpBasePath); err != nil { - return fmt.Errorf("UpdateRepoFiles: %v", err) - } - - // Copy uploaded files into repository. - for _, upload := range uploads { - file, err := os.Open(upload.LocalPath()) - if err != nil { - return err - } - defer file.Close() - - objectHash, err := repo.hashObject(tmpBasePath, file) - if err != nil { - return err - } - - // Add the object to the index - if err := repo.addObjectToIndex(tmpBasePath, "100644", objectHash, path.Join(opts.TreePath, upload.Name)); err != nil { - return err - } - } - - // Now write the tree - treeHash, err := repo.writeTree(tmpBasePath) - if err != nil { - return err - } - - // Now commit the tree - commitHash, err := repo.commitTree(tmpBasePath, doer, treeHash, opts.Message) - if err != nil { - return err - } - - // Then push this tree to NewBranch - if err := repo.actuallyPush(tmpBasePath, doer, commitHash, opts.NewBranch); err != nil { - return err - } - - // Simulate push event. - oldCommitID := opts.LastCommitID - if opts.NewBranch != opts.OldBranch { - oldCommitID = git.EmptySHA - } - - if err = repo.GetOwner(); err != nil { - return fmt.Errorf("GetOwner: %v", err) - } - err = PushUpdate( - opts.NewBranch, - PushUpdateOptions{ - PusherID: doer.ID, - PusherName: doer.Name, - RepoUserName: repo.Owner.Name, - RepoName: repo.Name, - RefFullName: git.BranchPrefix + opts.NewBranch, - OldCommitID: oldCommitID, - NewCommitID: commitHash, - }, - ) - if err != nil { - return fmt.Errorf("PushUpdate: %v", err) - } - - // FIXME: Should we UpdateRepoIndexer(repo) here? - return DeleteUploads(uploads...) -} diff --git a/models/upload.go b/models/upload.go new file mode 100644 index 000000000000..e015a916f48b --- /dev/null +++ b/models/upload.go @@ -0,0 +1,155 @@ +// Copyright 2016 The Gogs Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package models + +import ( + "fmt" + "io" + "mime/multipart" + "os" + "path" + + "github.com/Unknwon/com" + gouuid "github.com/satori/go.uuid" + + "code.gitea.io/gitea/modules/setting" +) + +// ____ ___ .__ .___ ___________.___.__ +// | | \______ | | _________ __| _/ \_ _____/| | | ____ ______ +// | | /\____ \| | / _ \__ \ / __ | | __) | | | _/ __ \ / ___/ +// | | / | |_> > |_( <_> ) __ \_/ /_/ | | \ | | |_\ ___/ \___ \ +// |______/ | __/|____/\____(____ /\____ | \___ / |___|____/\___ >____ > +// |__| \/ \/ \/ \/ \/ +// + +// Upload represent a uploaded file to a repo to be deleted when moved +type Upload struct { + ID int64 `xorm:"pk autoincr"` + UUID string `xorm:"uuid UNIQUE"` + Name string +} + +// UploadLocalPath returns where uploads is stored in local file system based on given UUID. +func UploadLocalPath(uuid string) string { + return path.Join(setting.Repository.Upload.TempPath, uuid[0:1], uuid[1:2], uuid) +} + +// LocalPath returns where uploads are temporarily stored in local file system. +func (upload *Upload) LocalPath() string { + return UploadLocalPath(upload.UUID) +} + +// NewUpload creates a new upload object. +func NewUpload(name string, buf []byte, file multipart.File) (_ *Upload, err error) { + upload := &Upload{ + UUID: gouuid.NewV4().String(), + Name: name, + } + + localPath := upload.LocalPath() + if err = os.MkdirAll(path.Dir(localPath), os.ModePerm); err != nil { + return nil, fmt.Errorf("MkdirAll: %v", err) + } + + fw, err := os.Create(localPath) + if err != nil { + return nil, fmt.Errorf("Create: %v", err) + } + defer fw.Close() + + if _, err = fw.Write(buf); err != nil { + return nil, fmt.Errorf("Write: %v", err) + } else if _, err = io.Copy(fw, file); err != nil { + return nil, fmt.Errorf("Copy: %v", err) + } + + if _, err := x.Insert(upload); err != nil { + return nil, err + } + + return upload, nil +} + +// GetUploadByUUID returns the Upload by UUID +func GetUploadByUUID(uuid string) (*Upload, error) { + upload := &Upload{UUID: uuid} + has, err := x.Get(upload) + if err != nil { + return nil, err + } else if !has { + return nil, ErrUploadNotExist{0, uuid} + } + return upload, nil +} + +// GetUploadsByUUIDs returns multiple uploads by UUIDS +func GetUploadsByUUIDs(uuids []string) ([]*Upload, error) { + if len(uuids) == 0 { + return []*Upload{}, nil + } + + // Silently drop invalid uuids. + uploads := make([]*Upload, 0, len(uuids)) + return uploads, x.In("uuid", uuids).Find(&uploads) +} + +// DeleteUploads deletes multiple uploads +func DeleteUploads(uploads ...*Upload) (err error) { + if len(uploads) == 0 { + return nil + } + + sess := x.NewSession() + defer sess.Close() + if err = sess.Begin(); err != nil { + return err + } + + ids := make([]int64, len(uploads)) + for i := 0; i < len(uploads); i++ { + ids[i] = uploads[i].ID + } + if _, err = sess. + In("id", ids). + Delete(new(Upload)); err != nil { + return fmt.Errorf("delete uploads: %v", err) + } + + for _, upload := range uploads { + localPath := upload.LocalPath() + if !com.IsFile(localPath) { + continue + } + + if err := os.Remove(localPath); err != nil { + return fmt.Errorf("remove upload: %v", err) + } + } + + return sess.Commit() +} + +// DeleteUpload delete a upload +func DeleteUpload(u *Upload) error { + return DeleteUploads(u) +} + +// DeleteUploadByUUID deletes a upload by UUID +func DeleteUploadByUUID(uuid string) error { + upload, err := GetUploadByUUID(uuid) + if err != nil { + if IsErrUploadNotExist(err) { + return nil + } + return fmt.Errorf("GetUploadByUUID: %v", err) + } + + if err := DeleteUpload(upload); err != nil { + return fmt.Errorf("DeleteUpload: %v", err) + } + + return nil +} diff --git a/modules/uploader/delete.go b/modules/uploader/delete.go new file mode 100644 index 000000000000..cf978b85cf10 --- /dev/null +++ b/modules/uploader/delete.go @@ -0,0 +1,100 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package uploader + +import ( + "fmt" + + "code.gitea.io/git" + "code.gitea.io/gitea/models" +) + +// DeleteRepoFileOptions holds the repository delete file options +type DeleteRepoFileOptions struct { + LastCommitID string + OldBranch string + NewBranch string + TreePath string + Message string +} + +// DeleteRepoFile deletes a file in the given repository +func DeleteRepoFile(repo *models.Repository, doer *models.User, opts DeleteRepoFileOptions) error { + t, err := NewTemporaryUploadRepository(repo) + defer t.Close() + if err != nil { + return err + } + if err := t.Clone(opts.OldBranch); err != nil { + return err + } + if err := t.SetDefaultIndex(); err != nil { + return err + } + + filesInIndex, err := t.LsFiles(opts.TreePath) + if err != nil { + return fmt.Errorf("UpdateRepoFile: %v", err) + } + + inFilelist := false + for _, file := range filesInIndex { + if file == opts.TreePath { + inFilelist = true + } + } + if !inFilelist { + return git.ErrNotExist{RelPath: opts.TreePath} + } + + if err := t.RemoveFilesFromIndex(opts.TreePath); err != nil { + return err + } + + // Now write the tree + treeHash, err := t.WriteTree() + if err != nil { + return err + } + + // Now commit the tree + commitHash, err := t.CommitTree(doer, treeHash, opts.Message) + if err != nil { + return err + } + + // Then push this tree to NewBranch + if err := t.Push(doer, commitHash, opts.NewBranch); err != nil { + return err + } + + // Simulate push event. + oldCommitID := opts.LastCommitID + if opts.NewBranch != opts.OldBranch { + oldCommitID = git.EmptySHA + } + + if err = repo.GetOwner(); err != nil { + return fmt.Errorf("GetOwner: %v", err) + } + err = models.PushUpdate( + opts.NewBranch, + models.PushUpdateOptions{ + PusherID: doer.ID, + PusherName: doer.Name, + RepoUserName: repo.Owner.Name, + RepoName: repo.Name, + RefFullName: git.BranchPrefix + opts.NewBranch, + OldCommitID: oldCommitID, + NewCommitID: commitHash, + }, + ) + if err != nil { + return fmt.Errorf("PushUpdate: %v", err) + } + + // FIXME: Should we UpdateRepoIndexer(repo) here? + return nil +} diff --git a/modules/uploader/diff.go b/modules/uploader/diff.go new file mode 100644 index 000000000000..e01947ea6127 --- /dev/null +++ b/modules/uploader/diff.go @@ -0,0 +1,38 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package uploader + +import ( + "strings" + + "code.gitea.io/gitea/models" +) + +// GetDiffPreview produces and returns diff result of a file which is not yet committed. +func GetDiffPreview(repo *models.Repository, branch, treePath, content string) (*models.Diff, error) { + t, err := NewTemporaryUploadRepository(repo) + defer t.Close() + if err != nil { + return nil, err + } + if err := t.Clone(branch); err != nil { + return nil, err + } + if err := t.SetDefaultIndex(); err != nil { + return nil, err + } + + // Add the object to the database + objectHash, err := t.HashObject(strings.NewReader(content)) + if err != nil { + return nil, err + } + + // Add the object to the index + if err := t.AddObjectToIndex("100644", objectHash, treePath); err != nil { + return nil, err + } + return t.DiffIndex() +} diff --git a/modules/uploader/repo.go b/modules/uploader/repo.go new file mode 100644 index 000000000000..93c10db791ce --- /dev/null +++ b/modules/uploader/repo.go @@ -0,0 +1,297 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package uploader + +import ( + "bytes" + "context" + "fmt" + "io" + "os" + "os/exec" + "path" + "strings" + "time" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/process" + "code.gitea.io/gitea/modules/setting" + "github.com/Unknwon/com" +) + +// TemporaryUploadRepository is a type to wrap our upload repositories +type TemporaryUploadRepository struct { + repo *models.Repository + basePath string +} + +// NewTemporaryUploadRepository creates a new temporary upload repository +func NewTemporaryUploadRepository(repo *models.Repository) (*TemporaryUploadRepository, error) { + timeStr := com.ToStr(time.Now().Nanosecond()) // SHOULD USE SOMETHING UNIQUE + basePath := path.Join(models.LocalCopyPath(), "upload-"+timeStr+".git") + if err := os.MkdirAll(path.Dir(basePath), os.ModePerm); err != nil { + return nil, fmt.Errorf("Failed to create dir %s: %v", basePath, err) + } + t := &TemporaryUploadRepository{repo: repo, basePath: basePath} + return t, nil +} + +// Close the repository cleaning up all files +func (t *TemporaryUploadRepository) Close() { + if _, err := os.Stat(t.basePath); !os.IsNotExist(err) { + os.RemoveAll(t.basePath) + } +} + +// Clone the base repository to our path and set branch as the HEAD +func (t *TemporaryUploadRepository) Clone(branch string) error { + if _, stderr, err := process.GetManager().ExecTimeout(5*time.Minute, + fmt.Sprintf("Clone (git clone -s --bare): %s", t.basePath), + "git", "clone", "-s", "--bare", "-b", branch, t.repo.RepoPath(), t.basePath); err != nil { + return fmt.Errorf("Clone: %v %s", err, stderr) + } + return nil +} + +// SetDefaultIndex sets the git index to our HEAD +func (t *TemporaryUploadRepository) SetDefaultIndex() error { + if _, stderr, err := process.GetManager().ExecDir(5*time.Minute, + t.basePath, + fmt.Sprintf("SetDefaultIndex (git read-tree HEAD): %s", t.basePath), + "git", "read-tree", "HEAD"); err != nil { + return fmt.Errorf("SetDefaultIndex: %v %s", err, stderr) + } + return nil +} + +// LsFiles checks if the given filename arguments are in the index +func (t *TemporaryUploadRepository) LsFiles(filenames ...string) ([]string, error) { + stdOut := new(bytes.Buffer) + stdErr := new(bytes.Buffer) + + timeout := 5 * time.Minute + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + cmdArgs := []string{"ls-files", "-z", "--"} + for _, arg := range filenames { + if arg != "" { + cmdArgs = append(cmdArgs, arg) + } + } + + cmd := exec.CommandContext(ctx, "git", cmdArgs...) + desc := fmt.Sprintf("lsFiles: (git ls-files) %v", cmdArgs) + cmd.Dir = t.basePath + cmd.Stdout = stdOut + cmd.Stderr = stdErr + + if err := cmd.Start(); err != nil { + return nil, fmt.Errorf("exec(%s) failed: %v(%v)", desc, err, ctx.Err()) + } + + pid := process.GetManager().Add(desc, cmd) + err := cmd.Wait() + process.GetManager().Remove(pid) + + if err != nil { + err = fmt.Errorf("exec(%d:%s) failed: %v(%v) stdout: %v stderr: %v", pid, desc, err, ctx.Err(), stdOut, stdErr) + return nil, err + } + + filelist := make([]string, len(filenames)) + for _, line := range bytes.Split(stdOut.Bytes(), []byte{'\000'}) { + filelist = append(filelist, string(line)) + } + + return filelist, err +} + +// RemoveFilesFromIndex removes the given files from the index +func (t *TemporaryUploadRepository) RemoveFilesFromIndex(filenames ...string) error { + stdOut := new(bytes.Buffer) + stdErr := new(bytes.Buffer) + stdIn := new(bytes.Buffer) + for _, file := range filenames { + if file != "" { + stdIn.WriteString("0 0000000000000000000000000000000000000000\t") + stdIn.WriteString(file) + stdIn.WriteByte('\000') + } + } + + timeout := 5 * time.Minute + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + cmdArgs := []string{"update-index", "--remove", "-z", "--index-info"} + cmd := exec.CommandContext(ctx, "git", cmdArgs...) + desc := fmt.Sprintf("removeFilesFromIndex: (git update-index) %v", filenames) + cmd.Dir = t.basePath + cmd.Stdout = stdOut + cmd.Stderr = stdErr + cmd.Stdin = bytes.NewReader(stdIn.Bytes()) + + if err := cmd.Start(); err != nil { + return fmt.Errorf("exec(%s) failed: %v(%v)", desc, err, ctx.Err()) + } + + pid := process.GetManager().Add(desc, cmd) + err := cmd.Wait() + process.GetManager().Remove(pid) + + if err != nil { + err = fmt.Errorf("exec(%d:%s) failed: %v(%v) stdout: %v stderr: %v", pid, desc, err, ctx.Err(), stdOut, stdErr) + } + + return err +} + +// HashObject writes the provided content to the object db and returns its hash +func (t *TemporaryUploadRepository) HashObject(content io.Reader) (string, error) { + timeout := 5 * time.Minute + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + hashCmd := exec.CommandContext(ctx, "git", "hash-object", "-w", "--stdin") + hashCmd.Dir = t.basePath + hashCmd.Stdin = content + stdOutBuffer := new(bytes.Buffer) + stdErrBuffer := new(bytes.Buffer) + hashCmd.Stdout = stdOutBuffer + hashCmd.Stderr = stdErrBuffer + desc := fmt.Sprintf("hashObject: (git hash-object)") + if err := hashCmd.Start(); err != nil { + return "", fmt.Errorf("git hash-object: %s", err) + } + + pid := process.GetManager().Add(desc, hashCmd) + err := hashCmd.Wait() + process.GetManager().Remove(pid) + + if err != nil { + err = fmt.Errorf("exec(%d:%s) failed: %v(%v) stdout: %v stderr: %v", pid, desc, err, ctx.Err(), stdOutBuffer, stdErrBuffer) + return "", err + } + + return strings.TrimSpace(stdOutBuffer.String()), nil +} + +// AddObjectToIndex adds the provided object hash to the index with the provided mode and path +func (t *TemporaryUploadRepository) AddObjectToIndex(mode, objectHash, objectPath string) error { + if _, stderr, err := process.GetManager().ExecDir(5*time.Minute, + t.basePath, + fmt.Sprintf("addObjectToIndex (git update-index): %s", t.basePath), + "git", "update-index", "--add", "--replace", "--cacheinfo", mode, objectHash, objectPath); err != nil { + return fmt.Errorf("git update-index: %s", stderr) + } + return nil +} + +// WriteTree writes the current index as a tree to the object db and returns its hash +func (t *TemporaryUploadRepository) WriteTree() (string, error) { + treeHash, stderr, err := process.GetManager().ExecDir(5*time.Minute, + t.basePath, + fmt.Sprintf("WriteTree (git write-tree): %s", t.basePath), + "git", "write-tree") + if err != nil { + return "", fmt.Errorf("git write-tree: %s", stderr) + } + return strings.TrimSpace(treeHash), nil + +} + +// CommitTree creates a commit from a given tree for the user with provided message +func (t *TemporaryUploadRepository) CommitTree(doer *models.User, treeHash string, message string) (string, error) { + commitTimeStr := time.Now().Format(time.UnixDate) + + // FIXME: Should we add SSH_ORIGINAL_COMMAND to this + // Because this may call hooks we should pass in the environment + env := append(os.Environ(), + "GIT_AUTHOR_NAME="+doer.DisplayName(), + "GIT_AUTHOR_EMAIL="+doer.Email, + "GIT_AUTHOR_DATE="+commitTimeStr, + "GIT_COMMITTER_NAME="+doer.DisplayName(), + "GIT_COMMITTER_EMAIL="+doer.Email, + "GIT_COMMITTER_DATE="+commitTimeStr, + ) + commitHash, stderr, err := process.GetManager().ExecDirEnv(5*time.Minute, + t.basePath, + fmt.Sprintf("commitTree (git commit-tree): %s", t.basePath), + env, + "git", "commit-tree", treeHash, "-p", "HEAD", "-m", message) + if err != nil { + return "", fmt.Errorf("git commit-tree: %s", stderr) + } + return strings.TrimSpace(commitHash), nil +} + +// Push the provided commitHash to the repository branch by the provided user +func (t *TemporaryUploadRepository) Push(doer *models.User, commitHash string, branch string) error { + isWiki := "false" + if strings.HasSuffix(t.repo.Name, ".wiki") { + isWiki = "true" + } + + // FIXME: Should we add SSH_ORIGINAL_COMMAND to this + // Because calls hooks we need to pass in the environment + env := append(os.Environ(), + "GIT_AUTHOR_NAME="+doer.DisplayName(), + "GIT_AUTHOR_EMAIL="+doer.Email, + "GIT_COMMITTER_NAME="+doer.DisplayName(), + "GIT_COMMITTER_EMAIL="+doer.Email, + models.EnvRepoName+"="+t.repo.Name, + models.EnvRepoUsername+"="+t.repo.OwnerName, + models.EnvRepoIsWiki+"="+isWiki, + models.EnvPusherName+"="+doer.Name, + models.EnvPusherID+"="+fmt.Sprintf("%d", doer.ID), + models.ProtectedBranchRepoID+"="+fmt.Sprintf("%d", t.repo.ID), + ) + + if _, stderr, err := process.GetManager().ExecDirEnv(5*time.Minute, + t.basePath, + fmt.Sprintf("actuallyPush (git push): %s", t.basePath), + env, + "git", "push", t.repo.RepoPath(), strings.TrimSpace(commitHash)+":refs/heads/"+strings.TrimSpace(branch)); err != nil { + return fmt.Errorf("git push: %s", stderr) + } + return nil +} + +// DiffIndex returns a Diff of the current index to the head +func (t *TemporaryUploadRepository) DiffIndex() (diff *models.Diff, err error) { + timeout := 5 * time.Minute + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + stdErr := new(bytes.Buffer) + + cmd := exec.CommandContext(ctx, "git", "diff-index", "--cached", "-p", "HEAD") + cmd.Dir = t.basePath + cmd.Stderr = stdErr + + stdout, err := cmd.StdoutPipe() + if err != nil { + return nil, fmt.Errorf("StdoutPipe: %v stderr %s", err, stdErr.String()) + } + + if err = cmd.Start(); err != nil { + return nil, fmt.Errorf("Start: %v stderr %s", err, stdErr.String()) + } + + pid := process.GetManager().Add(fmt.Sprintf("diffIndex [repo_path: %s]", t.repo.RepoPath()), cmd) + defer process.GetManager().Remove(pid) + + diff, err = models.ParsePatch(setting.Git.MaxGitDiffLines, setting.Git.MaxGitDiffLineCharacters, setting.Git.MaxGitDiffFiles, stdout) + if err != nil { + return nil, fmt.Errorf("ParsePatch: %v", err) + } + + if err = cmd.Wait(); err != nil { + return nil, fmt.Errorf("Wait: %v", err) + } + + return diff, nil +} diff --git a/modules/uploader/update.go b/modules/uploader/update.go new file mode 100644 index 000000000000..d426458c756f --- /dev/null +++ b/modules/uploader/update.go @@ -0,0 +1,121 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package uploader + +import ( + "fmt" + "strings" + + "code.gitea.io/git" + "code.gitea.io/gitea/models" +) + +// UpdateRepoFileOptions holds the repository file update options +type UpdateRepoFileOptions struct { + LastCommitID string + OldBranch string + NewBranch string + OldTreeName string + NewTreeName string + Message string + Content string + IsNewFile bool +} + +// UpdateRepoFile adds or updates a file in the given repository +func UpdateRepoFile(repo *models.Repository, doer *models.User, opts UpdateRepoFileOptions) error { + t, err := NewTemporaryUploadRepository(repo) + defer t.Close() + if err != nil { + return err + } + if err := t.Clone(opts.OldBranch); err != nil { + return err + } + if err := t.SetDefaultIndex(); err != nil { + return err + } + + filesInIndex, err := t.LsFiles(opts.NewTreeName, opts.OldTreeName) + if err != nil { + return fmt.Errorf("UpdateRepoFile: %v", err) + } + + if opts.IsNewFile { + for _, file := range filesInIndex { + if file == opts.NewTreeName { + return models.ErrRepoFileAlreadyExist{FileName: opts.NewTreeName} + } + } + } + + //var stdout string + if opts.OldTreeName != opts.NewTreeName && len(filesInIndex) > 0 { + for _, file := range filesInIndex { + if file == opts.OldTreeName { + if err := t.RemoveFilesFromIndex(opts.OldTreeName); err != nil { + return err + } + } + } + + } + + // Add the object to the database + objectHash, err := t.HashObject(strings.NewReader(opts.Content)) + if err != nil { + return err + } + + // Add the object to the index + if err := t.AddObjectToIndex("100644", objectHash, opts.NewTreeName); err != nil { + return err + } + + // Now write the tree + treeHash, err := t.WriteTree() + if err != nil { + return err + } + + // Now commit the tree + commitHash, err := t.CommitTree(doer, treeHash, opts.Message) + if err != nil { + return err + } + + // Then push this tree to NewBranch + if err := t.Push(doer, commitHash, opts.NewBranch); err != nil { + return err + } + + // Simulate push event. + oldCommitID := opts.LastCommitID + if opts.NewBranch != opts.OldBranch { + oldCommitID = git.EmptySHA + } + + if err = repo.GetOwner(); err != nil { + return fmt.Errorf("GetOwner: %v", err) + } + err = models.PushUpdate( + opts.NewBranch, + models.PushUpdateOptions{ + PusherID: doer.ID, + PusherName: doer.Name, + RepoUserName: repo.Owner.Name, + RepoName: repo.Name, + RefFullName: git.BranchPrefix + opts.NewBranch, + OldCommitID: oldCommitID, + NewCommitID: commitHash, + }, + ) + if err != nil { + return fmt.Errorf("PushUpdate: %v", err) + } + models.UpdateRepoIndexer(repo) + + return nil +} diff --git a/modules/uploader/upload.go b/modules/uploader/upload.go new file mode 100644 index 000000000000..31dbe8108683 --- /dev/null +++ b/modules/uploader/upload.go @@ -0,0 +1,112 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package uploader + +import ( + "fmt" + "os" + "path" + + "code.gitea.io/git" + "code.gitea.io/gitea/models" +) + +// UploadRepoFileOptions contains the uploaded repository file options +type UploadRepoFileOptions struct { + LastCommitID string + OldBranch string + NewBranch string + TreePath string + Message string + Files []string // In UUID format. +} + +// UploadRepoFiles uploads files to the given repository +func UploadRepoFiles(repo *models.Repository, doer *models.User, opts UploadRepoFileOptions) error { + if len(opts.Files) == 0 { + return nil + } + + uploads, err := models.GetUploadsByUUIDs(opts.Files) + if err != nil { + return fmt.Errorf("GetUploadsByUUIDs [uuids: %v]: %v", opts.Files, err) + } + + t, err := NewTemporaryUploadRepository(repo) + defer t.Close() + if err != nil { + return err + } + if err := t.Clone(opts.OldBranch); err != nil { + return err + } + if err := t.SetDefaultIndex(); err != nil { + return err + } + + // Copy uploaded files into repository. + for _, upload := range uploads { + file, err := os.Open(upload.LocalPath()) + if err != nil { + return err + } + defer file.Close() + + objectHash, err := t.HashObject(file) + if err != nil { + return err + } + + // Add the object to the index + if err := t.AddObjectToIndex("100644", objectHash, path.Join(opts.TreePath, upload.Name)); err != nil { + return err + } + } + + // Now write the tree + treeHash, err := t.WriteTree() + if err != nil { + return err + } + + // Now commit the tree + commitHash, err := t.CommitTree(doer, treeHash, opts.Message) + if err != nil { + return err + } + + // Then push this tree to NewBranch + if err := t.Push(doer, commitHash, opts.NewBranch); err != nil { + return err + } + + // Simulate push event. + oldCommitID := opts.LastCommitID + if opts.NewBranch != opts.OldBranch { + oldCommitID = git.EmptySHA + } + + if err = repo.GetOwner(); err != nil { + return fmt.Errorf("GetOwner: %v", err) + } + err = models.PushUpdate( + opts.NewBranch, + models.PushUpdateOptions{ + PusherID: doer.ID, + PusherName: doer.Name, + RepoUserName: repo.Owner.Name, + RepoName: repo.Name, + RefFullName: git.BranchPrefix + opts.NewBranch, + OldCommitID: oldCommitID, + NewCommitID: commitHash, + }, + ) + if err != nil { + return fmt.Errorf("PushUpdate: %v", err) + } + // FIXME: Should we models.UpdateRepoIndexer(repo) here? + + return models.DeleteUploads(uploads...) +} diff --git a/routers/repo/editor.go b/routers/repo/editor.go index 4a2eeebea965..7b65397e05bc 100644 --- a/routers/repo/editor.go +++ b/routers/repo/editor.go @@ -19,6 +19,7 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/templates" + "code.gitea.io/gitea/modules/uploader" ) const ( @@ -308,7 +309,7 @@ func editFilePost(ctx *context.Context, form auth.EditRepoFileForm, isNewFile bo message += "\n\n" + form.CommitMessage } - if err := ctx.Repo.Repository.UpdateRepoFile(ctx.User, models.UpdateRepoFileOptions{ + if err := uploader.UpdateRepoFile(ctx.Repo.Repository, ctx.User, uploader.UpdateRepoFileOptions{ LastCommitID: lastCommit, OldBranch: oldBranchName, NewBranch: branchName, @@ -353,7 +354,7 @@ func DiffPreviewPost(ctx *context.Context, form auth.EditPreviewDiffForm) { return } - diff, err := ctx.Repo.Repository.GetDiffPreview(ctx.Repo.BranchName, treePath, form.Content) + diff, err := uploader.GetDiffPreview(ctx.Repo.Repository, ctx.Repo.BranchName, treePath, form.Content) if err != nil { ctx.Error(500, "GetDiffPreview: "+err.Error()) return @@ -447,7 +448,7 @@ func DeleteFilePost(ctx *context.Context, form auth.DeleteRepoFileForm) { message += "\n\n" + form.CommitMessage } - if err := ctx.Repo.Repository.DeleteRepoFile(ctx.User, models.DeleteRepoFileOptions{ + if err := uploader.DeleteRepoFile(ctx.Repo.Repository, ctx.User, uploader.DeleteRepoFileOptions{ LastCommitID: ctx.Repo.CommitID, OldBranch: oldBranchName, NewBranch: branchName, @@ -582,7 +583,7 @@ func UploadFilePost(ctx *context.Context, form auth.UploadRepoFileForm) { message += "\n\n" + form.CommitMessage } - if err := ctx.Repo.Repository.UploadRepoFiles(ctx.User, models.UploadRepoFileOptions{ + if err := uploader.UploadRepoFiles(ctx.Repo.Repository, ctx.User, uploader.UploadRepoFileOptions{ LastCommitID: ctx.Repo.CommitID, OldBranch: oldBranchName, NewBranch: branchName, From 4e6d433559eca388f37aad84d2c15992c1dedb64 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Fri, 11 Jan 2019 18:23:09 +0000 Subject: [PATCH 10/19] Add LFS support Signed-off-by: Andrew Thornton --- models/lfs.go | 19 +++++++ modules/uploader/delete.go | 2 +- modules/uploader/repo.go | 58 ++++++++++++++++++++ modules/uploader/update.go | 42 ++++++++++++++- modules/uploader/upload.go | 108 ++++++++++++++++++++++++++++++++++--- routers/repo/editor.go | 6 +-- 6 files changed, 222 insertions(+), 13 deletions(-) diff --git a/models/lfs.go b/models/lfs.go index 711e5b049445..7f1fdb68d70a 100644 --- a/models/lfs.go +++ b/models/lfs.go @@ -1,7 +1,11 @@ package models import ( + "crypto/sha256" + "encoding/hex" "errors" + "fmt" + "io" "code.gitea.io/gitea/modules/util" ) @@ -16,6 +20,11 @@ type LFSMetaObject struct { CreatedUnix util.TimeStamp `xorm:"created"` } +// Pointer returns the string representation of an LFS pointer file +func (m *LFSMetaObject) Pointer() string { + return fmt.Sprintf("%s\n%s%s\nsize %d\n", LFSMetaFileIdentifier, LFSMetaFileOidPrefix, m.Oid, m.Size) +} + // LFSTokenResponse defines the JSON structure in which the JWT token is stored. // This structure is fetched via SSH and passed by the Git LFS client to the server // endpoint for authorization. @@ -67,6 +76,16 @@ func NewLFSMetaObject(m *LFSMetaObject) (*LFSMetaObject, error) { return m, sess.Commit() } +// GenerateLFSOid generates a Sha256Sum to represent an oid for arbitrary content +func GenerateLFSOid(content io.Reader) (string, error) { + h := sha256.New() + if _, err := io.Copy(h, content); err != nil { + return "", err + } + sum := h.Sum(nil) + return hex.EncodeToString(sum), nil +} + // GetLFSMetaObjectByOid selects a LFSMetaObject entry from database by its OID. // It may return ErrLFSObjectNotExist or a database error. If the error is nil, // the returned pointer is a valid LFSMetaObject. diff --git a/modules/uploader/delete.go b/modules/uploader/delete.go index cf978b85cf10..fbe451c5d065 100644 --- a/modules/uploader/delete.go +++ b/modules/uploader/delete.go @@ -21,7 +21,7 @@ type DeleteRepoFileOptions struct { } // DeleteRepoFile deletes a file in the given repository -func DeleteRepoFile(repo *models.Repository, doer *models.User, opts DeleteRepoFileOptions) error { +func DeleteRepoFile(repo *models.Repository, doer *models.User, opts *DeleteRepoFileOptions) error { t, err := NewTemporaryUploadRepository(repo) defer t.Close() if err != nil { diff --git a/modules/uploader/repo.go b/modules/uploader/repo.go index 93c10db791ce..03914c162f12 100644 --- a/modules/uploader/repo.go +++ b/modules/uploader/repo.go @@ -295,3 +295,61 @@ func (t *TemporaryUploadRepository) DiffIndex() (diff *models.Diff, err error) { return diff, nil } + +// CheckAttribute checks the given attribute of the provided files +func (t *TemporaryUploadRepository) CheckAttribute(attribute string, args ...string) (map[string]map[string]string, error) { + stdOut := new(bytes.Buffer) + stdErr := new(bytes.Buffer) + + timeout := 5 * time.Minute + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + cmdArgs := []string{"check-attr", "-z", attribute, "--cached", "--"} + for _, arg := range args { + if arg != "" { + cmdArgs = append(cmdArgs, arg) + } + } + + cmd := exec.CommandContext(ctx, "git", cmdArgs...) + desc := fmt.Sprintf("checkAttr: (git check-attr) %s %v", attribute, cmdArgs) + cmd.Dir = t.basePath + cmd.Stdout = stdOut + cmd.Stderr = stdErr + + if err := cmd.Start(); err != nil { + return nil, fmt.Errorf("exec(%s) failed: %v(%v)", desc, err, ctx.Err()) + } + + pid := process.GetManager().Add(desc, cmd) + err := cmd.Wait() + process.GetManager().Remove(pid) + + if err != nil { + err = fmt.Errorf("exec(%d:%s) failed: %v(%v) stdout: %v stderr: %v", pid, desc, err, ctx.Err(), stdOut, stdErr) + return nil, err + } + + fields := bytes.Split(stdOut.Bytes(), []byte{'\000'}) + + if len(fields)%3 != 1 { + return nil, fmt.Errorf("Wrong number of fields in return from check-attr") + } + + var name2attribute2info = make(map[string]map[string]string) + + for i := 0; i < (len(fields) / 3); i++ { + filename := string(fields[3*i]) + attribute := string(fields[3*i+1]) + info := string(fields[3*i+2]) + attribute2info := name2attribute2info[filename] + if attribute2info == nil { + attribute2info = make(map[string]string) + } + attribute2info[attribute] = info + name2attribute2info[filename] = attribute2info + } + + return name2attribute2info, err +} diff --git a/modules/uploader/update.go b/modules/uploader/update.go index d426458c756f..08caf11ee1fa 100644 --- a/modules/uploader/update.go +++ b/modules/uploader/update.go @@ -10,6 +10,8 @@ import ( "code.gitea.io/git" "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/lfs" + "code.gitea.io/gitea/modules/setting" ) // UpdateRepoFileOptions holds the repository file update options @@ -25,7 +27,7 @@ type UpdateRepoFileOptions struct { } // UpdateRepoFile adds or updates a file in the given repository -func UpdateRepoFile(repo *models.Repository, doer *models.User, opts UpdateRepoFileOptions) error { +func UpdateRepoFile(repo *models.Repository, doer *models.User, opts *UpdateRepoFileOptions) error { t, err := NewTemporaryUploadRepository(repo) defer t.Close() if err != nil { @@ -63,8 +65,27 @@ func UpdateRepoFile(repo *models.Repository, doer *models.User, opts UpdateRepoF } + // Check there is no way this can return multiple infos + filename2attribute2info, err := t.CheckAttribute("filter", opts.NewTreeName) + if err != nil { + return err + } + + content := opts.Content + var lfsMetaObject *models.LFSMetaObject + + if filename2attribute2info[opts.NewTreeName] != nil && filename2attribute2info[opts.NewTreeName]["filter"] == "lfs" { + // OK so we are supposed to LFS this data! + oid, err := models.GenerateLFSOid(strings.NewReader(opts.Content)) + if err != nil { + return err + } + lfsMetaObject = &models.LFSMetaObject{Oid: oid, Size: int64(len(opts.Content)), RepositoryID: repo.ID} + content = lfsMetaObject.Pointer() + } + // Add the object to the database - objectHash, err := t.HashObject(strings.NewReader(opts.Content)) + objectHash, err := t.HashObject(strings.NewReader(content)) if err != nil { return err } @@ -86,6 +107,23 @@ func UpdateRepoFile(repo *models.Repository, doer *models.User, opts UpdateRepoF return err } + if lfsMetaObject != nil { + // We have an LFS object - create it + lfsMetaObject, err = models.NewLFSMetaObject(lfsMetaObject) + if err != nil { + return err + } + contentStore := &lfs.ContentStore{BasePath: setting.LFS.ContentPath} + if !contentStore.Exists(lfsMetaObject) { + if err := contentStore.Put(lfsMetaObject, strings.NewReader(opts.Content)); err != nil { + if err2 := repo.RemoveLFSMetaObjectByOid(lfsMetaObject.Oid); err2 != nil { + return fmt.Errorf("Error whilst removing failed inserted LFS object %s: %v (Prev Error: %v)", lfsMetaObject.Oid, err2, err) + } + return err + } + } + } + // Then push this tree to NewBranch if err := t.Push(doer, commitHash, opts.NewBranch); err != nil { return err diff --git a/modules/uploader/upload.go b/modules/uploader/upload.go index 31dbe8108683..bee3f1b9b19f 100644 --- a/modules/uploader/upload.go +++ b/modules/uploader/upload.go @@ -8,6 +8,10 @@ import ( "fmt" "os" "path" + "strings" + + "code.gitea.io/gitea/modules/lfs" + "code.gitea.io/gitea/modules/setting" "code.gitea.io/git" "code.gitea.io/gitea/models" @@ -23,8 +27,27 @@ type UploadRepoFileOptions struct { Files []string // In UUID format. } +type uploadInfo struct { + upload *models.Upload + lfsMetaObject *models.LFSMetaObject +} + +func cleanUpAfterFailure(infos *[]uploadInfo, t *TemporaryUploadRepository, original error) error { + for _, info := range *infos { + if info.lfsMetaObject == nil { + continue + } + if !info.lfsMetaObject.Existing { + if err := t.repo.RemoveLFSMetaObjectByOid(info.lfsMetaObject.Oid); err != nil { + original = fmt.Errorf("%v, %v", original, err) + } + } + } + return original +} + // UploadRepoFiles uploads files to the given repository -func UploadRepoFiles(repo *models.Repository, doer *models.User, opts UploadRepoFileOptions) error { +func UploadRepoFiles(repo *models.Repository, doer *models.User, opts *UploadRepoFileOptions) error { if len(opts.Files) == 0 { return nil } @@ -46,22 +69,56 @@ func UploadRepoFiles(repo *models.Repository, doer *models.User, opts UploadRepo return err } + names := make([]string, len(uploads)) + infos := make([]uploadInfo, len(uploads)) + for i, upload := range uploads { + names[i] = upload.Name + infos[i] = uploadInfo{upload: upload} + } + + filename2attribute2info, err := t.CheckAttribute("filter", names...) + if err != nil { + return err + } + // Copy uploaded files into repository. - for _, upload := range uploads { - file, err := os.Open(upload.LocalPath()) + for i, uploadInfo := range infos { + file, err := os.Open(uploadInfo.upload.LocalPath()) if err != nil { return err } defer file.Close() - objectHash, err := t.HashObject(file) - if err != nil { - return err + var objectHash string + if filename2attribute2info[uploadInfo.upload.Name] != nil && filename2attribute2info[uploadInfo.upload.Name]["filter"] == "lfs" { + // Handle LFS + // FIXME: Inefficient! this should probably happen in models.Upload + oid, err := models.GenerateLFSOid(file) + if err != nil { + return err + } + fileInfo, err := file.Stat() + if err != nil { + return err + } + + uploadInfo.lfsMetaObject = &models.LFSMetaObject{Oid: oid, Size: fileInfo.Size(), RepositoryID: t.repo.ID} + + if objectHash, err = t.HashObject(strings.NewReader(uploadInfo.lfsMetaObject.Pointer())); err != nil { + return err + } + infos[i] = uploadInfo + + } else { + if objectHash, err = t.HashObject(file); err != nil { + return err + } } // Add the object to the index - if err := t.AddObjectToIndex("100644", objectHash, path.Join(opts.TreePath, upload.Name)); err != nil { + if err := t.AddObjectToIndex("100644", objectHash, path.Join(opts.TreePath, uploadInfo.upload.Name)); err != nil { return err + } } @@ -77,6 +134,43 @@ func UploadRepoFiles(repo *models.Repository, doer *models.User, opts UploadRepo return err } + // Now deal with LFS objects + for _, uploadInfo := range infos { + if uploadInfo.lfsMetaObject == nil { + continue + } + uploadInfo.lfsMetaObject, err = models.NewLFSMetaObject(uploadInfo.lfsMetaObject) + if err != nil { + // OK Now we need to cleanup + return cleanUpAfterFailure(&infos, t, err) + } + // Don't move the files yet - we need to ensure that + // everything can be inserted first + } + + // OK now we can insert the data into the store - there's no way to clean up the store + // once it's in there, it's in there. + contentStore := &lfs.ContentStore{BasePath: setting.LFS.ContentPath} + for _, uploadInfo := range infos { + if uploadInfo.lfsMetaObject == nil { + continue + } + if !contentStore.Exists(uploadInfo.lfsMetaObject) { + file, err := os.Open(uploadInfo.upload.LocalPath()) + if err != nil { + return cleanUpAfterFailure(&infos, t, err) + } + defer file.Close() + // FIXME: Put regenerates the hash and copies the file over. + // I guess this strictly ensures the soundness of the store but this is inefficient. + if err := contentStore.Put(uploadInfo.lfsMetaObject, file); err != nil { + // OK Now we need to cleanup + // Can't clean up the store, once uploaded there they're there. + return cleanUpAfterFailure(&infos, t, err) + } + } + } + // Then push this tree to NewBranch if err := t.Push(doer, commitHash, opts.NewBranch); err != nil { return err diff --git a/routers/repo/editor.go b/routers/repo/editor.go index 7b65397e05bc..01963d8dc68b 100644 --- a/routers/repo/editor.go +++ b/routers/repo/editor.go @@ -309,7 +309,7 @@ func editFilePost(ctx *context.Context, form auth.EditRepoFileForm, isNewFile bo message += "\n\n" + form.CommitMessage } - if err := uploader.UpdateRepoFile(ctx.Repo.Repository, ctx.User, uploader.UpdateRepoFileOptions{ + if err := uploader.UpdateRepoFile(ctx.Repo.Repository, ctx.User, &uploader.UpdateRepoFileOptions{ LastCommitID: lastCommit, OldBranch: oldBranchName, NewBranch: branchName, @@ -448,7 +448,7 @@ func DeleteFilePost(ctx *context.Context, form auth.DeleteRepoFileForm) { message += "\n\n" + form.CommitMessage } - if err := uploader.DeleteRepoFile(ctx.Repo.Repository, ctx.User, uploader.DeleteRepoFileOptions{ + if err := uploader.DeleteRepoFile(ctx.Repo.Repository, ctx.User, &uploader.DeleteRepoFileOptions{ LastCommitID: ctx.Repo.CommitID, OldBranch: oldBranchName, NewBranch: branchName, @@ -583,7 +583,7 @@ func UploadFilePost(ctx *context.Context, form auth.UploadRepoFileForm) { message += "\n\n" + form.CommitMessage } - if err := uploader.UploadRepoFiles(ctx.Repo.Repository, ctx.User, uploader.UploadRepoFileOptions{ + if err := uploader.UploadRepoFiles(ctx.Repo.Repository, ctx.User, &uploader.UploadRepoFileOptions{ LastCommitID: ctx.Repo.CommitID, OldBranch: oldBranchName, NewBranch: branchName, From 774d4c2a97055046dac0298190cf17232124fbc1 Mon Sep 17 00:00:00 2001 From: zeripath Date: Sun, 13 Jan 2019 07:01:17 +0000 Subject: [PATCH 11/19] Update upload.go attribution header Upload.go is essentially the remnants of repo_editor.go. The remaining code is essentially unchanged from the Gogs code, hence the Gogs attribution. --- models/upload.go | 1 + 1 file changed, 1 insertion(+) diff --git a/models/upload.go b/models/upload.go index e015a916f48b..c67dab5ebf95 100644 --- a/models/upload.go +++ b/models/upload.go @@ -1,4 +1,5 @@ // Copyright 2016 The Gogs Authors. All rights reserved. +// Copyright 2019 The Gitea Authors. All rights reserved. // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. From 78e27c21be09b25ea499bc46f1e94b82af76cf98 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Wed, 16 Jan 2019 20:43:37 +0000 Subject: [PATCH 12/19] Delete upload files after session committed --- models/upload.go | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/models/upload.go b/models/upload.go index c67dab5ebf95..65ce1223c8fb 100644 --- a/models/upload.go +++ b/models/upload.go @@ -119,6 +119,10 @@ func DeleteUploads(uploads ...*Upload) (err error) { return fmt.Errorf("delete uploads: %v", err) } + if err = sess.Commit(); err != nil { + return err + } + for _, upload := range uploads { localPath := upload.LocalPath() if !com.IsFile(localPath) { @@ -130,12 +134,7 @@ func DeleteUploads(uploads ...*Upload) (err error) { } } - return sess.Commit() -} - -// DeleteUpload delete a upload -func DeleteUpload(u *Upload) error { - return DeleteUploads(u) + return nil } // DeleteUploadByUUID deletes a upload by UUID @@ -148,7 +147,7 @@ func DeleteUploadByUUID(uuid string) error { return fmt.Errorf("GetUploadByUUID: %v", err) } - if err := DeleteUpload(upload); err != nil { + if err := DeleteUploads(upload); err != nil { return fmt.Errorf("DeleteUpload: %v", err) } From d0d28c98cd3f510625ff9bbc13b9eb7b416f3427 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sun, 20 Jan 2019 12:36:32 +0000 Subject: [PATCH 13/19] Ensure that GIT_AUTHOR_NAME etc. are valid for git see #5774 Signed-off-by: Andrew Thornton --- modules/uploader/repo.go | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/modules/uploader/repo.go b/modules/uploader/repo.go index 03914c162f12..7fad888683b8 100644 --- a/modules/uploader/repo.go +++ b/modules/uploader/repo.go @@ -206,15 +206,16 @@ func (t *TemporaryUploadRepository) WriteTree() (string, error) { // CommitTree creates a commit from a given tree for the user with provided message func (t *TemporaryUploadRepository) CommitTree(doer *models.User, treeHash string, message string) (string, error) { commitTimeStr := time.Now().Format(time.UnixDate) + sig := doer.NewGitSig() // FIXME: Should we add SSH_ORIGINAL_COMMAND to this // Because this may call hooks we should pass in the environment env := append(os.Environ(), - "GIT_AUTHOR_NAME="+doer.DisplayName(), - "GIT_AUTHOR_EMAIL="+doer.Email, + "GIT_AUTHOR_NAME="+sig.Name, + "GIT_AUTHOR_EMAIL="+sig.Email, "GIT_AUTHOR_DATE="+commitTimeStr, - "GIT_COMMITTER_NAME="+doer.DisplayName(), - "GIT_COMMITTER_EMAIL="+doer.Email, + "GIT_COMMITTER_NAME="+sig.Name, + "GIT_COMMITTER_EMAIL="+sig.Email, "GIT_COMMITTER_DATE="+commitTimeStr, ) commitHash, stderr, err := process.GetManager().ExecDirEnv(5*time.Minute, @@ -235,13 +236,15 @@ func (t *TemporaryUploadRepository) Push(doer *models.User, commitHash string, b isWiki = "true" } + sig := doer.NewGitSig() + // FIXME: Should we add SSH_ORIGINAL_COMMAND to this // Because calls hooks we need to pass in the environment env := append(os.Environ(), - "GIT_AUTHOR_NAME="+doer.DisplayName(), - "GIT_AUTHOR_EMAIL="+doer.Email, - "GIT_COMMITTER_NAME="+doer.DisplayName(), - "GIT_COMMITTER_EMAIL="+doer.Email, + "GIT_AUTHOR_NAME="+sig.Name, + "GIT_AUTHOR_EMAIL="+sig.Email, + "GIT_COMMITTER_NAME="+sig.Name, + "GIT_COMMITTER_EMAIL="+sig.Email, models.EnvRepoName+"="+t.repo.Name, models.EnvRepoUsername+"="+t.repo.OwnerName, models.EnvRepoIsWiki+"="+isWiki, From 2ba5664438523adc6af01ae1733dfe2be41fcf61 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Wed, 6 Feb 2019 21:49:37 +0000 Subject: [PATCH 14/19] Add in test cases per @lafriks comment --- routers/repo/editor_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/routers/repo/editor_test.go b/routers/repo/editor_test.go index 63518caff28f..3846592057fc 100644 --- a/routers/repo/editor_test.go +++ b/routers/repo/editor_test.go @@ -23,6 +23,8 @@ func TestCleanUploadName(t *testing.T) { "../../../acd": "acd", "../../.git/abc": "", "..\\..\\.git/abc": "..\\..\\.git/abc", + "..\\../.git/abc": "", + "..\\../.git": "", "abc/../def": "def", } for k, v := range kases { From 482cbfe152f5a99e5995d4d2dfc8807410f5cd84 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Thu, 7 Feb 2019 16:02:28 +0000 Subject: [PATCH 15/19] Add space between gitea and github imports Signed-off-by: Andrew Thornton --- modules/uploader/repo.go | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/uploader/repo.go b/modules/uploader/repo.go index 7fad888683b8..33cc160ca940 100644 --- a/modules/uploader/repo.go +++ b/modules/uploader/repo.go @@ -18,6 +18,7 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/process" "code.gitea.io/gitea/modules/setting" + "github.com/Unknwon/com" ) From b346a29c4cc0dae75f7862928b7663e0135579c0 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Thu, 7 Feb 2019 16:07:19 +0000 Subject: [PATCH 16/19] more examples in TestCleanUploadName Signed-off-by: Andrew Thornton --- routers/repo/editor_test.go | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/routers/repo/editor_test.go b/routers/repo/editor_test.go index 3846592057fc..547be156f49f 100644 --- a/routers/repo/editor_test.go +++ b/routers/repo/editor_test.go @@ -15,19 +15,24 @@ func TestCleanUploadName(t *testing.T) { models.PrepareTestEnv(t) var kases = map[string]string{ - ".git/refs/master": "", - "/root/abc": "root/abc", - "./../../abc": "abc", - "a/../.git": "", - "a/../../../abc": "abc", - "../../../acd": "acd", - "../../.git/abc": "", - "..\\..\\.git/abc": "..\\..\\.git/abc", - "..\\../.git/abc": "", - "..\\../.git": "", - "abc/../def": "def", + ".git/refs/master": "", + "/root/abc": "root/abc", + "./../../abc": "abc", + "a/../.git": "", + "a/../../../abc": "abc", + "../../../acd": "acd", + "../../.git/abc": "", + "..\\..\\.git/abc": "..\\..\\.git/abc", + "..\\../.git/abc": "", + "..\\../.git": "", + "abc/../def": "def", + ".drone.yml": ".drone.yml", + ".abc/def/.drone.yml": ".abc/def/.drone.yml", + "..drone.yml.": "..drone.yml.", + "..a.dotty...name...": "..a.dotty...name...", + "..a.dotty..folder/.with.a.dotty/.name...": "..a.dotty..folder/.with.a.dotty/.name...", } for k, v := range kases { - assert.EqualValues(t, v, cleanUploadFileName(k)) + assert.EqualValues(t, cleanUploadFileName(k), v) } } From 8a8b66bcdd22d1da4ced51ae10c831b5d145f567 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Thu, 7 Feb 2019 16:16:18 +0000 Subject: [PATCH 17/19] fix formatting Signed-off-by: Andrew Thornton --- routers/repo/editor_test.go | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/routers/repo/editor_test.go b/routers/repo/editor_test.go index 547be156f49f..b3d4314c2657 100644 --- a/routers/repo/editor_test.go +++ b/routers/repo/editor_test.go @@ -15,22 +15,22 @@ func TestCleanUploadName(t *testing.T) { models.PrepareTestEnv(t) var kases = map[string]string{ - ".git/refs/master": "", - "/root/abc": "root/abc", - "./../../abc": "abc", - "a/../.git": "", - "a/../../../abc": "abc", - "../../../acd": "acd", - "../../.git/abc": "", - "..\\..\\.git/abc": "..\\..\\.git/abc", - "..\\../.git/abc": "", - "..\\../.git": "", - "abc/../def": "def", - ".drone.yml": ".drone.yml", - ".abc/def/.drone.yml": ".abc/def/.drone.yml", - "..drone.yml.": "..drone.yml.", - "..a.dotty...name...": "..a.dotty...name...", - "..a.dotty..folder/.with.a.dotty/.name...": "..a.dotty..folder/.with.a.dotty/.name...", + ".git/refs/master": "", + "/root/abc": "root/abc", + "./../../abc": "abc", + "a/../.git": "", + "a/../../../abc": "abc", + "../../../acd": "acd", + "../../.git/abc": "", + "..\\..\\.git/abc": "..\\..\\.git/abc", + "..\\../.git/abc": "", + "..\\../.git": "", + "abc/../def": "def", + ".drone.yml": ".drone.yml", + ".abc/def/.drone.yml": ".abc/def/.drone.yml", + "..drone.yml.": "..drone.yml.", + "..a.dotty...name...": "..a.dotty...name...", + "..a.dotty../.folder../.name...": "..a.dotty../.folder../.name...", } for k, v := range kases { assert.EqualValues(t, cleanUploadFileName(k), v) From d3dddfff6db8f203efa06bb768f97cfa7788b741 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sat, 9 Feb 2019 14:49:11 +0000 Subject: [PATCH 18/19] Set the SSH_ORIGINAL_COMMAND to ensure hooks are run Signed-off-by: Andrew Thornton --- modules/uploader/repo.go | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/uploader/repo.go b/modules/uploader/repo.go index 33cc160ca940..0d32e879b2af 100644 --- a/modules/uploader/repo.go +++ b/modules/uploader/repo.go @@ -242,6 +242,7 @@ func (t *TemporaryUploadRepository) Push(doer *models.User, commitHash string, b // FIXME: Should we add SSH_ORIGINAL_COMMAND to this // Because calls hooks we need to pass in the environment env := append(os.Environ(), + "SSH_ORIGINAL_COMMAND=internal", "GIT_AUTHOR_NAME="+sig.Name, "GIT_AUTHOR_EMAIL="+sig.Email, "GIT_COMMITTER_NAME="+sig.Name, From 30ccd165d5f92cb9460bda70f6f7231c96490a4a Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sat, 9 Feb 2019 15:05:51 +0000 Subject: [PATCH 19/19] Switch off SSH_ORIGINAL_COMMAND Signed-off-by: Andrew Thornton --- modules/uploader/repo.go | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/uploader/repo.go b/modules/uploader/repo.go index 0d32e879b2af..33cc160ca940 100644 --- a/modules/uploader/repo.go +++ b/modules/uploader/repo.go @@ -242,7 +242,6 @@ func (t *TemporaryUploadRepository) Push(doer *models.User, commitHash string, b // FIXME: Should we add SSH_ORIGINAL_COMMAND to this // Because calls hooks we need to pass in the environment env := append(os.Environ(), - "SSH_ORIGINAL_COMMAND=internal", "GIT_AUTHOR_NAME="+sig.Name, "GIT_AUTHOR_EMAIL="+sig.Email, "GIT_COMMITTER_NAME="+sig.Name,