From 4535ccc406ec275bc884ecb2daba18e83e014a41 Mon Sep 17 00:00:00 2001 From: Amin Yahyaabadi Date: Sun, 23 Jun 2019 01:00:30 -0500 Subject: [PATCH 01/12] Fix AbstractTrees download command for local run Fix AbstractTrees download command for local run Fix running FemtoCleaner.cleanrepo command Added Julia 6.4 download link Fixed non-working URLs Brought local running in Readme upfront --- README.md | 53 +++++++++++++++++++++++++++++++---------------------- 1 file changed, 31 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 476013a..1546756 100644 --- a/README.md +++ b/README.md @@ -7,9 +7,39 @@ perform code formatting. The logic behind recognizing and rewriting deprecated c can be found in the [Deprecations.jl](https://github.com/JuliaComputing/Deprecations.jl) package, which makes use of [CSTParser.jl](https://github.com/ZacLN/CSTParser.jl) under the hood. +## Running FemtoCleaner locally + +It is possible to run FemtoCleaner locally (to fix, for example, deprecations in a private repository). + +Install `FemtoCleaner` (currently working on Julia v0.6.x only https://julialang.org/downloads/oldreleases.html ) using + +```jl +Pkg.clone("https://github.com/Keno/AbstractTrees.jl") +Pkg.checkout("AbstractTrees.jl","kf/for06") +Pkg.clone("https://github.com/JuliaComputing/Deprecations.jl") +Pkg.clone("https://github.com/JuliaComputing/FemtoCleaner.jl") +using FemtoCleaner +``` + +A repository of Julia code can be cleaned using + +```jl +using FemtoCleaner +FemtoCleaner.cleanrepo(path::String, show_diff = true, delete_local = true) +``` +For example on Windows: +```jl +using FemtoCleaner +FemtoCleaner.cleanrepo("C:\\repositoryFolder", show_diff = true, delete_local = true) +``` + +This clones the repo located at `path`, which can be a file system path or a URL, to a temporary directory +and fix the deprecations. If `show_diff` is `true`, the diff from applying the deprecations is showed. +If `delete_local` is `true` the cleaned repo, is deleted when the function is finished. + # User Manual -To set up FemtoCleaner on your repository, go to https://github.com/integration/femtocleaner and click "Configure" to select the repositories you wish to add. +To set up FemtoCleaner on your repository, go to https://github.com/apps/femtocleaner and click "Configure" to select the repositories you wish to add. ## Invoking FemtoCleaner @@ -64,27 +94,6 @@ publicly hosted version thereof. In particular: > THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -## Running FemtoCleaner locally - -It is possible to run FemtoCleaner locally (to fix, for example, deprecations in a private repository). - -Install `FemtoCleaner` (currently working on Julia v0.6.x only) using - -```jl -Pkg.clone("https://github.com/Keno/AbstractTrees.jl") -Pkg.clone("https://github.com/JuliaComputing/Deprecations.jl") -Pkg.clone("https://github.com/JuliaComputing/FemtoCleaner.jl") -``` - -A repository of Julia code can be cleaned using - -```jl -FemtoCleaner.cleanrepo(path::String; show_diff = true, delete_local = true) -``` - -This clones the repo located at `path`, which can be a file system path or a URL, to a temporary directory -and fix the deprecations. If `show_diff` is `true`, the diff from applying the deprecations is showed. -If `delete_local` is `true` the cleaned repo, is deleted when the function is finished. # Developer Manual From 06a0335146a9823911a3fa424e77f2fbaf2e4b2b Mon Sep 17 00:00:00 2001 From: aminya Date: Wed, 14 Aug 2019 17:58:44 -0500 Subject: [PATCH 02/12] Update REQUIRE --- REQUIRE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/REQUIRE b/REQUIRE index 0ca7977..81efb61 100644 --- a/REQUIRE +++ b/REQUIRE @@ -1,4 +1,4 @@ -julia 0.6 +julia 1.0 Deprecations CSTParser GitHub From d2c9487f320eabb9429a056a3863d2bb1f446bbd Mon Sep 17 00:00:00 2001 From: aminya Date: Wed, 14 Aug 2019 17:59:53 -0500 Subject: [PATCH 03/12] FemtoCleaner RQUIRE Julia 1 --- src/FemtoCleaner.jl | 1034 ++++++++++++++++++++--------------------- src/autodeployment.jl | 10 +- src/blame.jl | 22 +- src/interactions.jl | 4 +- src/workqueue.jl | 2 +- test/dry_runs.jl | 2 +- test/runtests.jl | 6 +- 7 files changed, 538 insertions(+), 542 deletions(-) diff --git a/src/FemtoCleaner.jl b/src/FemtoCleaner.jl index 2e15e0c..0874efd 100644 --- a/src/FemtoCleaner.jl +++ b/src/FemtoCleaner.jl @@ -1,519 +1,515 @@ -module FemtoCleaner - -# For interactive development -using Revise - -using Base.Distributed -using GitHub -using GitHub: GitHubAPI, GitHubWebAPI, Checks -using HTTP -using Deprecations -using CSTParser -using Deprecations: isexpr -using MbedTLS -using JSON -using AbstractTrees: children -using Base: LibGit2 - -include("workqueue.jl") - -function with_cloned_repo(f, api::GitHubWebAPI, repo, auth) - creds = LibGit2.UserPasswordCredentials(String(copy(Vector{UInt8}("x-access-token"))), String(copy(Vector{UInt8}(auth.token)))) - repo_url = "https://github.com/$(get(repo.full_name))" - local_dir = mktempdir() - try - enabled = gc_enable(false) - lrepo = LibGit2.clone(repo_url, local_dir; payload=Nullable(creds)) - gc_enable(enabled) - f((lrepo, local_dir)) - finally - rm(local_dir, force=true, recursive=true) - end -end - -function with_pr_branch(f, api, repo, auth) - with_cloned_repo(api, repo, auth) do x - LibGit2.branch!(lrepo, "fbot/deps", track=LibGit2.Consts.REMOTE_ORIGIN) - f(x) - end -end - -if VERSION < v"0.7.0-DEV.695" - include("blame.jl") -else - using LibGit2: GitBlame -end - -function deprecations_for_repo(lrepo, local_dir, is_julia_itself) - if is_julia_itself - ver = readstring(joinpath(local_dir, "VERSION")) - hunk = GitBlame(lrepo, "VERSION")[1] - l, r = LibGit2.revcount(lrepo, string(hunk.orig_commit_id), "HEAD") - vers = Pkg.Reqs.parse(IOBuffer("julia $ver+$(l+r)")) - else - vers = Pkg.Reqs.parse(joinpath(local_dir, "REQUIRE")) - end - deps = Deprecations.applicable_deprecations(vers) - -end - -function process_deprecations(lrepo, local_dir; is_julia_itself=false, deps = deprecations_for_repo(lrepo, local_dir, is_julia_itself)) - changed_any = false - problematic_files = String[] - all_files = String[] - all_files = String[] - for (root, dirs, files) in walkdir(local_dir) - for file in files - fpath = joinpath(root, file) - (endswith(fpath, ".jl") || endswith(fpath, ".md")) || continue - file == "NEWS.md" && continue - push!(all_files, fpath) - end - end - max_iterations = 30 - iteration_counter = fill(0, length(all_files)) - # Iterate. Some rewrites may expose others - while any(x->x != -1, iteration_counter) - # We need to redo the analysis after every fetmocleaning round, since - # things may have changes as the result of an applied rewrite. - analysis = Deprecations.process_all(filter(f->endswith(f, ".jl"), all_files)) - for (i, fpath) in enumerate(all_files) - iteration_counter[i] == -1 && continue - problematic_file = false - file_analysis = endswith(fpath, ".jl") ? (analysis[1], analysis[2][fpath]) : nothing - try - if !Deprecations.edit_file(fpath, deps, endswith(fpath, ".jl") ? edit_text : edit_markdown; - analysis = file_analysis) - # Nothing to change - iteration_counter[i] = -1 - elseif iteration_counter[i] > max_iterations - warn("Iterations did not converge for file $fpath") - problematic_file = true - else - iteration_counter[i] += 1 - changed_any = true - end - catch e - warn("Exception thrown when fixing file $fpath. Exception was:\n", - sprint(showerror, e, catch_backtrace())) - problematic_file = true - end - if problematic_file - push!(problematic_files, file) - iteration_counter[i] = -1 - end - changed_any && !problematic_file && LibGit2.add!(lrepo, relpath(fpath, local_dir)) - end - end - changed_any, problematic_files -end - -function push_repo(api::GitHubWebAPI, repo, auth; force=true, remote_branch="fbot/deps") - creds = LibGit2.UserPasswordCredentials(String(copy(Vector{UInt8}("x-access-token"))), String(copy(Vector{UInt8}(auth.token)))) - enabled = gc_enable(false) - LibGit2.push(repo, refspecs = ["+HEAD:refs/heads/$remote_branch"], force=force, - payload=Nullable(creds)) - gc_enable(enabled) -end - -struct SourceFile - data::Vector{UInt8} - offsets::Vector{UInt64} -end -Base.length(file::SourceFile) = length(file.offsets) - -function SourceFile(data) - offsets = UInt64[0] - buf = IOBuffer(data) - local line = "" - while !eof(buf) - line = readuntil(buf,'\n') - !eof(buf) && push!(offsets, position(buf)) - end - if !isempty(line) && line[end] == '\n' - push!(offsets, position(buf)+1) - end - SourceFile(data,offsets) -end - -function compute_line(file::SourceFile, offset) - ind = searchsortedfirst(file.offsets, offset) - ind <= length(file.offsets) && file.offsets[ind] == offset ? ind : ind - 1 -end - -function Base.getindex(file::SourceFile, line::Int) - if line == length(file.offsets) - return file.data[(file.offsets[end]+1):end] - else - # - 1 to skip the '\n' - return file.data[(file.offsets[line]+1):max(1, file.offsets[line+1]-1)] - end -end -Base.getindex(file::SourceFile, arr::AbstractArray) = [file[x] for x in arr] - -function repl_to_annotation(fpath, file, lrepo, local_dir, repo, repl) - # Compute blob hash for fpath - blob_hash = LibGit2.addblob!(lrepo, joinpath(local_dir, fpath)) - # Put together description - start_line = compute_line(file, first(repl.range)) - message = """ - $(repl.dep === nothing ? "" : repl.dep.description) - In file $fpath starting at line $(start_line): - $(strip(String(file[start_line]))) - """ - Checks.Annotation( - basename(fpath), - "https://github.com/$(GitHub.name(repo))/blob/$(blob_hash)/$(fpath)", - compute_line(file, first(repl.range)), compute_line(file, last(repl.range)), - "notice", - message, - string(typeof(repl.dep).name.name)[1:min(end, 40)], - "" - ) -end - -function collect_deprecation_annotations(api::GitHubAPI, lrepo, local_dir, repo, auth; is_julia_itself=false) - deps = deprecations_for_repo(lrepo, local_dir, is_julia_itself) - annotations = Checks.Annotation[] - problematic_files = String[] - for (root, dirs, files) in walkdir(local_dir) - for file in files - fpath = joinpath(root, file) - (endswith(fpath, ".jl") || endswith(fpath, ".md")) || continue - file == "NEWS.md" && continue - contents = readstring(fpath) - sfile = SourceFile(contents) - problematic_file = false - try - if endswith(fpath, ".md") - changed_any, _ = Deprecations.edit_markdown(contents, deps) - if changed_any - blob_hash = LibGit2.addblob!(lrepo, joinpath(local_dir, fpath)) - push!(annotations, Checks.Annotation( - basename(fpath), - "https://github.com/$(GitHub.name(repo))/blob/$(blob_hash)/$(fpath)", - 1, 1, - "notice", - """ - Code changes were found in this Markdown document: - $(fpath) - """, - "MarkdownCode", - "" - )) - end - else - repls = Deprecations.text_replacements(contents, deps) - for repl in repls - push!(annotations, repl_to_annotation(fpath, sfile, lrepo, local_dir, repo, repl)) - end - end - catch e - warn("Exception thrown when fixing file $file. Exception was:\n", - sprint(showerror, e, catch_backtrace())) - problematic_file = true - end - problematic_file && push!(problematic_files, file) - end - end - annotations, problematic_files -end - -function apply_deprecations(api::GitHubAPI, lrepo, local_dir, commit_sig, repo, auth; issue_number = 0) - is_julia_itself = GitHub.name(repo) == "JuliaLang/julia" - changed_any, problematic_files = process_deprecations(lrepo, local_dir; is_julia_itself=is_julia_itself) - if changed_any - LibGit2.commit(lrepo, "Fix deprecations"; author=commit_sig, committer=commit_sig, parent_ids=[LibGit2.GitHash(lrepo, "HEAD")]) - push_repo(api, lrepo, auth) - end - if issue_number != 0 - if changed_any - create_pull_request(api, repo, auth=auth, params = Dict( - :issue => issue_number, - :base => get(repo.default_branch), - :head => "fbot/deps" - ) - ) - if !isempty(problematic_files) - create_comment(api, repo, issue_number, :pr, auth=auth, params = Dict( - :body => string("Failed to process the following files: ", - join("`" .* problematic_files .* "`", ", "), - ". :(") - ) - ) - end - println("Created pull request for $(GitHub.name(repo))") - else - create_comment(api, repo, issue_number, :issue, params = Dict( - :body => "No applicable deprecations were found in this repository." - ), auth=auth) - println("Processing complete for $(GitHub.name(repo)): no changes made") - end - else - if changed_any - body = "I fixed a number of deprecations for you" - if !isempty(problematic_files) - body *= string(", but I failed to process the following files: ", - join("`" .* problematic_files .* "`", ", "), - ". :(") - end - create_pull_request(api, repo, auth=auth, params = Dict( - :title => "Fix deprecations", - :body => body, - :base => get(repo.default_branch), - :head => "fbot/deps" - ) - ) - println("Created pull request for $(GitHub.name(repo))") - else - println("Processing complete for $(GitHub.name(repo)): no changes made") - end - end -end - -function my_diff_tree(repo::LibGit2.GitRepo, oldtree::LibGit2.GitTree, newtree::LibGit2.GitTree; pathspecs::AbstractString="") - diff_ptr_ptr = Ref{Ptr{Void}}(C_NULL) - @LibGit2.check ccall((:git_diff_tree_to_tree, :libgit2), Cint, - (Ptr{Ptr{Void}}, Ptr{Void}, Ptr{Void}, Ptr{Void}, Ptr{LibGit2.DiffOptionsStruct}), - diff_ptr_ptr, repo.ptr, oldtree.ptr, newtree.ptr, isempty(pathspecs) ? C_NULL : pathspecs) - return LibGit2.GitDiff(repo, diff_ptr_ptr[]) -end - -function apply_deprecations_if_updated(api::GitHubAPI, lrepo, local_dir, before, after, commit_sig, repo, auth) - before = LibGit2.GitCommit(lrepo, before) - after = LibGit2.GitCommit(lrepo, after) - delta = my_diff_tree(lrepo, LibGit2.peel(before), LibGit2.peel(after); pathspecs="REQUIRE")[1] - old_blob = LibGit2.GitBlob(lrepo, delta.old_file.id) - new_blob = LibGit2.GitBlob(lrepo, delta.new_file.id) - vers_old = Pkg.Reqs.parse(IOBuffer(LibGit2.content(old_blob))) - vers_new = Pkg.Reqs.parse(IOBuffer(LibGit2.content(new_blob))) - if vers_new["julia"] != vers_old["julia"] - apply_deprecations(api, lrepo, local_dir, commit_sig, repo, auth) - end -end - -function cleanrepo(repo_url; show_diff = true, delete_local = true) - local_dir = mktempdir() - successful = true - try - enabled = gc_enable(false) - info("Cloning $repo_url to $local_dir...") - lrepo = LibGit2.clone(repo_url, local_dir) - gc_enable(enabled) - info("Processing deprecations...") - changed_any, problematic_files = process_deprecations(lrepo, local_dir; is_julia_itself=contains(repo_url, "JuliaLang/julia")) - isempty(problematic_files) || (successful = false) - catch e - bt = catch_backtrace() - Base.display_error(STDERR, e, bt) - successful = false - finally - if show_diff - cd(local_dir) do - run(`git status`) - run(`git diff --cached`) - end - end - if delete_local - info("Deleting cloned repo from $local_dir...") - rm(local_dir, force=true, recursive=true) - end - end - return successful -end - -include("interactions.jl") -include("autodeployment.jl") - -const autodeployment_enabled = haskey(ENV, "FEMTOCLEANER_AUTODEPLOY") ? - ENV["FEMTOCLEANER_AUTODEPLOY"] == "yes" : false - -let app_key = Ref{Any}(nothing) - global get_auth - function get_auth(app_id) - if app_key[] == nothing - app_key[] = MbedTLS.PKContext() - MbedTLS.parse_key!(app_key[], haskey(ENV, "FEMTOCLEANER_PRIVKEY") ? ENV["FEMTOCLEANER_PRIVKEY"] : readstring(joinpath(dirname(@__FILE__),"..","privkey.pem"))) - end - GitHub.JWTAuth(app_id, app_key[]) - end -end - -function event_callback(api::GitHubAPI, app_name, app_id, sourcerepo_installation, - commit_sig, listener, bug_repository, event) - # On installation, process every repository we just got installed into - if event.kind == "installation" - jwt = get_auth(app_id) - installation = Installation(event.payload["installation"]) - auth = create_access_token(api, installation, jwt) - for repo in event.payload["repositories"] - repo = GitHub.repo(api, GitHub.Repo(repo); auth=auth) - with_cloned_repo(api, repo, auth) do x - apply_deprecations(api, x..., commit_sig, repo, auth) - end - end - elseif event.kind == "check_run" - jwt = get_auth(app_id) - installation = Installation(event.payload["installation"]) - auth = create_access_token(api, installation, jwt) - if event.payload["action"] == "requested_action" && event.payload["requested_action"]["identifier"] == "fix" - repo = GitHub.Repo(event.payload["repository"]) - pr = PullRequest(event.payload["check_run"]["check_suite"]["pull_requests"][1]) - with_cloned_repo(api, repo, auth) do x - lrepo, local_dir = x - LibGit2.checkout!(lrepo, event.payload["check_run"]["check_suite"]["head_sha"]) - is_julia_itself = GitHub.name(repo) == "JuliaLang/julia" - changed_any, problematic_files = process_deprecations(lrepo, local_dir; is_julia_itself=is_julia_itself) - if changed_any - LibGit2.commit(lrepo, "Fix deprecations"; author=commit_sig, committer=commit_sig, parent_ids=[LibGit2.GitHash(lrepo, "HEAD")]) - push_repo(api, lrepo, auth; force=false, remote_branch=event.payload["check_run"]["check_suite"]["head_branch"]) - end - end - end - elseif event.kind == "pull_request" - if !(event.payload["action"] in ("opened", "reopened", "synchronize")) - return HTTP.Response(200) - end - jwt = get_auth(app_id) - installation = Installation(event.payload["installation"]) - auth = create_access_token(api, installation, jwt) - repo = Repo(event.payload["repository"]) - pr = PullRequest(event.payload["pull_request"]) - local annotations - with_cloned_repo(api, repo, auth) do x - lrepo, local_dir = x - LibGit2.checkout!(lrepo, get(get(pr.head).sha)) - annotations, _ = collect_deprecation_annotations(api, x..., repo, auth) - end - actions = Checks.Action[] - conclusion = "neutral" - if length(annotations) == 0 - message = """ - No applicable deprecations were detected. - """ - output = Checks.Output( - "Femtocleaning", - message, - "", - Checks.Annotation[], - Checks.Image[] - ) - conclusion = "success" - else - truncated = length(annotations) > 50 - message = """ - Several femtocleaning opportunities were detected - """ - output = Checks.Output( - "Femtocleaning", - message, - "See below", - annotations[1:min(50, length(annotations))], - GitHub.Image[] - ) - actions = Checks.Action[ - Checks.Action( - "Fix it!", - "Fixes issues in this PR (adds commit).", - "fix" - ) - ] - end - max_annotation = 50 - cr = GitHub.create_check_run(api, repo, auth=auth, params = Dict( - :name => "femtocleaner", - :head_branch => get(pr.head).ref, - :head_sha => get(pr.head).sha, - :status => "completed", - :conclusion => conclusion, - :completed_at => now(), - :actions => actions, - :output => output - )) - while max_annotation < length(annotations) - empty!(output.annotations) - append!(output.annotations, annotations[max_annotation+1:min(max_annotation+50, end)]) - max_annotation += 50 - GitHub.update_check_run(api, repo, get(cr.id), auth=auth, params = Dict( - :output => output, - :actions => actions - )) - end - elseif event.kind == "installation_repositories" - jwt = get_auth(app_id) - installation = Installation(event.payload["installation"]) - auth = create_access_token(api, installation, jwt) - for repo in event.payload["repositories_added"] - repo = GitHub.repo(api, GitHub.Repo(repo); auth=auth) - with_cloned_repo(api, repo, auth) do x - apply_deprecations(api, x..., commit_sig, repo, auth) - end - end - elseif event.kind == "pull_request_review" - jwt = get_auth(app_id) - pr_response(api, event, jwt, commit_sig, app_name, sourcerepo_installation, bug_repository) - elseif event.kind == "push" - jwt = get_auth(app_id) - installation = Installation(event.payload["installation"]) - auth = create_access_token(api, installation, jwt) - repo = Repo(event.payload["repository"]) - # Check if REQUIRE was updated - for commit in event.payload["commits"] - if "REQUIRE" in commit["modified"] - with_cloned_repo(api, repo, auth) do x - apply_deprecations_if_updated(api, x..., - event.payload["before"], event.payload["after"], - commit_sig, repo, auth) - end - break - end - end - maybe_autdodeploy(event, listener, jwt, sourcerepo_installation, autodeployment_enabled) - elseif event.kind == "issues" && event.payload["action"] == "opened" - jwt = get_auth(app_id) - iss = Issue(event.payload["issue"]) - repo = Repo(event.payload["repository"]) - installation = Installation(event.payload["installation"]) - auth = create_access_token(api, installation, jwt) - if lowercase(get(iss.title)) == "run femtocleaner" - with_cloned_repo(api, repo, auth) do x - apply_deprecations(api, x..., commit_sig, repo, auth; issue_number = get(iss.number)) - end - end - end - return HTTP.Response(200) -end - -function run_server() - app_id = parse(Int, strip(haskey(ENV, "FEMTOCLEANER_APPID") ? ENV["FEMTOCLEANER_APPID"] : readstring(joinpath(dirname(@__FILE__),"..","app_id")))) - secret = haskey(ENV, "FEMTOCLEANER_SECRET") ? ENV["FEMTOCLEANER_SECRET"] : nothing - sourcerepo_installation = haskey(ENV, "FEMTOCLEANER_INSTALLATION") ? parse(Int, ENV["FEMTOCLEANER_INSTALLATION"]) : 0 - bug_repository = haskey(ENV, "FEMTOCLEANER_BUGREPO") ? ENV["FEMTOCLEANER_BUGREPO"] : "" - (secret == nothing) && warn("Webhook secret not set. All events will be accepted. This is an insecure configuration!") - jwt = get_auth(app_id) - app_name = get(GitHub.app(; auth=jwt).name) - commit_sig = LibGit2.Signature("$(app_name)[bot]", "$(app_name)[bot]@users.noreply.github.com") - api = GitHub.DEFAULT_API - local listener - if nprocs() != 1 - #@everywhere filter(x->x != 1, procs()) worker_loop() - for p in filter(x->x != 1, procs()) - @spawnat p begin - myid() == 2 && update_existing_repos(api, commit_sig, app_id) - worker_loop() - end - end - end - listener = GitHub.EventListener(secret=secret) do event - queue() do - revise() - Base.invokelatest(event_callback, api, app_name, app_id, - sourcerepo_installation, commit_sig, listener, bug_repository, event) - end - HTTP.Response(200) - end - GitHub.run(listener, IPv4(0,0,0,0), 10000+app_id) - wait() -end - -end # module +module FemtoCleaner + +# For interactive development +using Revise + +using Base.Distributed +using GitHub +using GitHub: GitHubAPI, GitHubWebAPI, Checks +using HTTP +using Deprecations +using CSTParser +using Deprecations: isexpr +using MbedTLS +using JSON +using AbstractTrees: children +using Base: LibGit2 + +include("workqueue.jl") + +function with_cloned_repo(f, api::GitHubWebAPI, repo, auth) + creds = LibGit2.UserPasswordCredentials(String(copy(Vector{UInt8}("x-access-token"))), String(copy(Vector{UInt8}(auth.token)))) + repo_url = "https://github.com/$(get(repo.full_name))" + local_dir = mktempdir() + try + enabled = gc_enable(false) + lrepo = LibGit2.clone(repo_url, local_dir; payload=Nullable(creds)) + gc_enable(enabled) + f((lrepo, local_dir)) + finally + rm(local_dir, force=true, recursive=true) + end +end + +function with_pr_branch(f, api, repo, auth) + with_cloned_repo(api, repo, auth) do x + LibGit2.branch!(lrepo, "fbot/deps", track=LibGit2.Consts.REMOTE_ORIGIN) + f(x) + end +end + +using LibGit2: GitBlame + +function deprecations_for_repo(lrepo, local_dir, is_julia_itself) + if is_julia_itself + ver = read(joinpath(local_dir, "VERSION"), String) + hunk = GitBlame(lrepo, "VERSION")[1] + l, r = LibGit2.revcount(lrepo, string(hunk.orig_commit_id), "HEAD") + vers = Pkg.Reqs.parse(IOBuffer("julia $ver+$(l+r)")) + else + vers = Pkg.Reqs.parse(joinpath(local_dir, "REQUIRE")) + end + deps = Deprecations.applicable_deprecations(vers) + +end + +function process_deprecations(lrepo, local_dir; is_julia_itself=false, deps = deprecations_for_repo(lrepo, local_dir, is_julia_itself)) + changed_any = false + problematic_files = String[] + all_files = String[] + all_files = String[] + for (root, dirs, files) in walkdir(local_dir) + for file in files + fpath = joinpath(root, file) + (endswith(fpath, ".jl") || endswith(fpath, ".md")) || continue + file == "NEWS.md" && continue + push!(all_files, fpath) + end + end + max_iterations = 30 + iteration_counter = fill(0, length(all_files)) + # Iterate. Some rewrites may expose others + while any(x->x != -1, iteration_counter) + # We need to redo the analysis after every fetmocleaning round, since + # things may have changes as the result of an applied rewrite. + analysis = Deprecations.process_all(filter(f->endswith(f, ".jl"), all_files)) + for (i, fpath) in enumerate(all_files) + iteration_counter[i] == -1 && continue + problematic_file = false + file_analysis = endswith(fpath, ".jl") ? (analysis[1], analysis[2][fpath]) : nothing + try + if !Deprecations.edit_file(fpath, deps, endswith(fpath, ".jl") ? edit_text : edit_markdown; + analysis = file_analysis) + # Nothing to change + iteration_counter[i] = -1 + elseif iteration_counter[i] > max_iterations + warn("Iterations did not converge for file $fpath") + problematic_file = true + else + iteration_counter[i] += 1 + changed_any = true + end + catch e + warn("Exception thrown when fixing file $fpath. Exception was:\n", + sprint(showerror, e, catch_backtrace())) + problematic_file = true + end + if problematic_file + push!(problematic_files, file) + iteration_counter[i] = -1 + end + changed_any && !problematic_file && LibGit2.add!(lrepo, relpath(fpath, local_dir)) + end + end + changed_any, problematic_files +end + +function push_repo(api::GitHubWebAPI, repo, auth; force=true, remote_branch="fbot/deps") + creds = LibGit2.UserPasswordCredentials(String(copy(Vector{UInt8}("x-access-token"))), String(copy(Vector{UInt8}(auth.token)))) + enabled = gc_enable(false) + LibGit2.push(repo, refspecs = ["+HEAD:refs/heads/$remote_branch"], force=force, + payload=Nullable(creds)) + gc_enable(enabled) +end + +struct SourceFile + data::Vector{UInt8} + offsets::Vector{UInt64} +end +Base.length(file::SourceFile) = length(file.offsets) + +function SourceFile(data) + offsets = UInt64[0] + buf = IOBuffer(data) + local line = "" + while !eof(buf) + line = readuntil(buf,'\n') + !eof(buf) && push!(offsets, position(buf)) + end + if !isempty(line) && line[end] == '\n' + push!(offsets, position(buf)+1) + end + SourceFile(data,offsets) +end + +function compute_line(file::SourceFile, offset) + ind = searchsortedfirst(file.offsets, offset) + ind <= length(file.offsets) && file.offsets[ind] == offset ? ind : ind - 1 +end + +function Base.getindex(file::SourceFile, line::Int) + if line == length(file.offsets) + return file.data[(file.offsets[end]+1):end] + else + # - 1 to skip the '\n' + return file.data[(file.offsets[line]+1):max(1, file.offsets[line+1]-1)] + end +end +Base.getindex(file::SourceFile, arr::AbstractArray) = [file[x] for x in arr] + +function repl_to_annotation(fpath, file, lrepo, local_dir, repo, repl) + # Compute blob hash for fpath + blob_hash = LibGit2.addblob!(lrepo, joinpath(local_dir, fpath)) + # Put together description + start_line = compute_line(file, first(repl.range)) + message = """ + $(repl.dep === nothing ? "" : repl.dep.description) + In file $fpath starting at line $(start_line): + $(strip(String(file[start_line]))) + """ + Checks.Annotation( + basename(fpath), + "https://github.com/$(GitHub.name(repo))/blob/$(blob_hash)/$(fpath)", + compute_line(file, first(repl.range)), compute_line(file, last(repl.range)), + "notice", + message, + string(typeof(repl.dep).name.name)[1:min(end, 40)], + "" + ) +end + +function collect_deprecation_annotations(api::GitHubAPI, lrepo, local_dir, repo, auth; is_julia_itself=false) + deps = deprecations_for_repo(lrepo, local_dir, is_julia_itself) + annotations = Checks.Annotation[] + problematic_files = String[] + for (root, dirs, files) in walkdir(local_dir) + for file in files + fpath = joinpath(root, file) + (endswith(fpath, ".jl") || endswith(fpath, ".md")) || continue + file == "NEWS.md" && continue + contents = read(fpath, String) + sfile = SourceFile(contents) + problematic_file = false + try + if endswith(fpath, ".md") + changed_any, _ = Deprecations.edit_markdown(contents, deps) + if changed_any + blob_hash = LibGit2.addblob!(lrepo, joinpath(local_dir, fpath)) + push!(annotations, Checks.Annotation( + basename(fpath), + "https://github.com/$(GitHub.name(repo))/blob/$(blob_hash)/$(fpath)", + 1, 1, + "notice", + """ + Code changes were found in this Markdown document: + $(fpath) + """, + "MarkdownCode", + "" + )) + end + else + repls = Deprecations.text_replacements(contents, deps) + for repl in repls + push!(annotations, repl_to_annotation(fpath, sfile, lrepo, local_dir, repo, repl)) + end + end + catch e + warn("Exception thrown when fixing file $file. Exception was:\n", + sprint(showerror, e, catch_backtrace())) + problematic_file = true + end + problematic_file && push!(problematic_files, file) + end + end + annotations, problematic_files +end + +function apply_deprecations(api::GitHubAPI, lrepo, local_dir, commit_sig, repo, auth; issue_number = 0) + is_julia_itself = GitHub.name(repo) == "JuliaLang/julia" + changed_any, problematic_files = process_deprecations(lrepo, local_dir; is_julia_itself=is_julia_itself) + if changed_any + LibGit2.commit(lrepo, "Fix deprecations"; author=commit_sig, committer=commit_sig, parent_ids=[LibGit2.GitHash(lrepo, "HEAD")]) + push_repo(api, lrepo, auth) + end + if issue_number != 0 + if changed_any + create_pull_request(api, repo, auth=auth, params = Dict( + :issue => issue_number, + :base => get(repo.default_branch), + :head => "fbot/deps" + ) + ) + if !isempty(problematic_files) + create_comment(api, repo, issue_number, :pr, auth=auth, params = Dict( + :body => string("Failed to process the following files: ", + join("`" .* problematic_files .* "`", ", "), + ". :(") + ) + ) + end + println("Created pull request for $(GitHub.name(repo))") + else + create_comment(api, repo, issue_number, :issue, params = Dict( + :body => "No applicable deprecations were found in this repository." + ), auth=auth) + println("Processing complete for $(GitHub.name(repo)): no changes made") + end + else + if changed_any + body = "I fixed a number of deprecations for you" + if !isempty(problematic_files) + body *= string(", but I failed to process the following files: ", + join("`" .* problematic_files .* "`", ", "), + ". :(") + end + create_pull_request(api, repo, auth=auth, params = Dict( + :title => "Fix deprecations", + :body => body, + :base => get(repo.default_branch), + :head => "fbot/deps" + ) + ) + println("Created pull request for $(GitHub.name(repo))") + else + println("Processing complete for $(GitHub.name(repo)): no changes made") + end + end +end + +function my_diff_tree(repo::LibGit2.GitRepo, oldtree::LibGit2.GitTree, newtree::LibGit2.GitTree; pathspecs::AbstractString="") + diff_ptr_ptr = Ref{Ptr{Cvoid}}(C_NULL) + @LibGit2.check ccall((:git_diff_tree_to_tree, :libgit2), Cint, + (Ptr{Ptr{Cvoid}}, Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Cvoid}, Ptr{LibGit2.DiffOptionsStruct}), + diff_ptr_ptr, repo.ptr, oldtree.ptr, newtree.ptr, isempty(pathspecs) ? C_NULL : pathspecs) + return LibGit2.GitDiff(repo, diff_ptr_ptr[]) +end + +function apply_deprecations_if_updated(api::GitHubAPI, lrepo, local_dir, before, after, commit_sig, repo, auth) + before = LibGit2.GitCommit(lrepo, before) + after = LibGit2.GitCommit(lrepo, after) + delta = my_diff_tree(lrepo, LibGit2.peel(before), LibGit2.peel(after); pathspecs="REQUIRE")[1] + old_blob = LibGit2.GitBlob(lrepo, delta.old_file.id) + new_blob = LibGit2.GitBlob(lrepo, delta.new_file.id) + vers_old = Pkg.Reqs.parse(IOBuffer(LibGit2.content(old_blob))) + vers_new = Pkg.Reqs.parse(IOBuffer(LibGit2.content(new_blob))) + if vers_new["julia"] != vers_old["julia"] + apply_deprecations(api, lrepo, local_dir, commit_sig, repo, auth) + end +end + +function cleanrepo(repo_url; show_diff = true, delete_local = true) + local_dir = mktempdir() + successful = true + try + enabled = gc_enable(false) + info("Cloning $repo_url to $local_dir...") + lrepo = LibGit2.clone(repo_url, local_dir) + gc_enable(enabled) + @info("Processing deprecations...") + changed_any, problematic_files = process_deprecations(lrepo, local_dir; is_julia_itself=contains(repo_url, "JuliaLang/julia")) + isempty(problematic_files) || (successful = false) + catch e + bt = catch_backtrace() + Base.display_error(stderr, e, bt) + successful = false + finally + if show_diff + cd(local_dir) do + run(`git status`) + run(`git diff --cached`) + end + end + if delete_local + info("Deleting cloned repo from $local_dir...") + rm(local_dir, force=true, recursive=true) + end + end + return successful +end + +include("interactions.jl") +include("autodeployment.jl") + +const autodeployment_enabled = haskey(ENV, "FEMTOCLEANER_AUTODEPLOY") ? + ENV["FEMTOCLEANER_AUTODEPLOY"] == "yes" : false + +let app_key = Ref{Any}(nothing) + global get_auth + function get_auth(app_id) + if app_key[] == nothing + app_key[] = MbedTLS.PKContext() + MbedTLS.parse_key!(app_key[], haskey(ENV, "FEMTOCLEANER_PRIVKEY") ? ENV["FEMTOCLEANER_PRIVKEY"] : read(joinpath(dirname(@__FILE__),"..","privkey.pem"), String)) + end + GitHub.JWTAuth(app_id, app_key[]) + end +end + +function event_callback(api::GitHubAPI, app_name, app_id, sourcerepo_installation, + commit_sig, listener, bug_repository, event) + # On installation, process every repository we just got installed into + if event.kind == "installation" + jwt = get_auth(app_id) + installation = Installation(event.payload["installation"]) + auth = create_access_token(api, installation, jwt) + for repo in event.payload["repositories"] + repo = GitHub.repo(api, GitHub.Repo(repo); auth=auth) + with_cloned_repo(api, repo, auth) do x + apply_deprecations(api, x..., commit_sig, repo, auth) + end + end + elseif event.kind == "check_run" + jwt = get_auth(app_id) + installation = Installation(event.payload["installation"]) + auth = create_access_token(api, installation, jwt) + if event.payload["action"] == "requested_action" && event.payload["requested_action"]["identifier"] == "fix" + repo = GitHub.Repo(event.payload["repository"]) + pr = PullRequest(event.payload["check_run"]["check_suite"]["pull_requests"][1]) + with_cloned_repo(api, repo, auth) do x + lrepo, local_dir = x + LibGit2.checkout!(lrepo, event.payload["check_run"]["check_suite"]["head_sha"]) + is_julia_itself = GitHub.name(repo) == "JuliaLang/julia" + changed_any, problematic_files = process_deprecations(lrepo, local_dir; is_julia_itself=is_julia_itself) + if changed_any + LibGit2.commit(lrepo, "Fix deprecations"; author=commit_sig, committer=commit_sig, parent_ids=[LibGit2.GitHash(lrepo, "HEAD")]) + push_repo(api, lrepo, auth; force=false, remote_branch=event.payload["check_run"]["check_suite"]["head_branch"]) + end + end + end + elseif event.kind == "pull_request" + if !(event.payload["action"] in ("opened", "reopened", "synchronize")) + return HTTP.Response(200) + end + jwt = get_auth(app_id) + installation = Installation(event.payload["installation"]) + auth = create_access_token(api, installation, jwt) + repo = Repo(event.payload["repository"]) + pr = PullRequest(event.payload["pull_request"]) + local annotations + with_cloned_repo(api, repo, auth) do x + lrepo, local_dir = x + LibGit2.checkout!(lrepo, get(get(pr.head).sha)) + annotations, _ = collect_deprecation_annotations(api, x..., repo, auth) + end + actions = Checks.Action[] + conclusion = "neutral" + if length(annotations) == 0 + message = """ + No applicable deprecations were detected. + """ + output = Checks.Output( + "Femtocleaning", + message, + "", + Checks.Annotation[], + Checks.Image[] + ) + conclusion = "success" + else + truncated = length(annotations) > 50 + message = """ + Several femtocleaning opportunities were detected + """ + output = Checks.Output( + "Femtocleaning", + message, + "See below", + annotations[1:min(50, length(annotations))], + GitHub.Image[] + ) + actions = Checks.Action[ + Checks.Action( + "Fix it!", + "Fixes issues in this PR (adds commit).", + "fix" + ) + ] + end + max_annotation = 50 + cr = GitHub.create_check_run(api, repo, auth=auth, params = Dict( + :name => "femtocleaner", + :head_branch => get(pr.head).ref, + :head_sha => get(pr.head).sha, + :status => "completed", + :conclusion => conclusion, + :completed_at => now(), + :actions => actions, + :output => output + )) + while max_annotation < length(annotations) + empty!(output.annotations) + append!(output.annotations, annotations[max_annotation+1:min(max_annotation+50, end)]) + max_annotation += 50 + GitHub.update_check_run(api, repo, get(cr.id), auth=auth, params = Dict( + :output => output, + :actions => actions + )) + end + elseif event.kind == "installation_repositories" + jwt = get_auth(app_id) + installation = Installation(event.payload["installation"]) + auth = create_access_token(api, installation, jwt) + for repo in event.payload["repositories_added"] + repo = GitHub.repo(api, GitHub.Repo(repo); auth=auth) + with_cloned_repo(api, repo, auth) do x + apply_deprecations(api, x..., commit_sig, repo, auth) + end + end + elseif event.kind == "pull_request_review" + jwt = get_auth(app_id) + pr_response(api, event, jwt, commit_sig, app_name, sourcerepo_installation, bug_repository) + elseif event.kind == "push" + jwt = get_auth(app_id) + installation = Installation(event.payload["installation"]) + auth = create_access_token(api, installation, jwt) + repo = Repo(event.payload["repository"]) + # Check if REQUIRE was updated + for commit in event.payload["commits"] + if "REQUIRE" in commit["modified"] + with_cloned_repo(api, repo, auth) do x + apply_deprecations_if_updated(api, x..., + event.payload["before"], event.payload["after"], + commit_sig, repo, auth) + end + break + end + end + maybe_autdodeploy(event, listener, jwt, sourcerepo_installation, autodeployment_enabled) + elseif event.kind == "issues" && event.payload["action"] == "opened" + jwt = get_auth(app_id) + iss = Issue(event.payload["issue"]) + repo = Repo(event.payload["repository"]) + installation = Installation(event.payload["installation"]) + auth = create_access_token(api, installation, jwt) + if lowercase(get(iss.title)) == "run femtocleaner" + with_cloned_repo(api, repo, auth) do x + apply_deprecations(api, x..., commit_sig, repo, auth; issue_number = get(iss.number)) + end + end + end + return HTTP.Response(200) +end + +function run_server() + app_id = parse(Int, strip(haskey(ENV, "FEMTOCLEANER_APPID") ? ENV["FEMTOCLEANER_APPID"] : read(joinpath(dirname(@__FILE__),"..","app_id"), String))) + secret = haskey(ENV, "FEMTOCLEANER_SECRET") ? ENV["FEMTOCLEANER_SECRET"] : nothing + sourcerepo_installation = haskey(ENV, "FEMTOCLEANER_INSTALLATION") ? parse(Int, ENV["FEMTOCLEANER_INSTALLATION"]) : 0 + bug_repository = haskey(ENV, "FEMTOCLEANER_BUGREPO") ? ENV["FEMTOCLEANER_BUGREPO"] : "" + (secret == nothing) && @warn("Webhook secret not set. All events will be accepted. This is an insecure configuration!") + jwt = get_auth(app_id) + app_name = get(GitHub.app(; auth=jwt).name) + commit_sig = LibGit2.Signature("$(app_name)[bot]", "$(app_name)[bot]@users.noreply.github.com") + api = GitHub.DEFAULT_API + local listener + if nprocs() != 1 + #@everywhere filter(x->x != 1, procs()) worker_loop() + for p in filter(x->x != 1, procs()) + @spawnat p begin + myid() == 2 && update_existing_repos(api, commit_sig, app_id) + worker_loop() + end + end + end + listener = GitHub.EventListener(secret=secret) do event + queue() do + revise() + Base.invokelatest(event_callback, api, app_name, app_id, + sourcerepo_installation, commit_sig, listener, bug_repository, event) + end + HTTP.Response(200) + end + GitHub.run(listener, IPv4(0,0,0,0), 10000+app_id) + wait() +end + +end # module diff --git a/src/autodeployment.jl b/src/autodeployment.jl index ba84246..473e48c 100644 --- a/src/autodeployment.jl +++ b/src/autodeployment.jl @@ -56,7 +56,7 @@ function update_existing_repos(api, commit_sig, app_id) end catch e bt = catch_backtrace() - Base.display_error(STDERR, e, bt) + Base.display_error(stderr, e, bt) end end end @@ -67,22 +67,22 @@ function maybe_autdodeploy(event, listener, jwt, sourcerepo_installation, enable (GitHub.name(repo) == "JuliaComputing/FemtoCleaner.jl") || return (event.payload["ref"] == "refs/heads/master") || return if !enabled - warn("Push event received, but auto deployment is disabled") + @warn("Push event received, but auto deployment is disabled") return end - info("Commencing auto deployment") + @info("Commencing auto deployment") with(GitRepo, Pkg.dir("FemtoCleaner")) do repo LibGit2.fetch(repo) ahead_remote, ahead_local = LibGit2.revcount(repo, "origin/master", "master") rcount = min(ahead_remote, ahead_local) if ahead_local-rcount > 0 - warn("Local repository has more commits that origin. Aborting") + @warn("Local repository has more commits that origin. Aborting") return false end # Shut down the server, so the new process can replace it close(listener.server) LibGit2.reset!(repo, LibGit2.GitHash(event.payload["after"]), LibGit2.Consts.RESET_HARD) - for (pkg, version) in JSON.parse(readstring(joinpath(dirname(@__FILE__),"..","dependencies.json"))) + for (pkg, version) in JSON.parse(read(joinpath(dirname(@__FILE__),"..","dependencies.json"), String)) with(GitRepo, Pkg.dir(pkg)) do deprepo for remote in LibGit2.remotes(deprepo) LibGit2.fetch(deprepo; remote=remote) diff --git a/src/blame.jl b/src/blame.jl index 2bd2981..d9827b7 100644 --- a/src/blame.jl +++ b/src/blame.jl @@ -23,8 +23,8 @@ for (typ, owntyp, sup, cname) in [ if owntyp === nothing @eval mutable struct $typ <: $sup - ptr::Ptr{Void} - function $typ(ptr::Ptr{Void}, fin::Bool=true) + ptr::Ptr{Cvoid} + function $typ(ptr::Ptr{Cvoid}, fin::Bool=true) # fin=false should only be used when the pointer should not be free'd # e.g. from within callback functions which are passed a pointer @assert ptr != C_NULL @@ -39,8 +39,8 @@ for (typ, owntyp, sup, cname) in [ else @eval mutable struct $typ <: $sup owner::$owntyp - ptr::Ptr{Void} - function $typ(owner::$owntyp, ptr::Ptr{Void}, fin::Bool=true) + ptr::Ptr{Cvoid} + function $typ(owner::$owntyp, ptr::Ptr{Cvoid}, fin::Bool=true) @assert ptr != C_NULL obj = new(owner, ptr) if fin @@ -52,15 +52,15 @@ for (typ, owntyp, sup, cname) in [ end if isa(owntyp, Expr) && owntyp.args[1] == :Nullable @eval begin - $typ(ptr::Ptr{Void}, fin::Bool=true) = $typ($owntyp(), ptr, fin) - $typ(owner::$(owntyp.args[2]), ptr::Ptr{Void}, fin::Bool=true) = + $typ(ptr::Ptr{Cvoid}, fin::Bool=true) = $typ($owntyp(), ptr, fin) + $typ(owner::$(owntyp.args[2]), ptr::Ptr{Cvoid}, fin::Bool=true) = $typ($owntyp(owner), ptr, fin) end end end @eval function Base.close(obj::$typ) if obj.ptr != C_NULL - ccall(($(string(cname, :_free)), :libgit2), Void, (Ptr{Void},), obj.ptr) + ccall(($(string(cname, :_free)), :libgit2), Cvoid, (Ptr{Cvoid},), obj.ptr) obj.ptr = C_NULL if Threads.atomic_sub!(REFCOUNT, UInt(1)) == 1 # will the last finalizer please turn out the lights? @@ -108,15 +108,15 @@ The fields represent: end function GitBlame(repo::GitRepo, path::AbstractString; options::BlameOptions=BlameOptions()) - blame_ptr_ptr = Ref{Ptr{Void}}(C_NULL) + blame_ptr_ptr = Ref{Ptr{Cvoid}}(C_NULL) @Base.LibGit2.check ccall((:git_blame_file, :libgit2), Cint, - (Ptr{Ptr{Void}}, Ptr{Void}, Cstring, Ptr{BlameOptions}), + (Ptr{Ptr{Cvoid}}, Ptr{Cvoid}, Cstring, Ptr{BlameOptions}), blame_ptr_ptr, repo.ptr, path, Ref(options)) return GitBlame(repo, blame_ptr_ptr[]) end function counthunks(blame::GitBlame) - return ccall((:git_blame_get_hunk_count, :libgit2), Int32, (Ptr{Void},), blame.ptr) + return ccall((:git_blame_get_hunk_count, :libgit2), Int32, (Ptr{Cvoid},), blame.ptr) end function Base.getindex(blame::GitBlame, i::Integer) @@ -125,7 +125,7 @@ function Base.getindex(blame::GitBlame, i::Integer) end hunk_ptr = ccall((:git_blame_get_hunk_byindex, :libgit2), Ptr{BlameHunk}, - (Ptr{Void}, Csize_t), blame.ptr, i-1) + (Ptr{Cvoid}, Csize_t), blame.ptr, i-1) return unsafe_load(hunk_ptr) end diff --git a/src/interactions.jl b/src/interactions.jl index a8cb4ff..a1fe237 100644 --- a/src/interactions.jl +++ b/src/interactions.jl @@ -22,7 +22,7 @@ function first_expr_in_range(tree, range) end function with_node_at_line(f, path, line) - text = readstring(path) + text = read(path, String) p = Deprecations.overlay_parse(text, true) @assert !isexpr(p, CSTParser.ERROR) range = byte_range_for_line(text, line) @@ -130,7 +130,7 @@ function pr_response(api, event, jwt, commit_sig, app_name, sourcerepo_installat end paths = unique(map(r->r.path, resolutions)) for path in paths - text = readstring(joinpath(local_dir, path)) + text = read(joinpath(local_dir, path), String) open(joinpath(local_dir, path), "w") do f changes = collect(map(x->x.repl, filter(x->x.path == path, resolutions))) diff --git a/src/workqueue.jl b/src/workqueue.jl index 2bcd2d1..3b8dad3 100644 --- a/src/workqueue.jl +++ b/src/workqueue.jl @@ -12,7 +12,7 @@ function worker_loop() end)() catch e bt = catch_backtrace() - Base.showerror(STDERR, e, bt) + Base.showerror(stderr, e, bt) end end end diff --git a/test/dry_runs.jl b/test/dry_runs.jl index f2fc28c..4df832d 100644 --- a/test/dry_runs.jl +++ b/test/dry_runs.jl @@ -1,5 +1,5 @@ import FemtoCleaner -using Base.Test +using Test println("Dry running DataFrames") @test FemtoCleaner.cleanrepo("https://github.com/JuliaData/DataFrames.jl"; show_diff = false) diff --git a/test/runtests.jl b/test/runtests.jl index 18629d7..f161045 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,5 +1,5 @@ using FemtoCleaner -using Base.Test +using Test using GitHub using GitHub: WebhookEvent, GitHubAPI @@ -77,8 +77,8 @@ end fake_app_key = MbedTLS.PKContext() -MbedTLS.parse_key!(fake_app_key, readstring( - joinpath(Pkg.dir("GitHub","test"), "not_a_real_key.pem"))) +MbedTLS.parse_key!(fake_app_key, read( + joinpath(Pkg.dir("GitHub","test"), "not_a_real_key.pem"), String)) app_name = "femtocleaner-test" test_commit_sig = LibGit2.Signature("$(app_name)[bot]", "$(app_name)[bot]@users.noreply.github.com") From ff6595f3a6de3290fd933cd94835c73aee289431 Mon Sep 17 00:00:00 2001 From: aminya Date: Wed, 14 Aug 2019 18:06:27 -0500 Subject: [PATCH 04/12] Project.toml Update README.md Update .travis.yml Create Project.toml --- .travis.yml | 14 ++++++++++---- Project.toml | 19 +++++++++++++++++++ README.md | 2 ++ 3 files changed, 31 insertions(+), 4 deletions(-) create mode 100644 Project.toml diff --git a/.travis.yml b/.travis.yml index 7f22131..229f63c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,15 +2,21 @@ language: julia os: - linux + - osx + - windows julia: - - 0.6 + - 1.0 + - nightly +matrix: + allow_failures: + - julia: nightly + fast_finish: true notifications: - email: false -# uncomment the following lines to override the default test script + email: falseript script: - if [[ -a .git/shallow ]]; then git fetch --unshallow; fi - julia -e ' - Pkg.clone("https://github.com/JuliaComputing/Deprecations.jl"); + Pkg.clone("https://github.com/aminya/Deprecations.jl"); Pkg.clone(pwd()); Pkg.build("FemtoCleaner"); Pkg.checkout("AbstractTrees"); import JSON; diff --git a/Project.toml b/Project.toml new file mode 100644 index 0000000..68be0a0 --- /dev/null +++ b/Project.toml @@ -0,0 +1,19 @@ +name = "FemtoCleaner" +uuid = "9af2f830-beea-11e9-1088-b32cf2144e57" +authors = ["aminya "] +version = "0.1.0" + +[compat] +julia = "1" + +[deps] +Deprecations="687c44b0-beea-11e9-3666-0b52fd97bbca" +CSTParser = "00ebfdb7-1f24-5e51-bd34-a7502290713f" +GitHub="bc5e4493-9b4d-5f90-b8aa-2b2bcaad7a26" +Revise="295af30f-e4ad-537b-8983-00126c2a3abe" + +[extras] +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[targets] +test = ["Test"] diff --git a/README.md b/README.md index 1546756..412d16b 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,6 @@ # FemtoCleaner +This pull request is under development for Julia 1.0. Use https://github.com/aminya/FemtoCleaner.jl for running locally on Julia 0.6.4 +

serious femtocleaning

From 64129c28b2cc9a0a07748e99a89ba93e50151b32 Mon Sep 17 00:00:00 2001 From: aminya Date: Wed, 14 Aug 2019 19:26:38 -0500 Subject: [PATCH 05/12] Update Project.toml --- Project.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/Project.toml b/Project.toml index 68be0a0..1f9ad22 100644 --- a/Project.toml +++ b/Project.toml @@ -7,7 +7,6 @@ version = "0.1.0" julia = "1" [deps] -Deprecations="687c44b0-beea-11e9-3666-0b52fd97bbca" CSTParser = "00ebfdb7-1f24-5e51-bd34-a7502290713f" GitHub="bc5e4493-9b4d-5f90-b8aa-2b2bcaad7a26" Revise="295af30f-e4ad-537b-8983-00126c2a3abe" From e21f5752d553c43223144ebd281d7be729c2e082 Mon Sep 17 00:00:00 2001 From: aminya Date: Wed, 14 Aug 2019 19:43:30 -0500 Subject: [PATCH 06/12] Adding Manifest.jl to add dependency for custom Deprecations.jl --- Manifest.toml | 58 +++++++++++++++++++++++++++++++++++++++++++++++++++ Project.toml | 1 + 2 files changed, 59 insertions(+) create mode 100644 Manifest.toml diff --git a/Manifest.toml b/Manifest.toml new file mode 100644 index 0000000..5ec8764 --- /dev/null +++ b/Manifest.toml @@ -0,0 +1,58 @@ +# This file is machine-generated - editing it directly is not advised + +[[AbstractTrees]] +deps = ["Markdown", "Test"] +git-tree-sha1 = "6621d9645702c1c4e6970cc6a3eae440c768000b" +uuid = "1520ce14-60c1-5f80-bbc7-55ef81b5835c" +version = "0.2.1" + +[[Base64]] +uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" + +[[CSTParser]] +deps = ["Tokenize"] +git-tree-sha1 = "c69698c3d4a7255bc1b4bc2afc09f59db910243b" +uuid = "00ebfdb7-1f24-5e51-bd34-a7502290713f" +version = "0.6.2" + +[[Deprecations]] +deps = ["AbstractTrees", "CSTParser", "Tokenize"] +git-tree-sha1 = "0b93c922f6f665ee21b156c94ef81bd4e738e5e7" +repo-rev = "master" +repo-url = "https://github.com/aminya/Deprecations.jl" +uuid = "687c44b0-beea-11e9-3666-0b52fd97bbca" +version = "0.1.0" + +[[Distributed]] +deps = ["Random", "Serialization", "Sockets"] +uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b" + +[[InteractiveUtils]] +deps = ["Markdown"] +uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" + +[[Logging]] +uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" + +[[Markdown]] +deps = ["Base64"] +uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" + +[[Random]] +deps = ["Serialization"] +uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" + +[[Serialization]] +uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" + +[[Sockets]] +uuid = "6462fe0b-24de-5631-8697-dd941f90decc" + +[[Test]] +deps = ["Distributed", "InteractiveUtils", "Logging", "Random"] +uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[[Tokenize]] +git-tree-sha1 = "dfcdbbfb2d0370716c815cbd6f8a364efb6f42cf" +uuid = "0796e94c-ce3b-5d07-9a54-7f471281c624" +version = "0.5.6" diff --git a/Project.toml b/Project.toml index 1f9ad22..6f5d608 100644 --- a/Project.toml +++ b/Project.toml @@ -7,6 +7,7 @@ version = "0.1.0" julia = "1" [deps] +Deprecations = "687c44b0-beea-11e9-3666-0b52fd97bbca" CSTParser = "00ebfdb7-1f24-5e51-bd34-a7502290713f" GitHub="bc5e4493-9b4d-5f90-b8aa-2b2bcaad7a26" Revise="295af30f-e4ad-537b-8983-00126c2a3abe" From 0d101db163664fd8ead0afde27d0dea277949dfa Mon Sep 17 00:00:00 2001 From: aminya Date: Wed, 14 Aug 2019 19:52:52 -0500 Subject: [PATCH 07/12] Add instruction for adding --- README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 412d16b..c9b4fa9 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,10 @@ # FemtoCleaner This pull request is under development for Julia 1.0. Use https://github.com/aminya/FemtoCleaner.jl for running locally on Julia 0.6.4 - +To add this current developing package: +```julia +] add https://github.com/aminya/Deprecations.jl +] add https://github.com/aminya/FemtoCleaner.jl.git#Amin_Julia1 +```

serious femtocleaning

From f976085d3a8994d2ec9bdcdfbef53d50620a36c8 Mon Sep 17 00:00:00 2001 From: aminya Date: Wed, 14 Aug 2019 19:57:02 -0500 Subject: [PATCH 08/12] Remove Base. for Distributed --- src/FemtoCleaner.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/FemtoCleaner.jl b/src/FemtoCleaner.jl index 0874efd..3ea5850 100644 --- a/src/FemtoCleaner.jl +++ b/src/FemtoCleaner.jl @@ -3,7 +3,7 @@ module FemtoCleaner # For interactive development using Revise -using Base.Distributed +using Distributed using GitHub using GitHub: GitHubAPI, GitHubWebAPI, Checks using HTTP From f00415d86d47322160cf9cef31515247c222d68b Mon Sep 17 00:00:00 2001 From: aminya Date: Wed, 14 Aug 2019 20:01:32 -0500 Subject: [PATCH 09/12] Add HTTP dependency --- Project.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/Project.toml b/Project.toml index 6f5d608..b10116f 100644 --- a/Project.toml +++ b/Project.toml @@ -11,6 +11,7 @@ Deprecations = "687c44b0-beea-11e9-3666-0b52fd97bbca" CSTParser = "00ebfdb7-1f24-5e51-bd34-a7502290713f" GitHub="bc5e4493-9b4d-5f90-b8aa-2b2bcaad7a26" Revise="295af30f-e4ad-537b-8983-00126c2a3abe" +HTTP="cd3eb016-35fb-5094-929b-558a96fad6f3" [extras] Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" From eb065044f084c4e3f1d65c4f99fe099eacd259fc Mon Sep 17 00:00:00 2001 From: aminya Date: Wed, 14 Aug 2019 21:25:29 -0500 Subject: [PATCH 10/12] Compat CSTParser --- Manifest.toml | 2 +- Project.toml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Manifest.toml b/Manifest.toml index 5ec8764..5f35427 100644 --- a/Manifest.toml +++ b/Manifest.toml @@ -13,7 +13,7 @@ uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" deps = ["Tokenize"] git-tree-sha1 = "c69698c3d4a7255bc1b4bc2afc09f59db910243b" uuid = "00ebfdb7-1f24-5e51-bd34-a7502290713f" -version = "0.6.2" +version = "0.4" [[Deprecations]] deps = ["AbstractTrees", "CSTParser", "Tokenize"] diff --git a/Project.toml b/Project.toml index b10116f..7584779 100644 --- a/Project.toml +++ b/Project.toml @@ -5,6 +5,7 @@ version = "0.1.0" [compat] julia = "1" +CSTParser="0.4" [deps] Deprecations = "687c44b0-beea-11e9-3666-0b52fd97bbca" From 5e59da2f9586e5ed8556234f70bf718bdcaf617a Mon Sep 17 00:00:00 2001 From: aminya Date: Wed, 14 Aug 2019 21:35:20 -0500 Subject: [PATCH 11/12] Update REQUIRE --- REQUIRE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/REQUIRE b/REQUIRE index 81efb61..e61da6f 100644 --- a/REQUIRE +++ b/REQUIRE @@ -1,5 +1,5 @@ julia 1.0 Deprecations -CSTParser +CSTParser 0.4 GitHub Revise From 3036cfce993e1c60036b2211d306bdb84577d753 Mon Sep 17 00:00:00 2001 From: aminya Date: Wed, 14 Aug 2019 21:57:06 -0500 Subject: [PATCH 12/12] CSTParser 0.5 --- Manifest.toml | 2 +- Project.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Manifest.toml b/Manifest.toml index 5f35427..18d2d5d 100644 --- a/Manifest.toml +++ b/Manifest.toml @@ -13,7 +13,7 @@ uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" deps = ["Tokenize"] git-tree-sha1 = "c69698c3d4a7255bc1b4bc2afc09f59db910243b" uuid = "00ebfdb7-1f24-5e51-bd34-a7502290713f" -version = "0.4" +version = "0.5" [[Deprecations]] deps = ["AbstractTrees", "CSTParser", "Tokenize"] diff --git a/Project.toml b/Project.toml index 7584779..1a1fd91 100644 --- a/Project.toml +++ b/Project.toml @@ -5,7 +5,7 @@ version = "0.1.0" [compat] julia = "1" -CSTParser="0.4" +CSTParser="0.5" [deps] Deprecations = "687c44b0-beea-11e9-3666-0b52fd97bbca"