Skip to content

Commit

Permalink
Avoid duplicate stat calls during startup/loading (#55331)
Browse files Browse the repository at this point in the history
Avoids immediately successive stat calls for the same path during
startup & loading.

According to MacOS Instruments this reduces `stat64` calls during
`--start=no -e "using Pkg"` from 844 to 672.

On MacOS I don't see a speed improvement, but on WSL2 @timholy reported
the test from #55171 sees a
modest improvement.

This PR (10 iterations)
```
tim@diva:~/.julia/bin$ time for i in {1..10}; do ./cli --help &> /dev/null; done

real    0m2.999s
user    0m3.547s
sys     0m6.552s
```

master
```
real    0m3.217s
user    0m3.794s
sys     0m6.700s
```
1.11-rc2:
```
real    0m3.746s
user    0m4.169s
sys     0m6.755s
```

I left the `@debug`s in as they might be useful for similar
investigations.
  • Loading branch information
IanButterworth authored Aug 2, 2024
1 parent 19165be commit c6732a7
Show file tree
Hide file tree
Showing 3 changed files with 17 additions and 7 deletions.
2 changes: 2 additions & 0 deletions base/client.jl
Original file line number Diff line number Diff line change
Expand Up @@ -339,11 +339,13 @@ function _global_julia_startup_file()
# If it is not found, then continue on to the relative path based on Sys.BINDIR
BINDIR = Sys.BINDIR
SYSCONFDIR = Base.SYSCONFDIR
p1 = nothing
if !isempty(SYSCONFDIR)
p1 = abspath(BINDIR, SYSCONFDIR, "julia", "startup.jl")
isfile(p1) && return p1
end
p2 = abspath(BINDIR, "..", "etc", "julia", "startup.jl")
p1 == p2 && return nothing # don't check the same path twice
isfile(p2) && return p2
return nothing
end
Expand Down
12 changes: 7 additions & 5 deletions base/loading.jl
Original file line number Diff line number Diff line change
Expand Up @@ -822,14 +822,15 @@ end
# given a project directory (implicit env from LOAD_PATH) and a name,
# find an entry point for `name`, and see if it has an associated project file
function entry_point_and_project_file(dir::String, name::String)::Union{Tuple{Nothing,Nothing},Tuple{String,Nothing},Tuple{String,String}}
path = normpath(joinpath(dir, "$name.jl"))
isfile_casesensitive(path) && return path, nothing
dir_name = joinpath(dir, name)
path, project_file = entry_point_and_project_file_inside(dir_name, name)
path === nothing || return path, project_file
dir_jl = dir_name * ".jl"
path, project_file = entry_point_and_project_file_inside(dir_jl, name)
path === nothing || return path, project_file
# check for less likely case with a bare file and no src directory last to minimize stat calls
path = normpath(joinpath(dir, "$name.jl"))
isfile_casesensitive(path) && return path, nothing
return nothing, nothing
end

Expand Down Expand Up @@ -3809,7 +3810,7 @@ end
end
if !ispath(f)
_f = fixup_stdlib_path(f)
if isfile(_f) && startswith(_f, Sys.STDLIB)
if _f != f && isfile(_f) && startswith(_f, Sys.STDLIB)
continue
end
@debug "Rejecting stale cache file $cachefile because file $f does not exist"
Expand All @@ -3831,13 +3832,14 @@ end
return true
end
else
fsize = filesize(f)
fstat = stat(f)
fsize = filesize(fstat)
if fsize != fsize_req
@debug "Rejecting stale cache file $cachefile because file size of $f has changed (file size $fsize, before $fsize_req)"
record_reason(reasons, "include_dependency fsize change")
return true
end
hash = isdir(f) ? _crc32c(join(readdir(f))) : open(_crc32c, f, "r")
hash = isdir(fstat) ? _crc32c(join(readdir(f))) : open(_crc32c, f, "r")
if hash != hash_req
@debug "Rejecting stale cache file $cachefile because hash of $f has changed (hash $hash, before $hash_req)"
record_reason(reasons, "include_dependency fhash change")
Expand Down
10 changes: 8 additions & 2 deletions base/stat.jl
Original file line number Diff line number Diff line change
Expand Up @@ -184,8 +184,14 @@ macro stat_call(sym, arg1type, arg)
end

stat(fd::OS_HANDLE) = @stat_call jl_fstat OS_HANDLE fd
stat(path::AbstractString) = @stat_call jl_stat Cstring path
lstat(path::AbstractString) = @stat_call jl_lstat Cstring path
function stat(path::AbstractString)
# @info "stat($(repr(path)))" exception=(ErrorException("Fake error for backtrace printing"),stacktrace())
@stat_call jl_stat Cstring path
end
function lstat(path::AbstractString)
# @info "lstat($(repr(path)))" exception=(ErrorException("Fake error for backtrace printing"),stacktrace())
@stat_call jl_lstat Cstring path
end
if RawFD !== OS_HANDLE
global stat(fd::RawFD) = stat(Libc._get_osfhandle(fd))
end
Expand Down

0 comments on commit c6732a7

Please sign in to comment.