Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Make the default GC_INITIAL_HEAP_SIZE a lot bigger #11130

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Make the default GC_INITIAL_HEAP_SIZE a lot bigger #11130

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 40 additions & 9 deletions src/libexpr/eval-gc.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
#include "environment-variables.hh"
#include "serialise.hh"
#include "eval-gc.hh"
#include "file-system.hh"

#if HAVE_BOEHMGC

Expand Down Expand Up @@ -143,6 +144,39 @@ class BoehmDisableGC
};
};

static size_t getFreeMem()
{
/* On Linux, use the `MemAvailable` or `MemFree` fields from
/proc/cpuinfo. */
# if __linux__
{
std::unordered_map<std::string, std::string> fields;
for (auto & line : tokenizeString<std::vector<std::string>>(readFile("/proc/meminfo"), "\n")) {
auto colon = line.find(':');
if (colon == line.npos)
continue;
fields.emplace(line.substr(0, colon), trim(line.substr(colon + 1)));
}

auto i = fields.find("MemAvailable");
if (i == fields.end())
i = fields.find("MemFree");
if (i != fields.end()) {
Comment on lines +161 to +164
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think this number might be misleading on zfs (which is quite popular with NixOS user) as zfs ARC allocations are not accounted in page cache. There was a trick though that htop uses to get a more correct number.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more. 

# Get total memory in kB
total_mem=$(grep MemTotal /proc/meminfo | awk '{print $2}')

# Get free memory in kB
free_mem=$(grep MemFree /proc/meminfo | awk '{print $2}')

# Get available memory in kB
avail_mem=$(grep MemAvailable /proc/meminfo | awk '{print $2}')

# Get ARC memory in bytes from arcstats
arc_mem=$(awk '/^size/ {print $3}' /proc/spl/kstat/zfs/arcstats)

# Convert total memory to bytes for consistency
total_mem=$((total_mem * 1024))

# Calculate used memory excluding ARC (in bytes)
used_mem_excluding_arc=$((total_mem - free_mem * 1024 - avail_mem * 1024 + arc_mem))

# Convert result to a more readable format (kB)
used_mem_excluding_arc_kb=$((used_mem_excluding_arc / 1024))

# Print the result
echo "Used memory excluding ARC: $used_mem_excluding_arc_kb kB"

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Here is an example arcstats: https://gist.github.com/Mic92/b2d37f5f8ed7f93dff432030ec8f1c67

auto kb = tokenizeString<std::vector<std::string>>(i->second, " ");
if (kb.size() == 2 && kb[1] == "kB")
return string2Int<size_t>(kb[0]).value_or(0) * 1024;
}
}
# endif

/* On non-Linux systems, conservatively assume that 25% of memory is free. */
long pageSize = sysconf(_SC_PAGESIZE);
long pages = sysconf(_SC_PHYS_PAGES);
if (pageSize != -1)
return (pageSize * pages) / 4;
return 0;
}

static inline void initGCReal()
{
/* Initialise the Boehm garbage collector. */
Expand Down Expand Up @@ -177,8 +211,8 @@ static inline void initGCReal()
"BoehmGC version does not support GC while coroutine exists. GC will be disabled inside coroutines. Consider updating bdw-gc to 8.2.4 or later."
# endif

/* Set the initial heap size to something fairly big (25% of
physical RAM, up to a maximum of 384 MiB) so that in most cases
/* Set the initial heap size to something fairly big (80% of
free RAM, up to a maximum of 8 GiB) so that in most cases
we don't need to garbage collect at all. (Collection has a
fairly significant overhead.) The heap size can be overridden
through libgc's GC_INITIAL_HEAP_SIZE environment variable. We
Expand All @@ -189,13 +223,10 @@ static inline void initGCReal()
if (!getEnv("GC_INITIAL_HEAP_SIZE")) {
size_t size = 32 * 1024 * 1024;
# if HAVE_SYSCONF && defined(_SC_PAGESIZE) && defined(_SC_PHYS_PAGES)
size_t maxSize = 384 * 1024 * 1024;
long pageSize = sysconf(_SC_PAGESIZE);
long pages = sysconf(_SC_PHYS_PAGES);
if (pageSize != -1)
size = (pageSize * pages) / 4; // 25% of RAM
if (size > maxSize)
size = maxSize;
size_t maxSize = 8ULL * 1024 * 1024 * 1024;
auto free = getFreeMem();
debug("free memory is %d bytes", free);
size = std::min((size_t) (free * 0.8), maxSize);
# endif
debug("setting initial heap size to %1% bytes", size);
GC_expand_hp(size);
Expand Down
Loading