From 13f37efe12fba0b778f2caff4188cbd71136a3cb Mon Sep 17 00:00:00 2001 From: Ryan Houdek Date: Sun, 12 Dec 2021 22:40:26 -0800 Subject: [PATCH 1/4] SquashFS: Work around a race condition on FEXMountDaemon ack On an overburdened system then FEXMountDaemon may notrespond within the two seconds of FEX waiting for an ack. In this case then we hit a bad edge case where we overwrite the lock file with a new FEXMountDaemon instance then spin up a new FEXMountDaemon. This new FEXMountDaemon will recognize that one is still running and early exit, but now that we've overwritten the lock file with a new location, all future FEX instances won't be able to find the squashfs rootfs. Instead, ignore that the ack wasn't received and attempt running instead. FEXMountDaemon should still pick up the pipe message and monitor FEX regardless. This fixes the `/usr not found` messages and crashes resulting from it. --- Source/Common/RootFSSetup.cpp | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/Source/Common/RootFSSetup.cpp b/Source/Common/RootFSSetup.cpp index ffe8a8933..4fe4ebfb3 100644 --- a/Source/Common/RootFSSetup.cpp +++ b/Source/Common/RootFSSetup.cpp @@ -155,14 +155,13 @@ bool SendSocketPipe(std::string const &MountPath) { int Result = ppoll(&pfd, 1, &ts, nullptr); if (Result == -1 || Result == 0) { // didn't get ack back in time - // Close our read pipe - close(fds[0]); - // close our write pipe - close(fds[1]); - - // close socket - close(socket_fd); - return false; + // Assume an overburdened system at this point + // If the FEXMountDaemon is alive but slept for more than our timeout + // then we can spuriously throw errors + // + // Returning false here would cause FEX to try and spin up a new FEXMountDaemon + // and then the FEXMountDaemon would check to see if the lock exists + // Then would early exit and not mount a new path } // We've sent the message which means we're done with the socket From e2004f49995b9665a1d62b68b139555b79c6f630 Mon Sep 17 00:00:00 2001 From: Ryan Houdek Date: Sun, 12 Dec 2021 22:45:40 -0800 Subject: [PATCH 2/4] FEXMountDaemon: Fix leaked pipe FDs Even though we were removing the pipe from the epoll interest list, we were failing to close the pipe fd. This was leaking the FD and running in to the Linux process FD limitation over time. This fixes a class of issues where if you were running FEXMountDaemon that hit the max, then there were FEX processes not being watched. Which means that FEXMountDaemon could exit while FEX was still trying to run. Usually a non-issue since squashfuse would keep running in the background since open files in the mount were still in use. Could hit fun race conditions because of it though. --- Source/Tools/FEXMountDaemon/Main.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Source/Tools/FEXMountDaemon/Main.cpp b/Source/Tools/FEXMountDaemon/Main.cpp index dc36c8984..9d402688c 100644 --- a/Source/Tools/FEXMountDaemon/Main.cpp +++ b/Source/Tools/FEXMountDaemon/Main.cpp @@ -60,6 +60,9 @@ namespace EPollWatcher { void RemovePipeToWatch(int pipe) { int Result = epoll_ctl(epoll_fd, EPOLL_CTL_DEL, pipe, nullptr); + + // Make sure to close the pipe to not leak the FD + close(pipe); if (Result == -1) { fprintf(stderr, "[FEXMountDaemon] epoll_ctl returned error %d %s\n", errno, strerror(errno)); } From 3e0e922fefe5d0fd4bac3b0d310d49016cd6e330 Mon Sep 17 00:00:00 2001 From: Ryan Houdek Date: Sun, 12 Dec 2021 22:49:17 -0800 Subject: [PATCH 3/4] FEXMountDaemon: Fixes EPoll thread not shutting down on SIGTERM This is another edge case where if the process was sent a SIGTERM then it can get in to a weird state where pipe tracking is broken. We already check for zero pipes being tracked at the end of this loop, just check for the shutdown variable to be set instead --- Source/Tools/FEXMountDaemon/Main.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/Tools/FEXMountDaemon/Main.cpp b/Source/Tools/FEXMountDaemon/Main.cpp index 9d402688c..144c203e3 100644 --- a/Source/Tools/FEXMountDaemon/Main.cpp +++ b/Source/Tools/FEXMountDaemon/Main.cpp @@ -84,7 +84,7 @@ namespace EPollWatcher { // Spin while we are not shutting down // Also spin while we have pipes to watch - while (!EPollWatcherShutdown.load() || NumPipesRemaining() != 0) { + while (!EPollWatcherShutdown.load()) { // Loop every ten seconds // epoll_pwait2 only available since kernel 5.11... int Result = epoll_pwait(epoll_fd, Events, MAX_EVENTS, 10 * 1000, nullptr); From 5d0734a7f2248a18e7a4929f192bd2ca75cdfdde Mon Sep 17 00:00:00 2001 From: Ryan Houdek Date: Sun, 12 Dec 2021 22:51:53 -0800 Subject: [PATCH 4/4] FEXMountDaemon: Attempt to increase FD max when close to limit On FEXMountDaemon startup, fetch the FD limit and current number of open files. This allows us to track the number of open FDs we have. Once we get close the the safe threshold we then try increasing the soft limit towards the hard limit. If we bump up to the max (Say someone setting the hard limit to 256) then print some fairly big warning messages. With the previously fixed FD leak, we could very quickly hit the pipe limit because Steam is constantly opening new processes all the time. With the leak fixed, running Steam and idling, FEXMountDaemon only has around 16 pipes being watched. --- Source/Tools/FEXMountDaemon/Main.cpp | 54 ++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/Source/Tools/FEXMountDaemon/Main.cpp b/Source/Tools/FEXMountDaemon/Main.cpp index 144c203e3..2b1008e03 100644 --- a/Source/Tools/FEXMountDaemon/Main.cpp +++ b/Source/Tools/FEXMountDaemon/Main.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -35,7 +36,10 @@ namespace EPollWatcher { static int epoll_fd{}; static std::thread EPollThread{}; static std::atomic EPollThreadTID{}; + static rlimit MaxFDs{}; + constexpr size_t static MAX_FD_DISTANCE = 32; std::atomic NumPipesWatched{}; + std::atomic NumFilesOpened{}; std::atomic EPollWatcherShutdown {false}; std::chrono::time_point TimeWhileZeroFDs{}; // Timeout is ten seconds @@ -45,6 +49,49 @@ namespace EPollWatcher { return NumPipesWatched.load(); } + size_t GetNumFilesOpen() { + // Walk /proc/self/fd/ to see how many open files we currently have + const std::filesystem::path self{"/proc/self/fd/"}; + + return std::distance(std::filesystem::directory_iterator{self}, std::filesystem::directory_iterator{}); + } + + void GetMaxFDs() { + // Get our kernel limit for the number of open files + if (getrlimit(RLIMIT_NOFILE, &MaxFDs) != 0) { + fprintf(stderr, "[FEXMountDaemon] getrlimit(RLIMIT_NOFILE) returned error %d %s\n", errno, strerror(errno)); + } + + // Walk /proc/self/fd/ to see how many open files we currently have + NumFilesOpened = GetNumFilesOpen(); + } + + void RaiseFDLimit() { + if (MaxFDs.rlim_cur == MaxFDs.rlim_max) { + fprintf(stderr, "[FEXMountDaemon] Our open FD limit is already set to max and we are wanting to increase it\n"); + fprintf(stderr, "[FEXMountDaemon] FEXMountDaemon will now no longer be able to track new instances of FEX\n"); + fprintf(stderr, "[FEXMountDaemon] Current limit is %zd(hard %zd) FDs and we are at %zd\n", MaxFDs.rlim_cur, MaxFDs.rlim_max, GetNumFilesOpen()); + fprintf(stderr, "[FEXMountDaemon] Ask your administrator to raise your kernel's hard limit on open FDs\n"); + return; + } + + rlimit NewLimit = MaxFDs; + + // Just multiply by two + NewLimit.rlim_cur <<= 1; + + // Now limit to the hard max + NewLimit.rlim_cur = std::min(NewLimit.rlim_cur, NewLimit.rlim_max); + + if (setrlimit(RLIMIT_NOFILE, &NewLimit) != 0) { + fprintf(stderr, "[FEXMountDaemon] Couldn't raise FD limit to %zd even though our hard limit is %zd\n", NewLimit.rlim_cur, NewLimit.rlim_max); + } + else { + // Set the new limit + MaxFDs = NewLimit; + } + } + void AddPipeToWatch(int pipe) { struct epoll_event evt{}; evt.events = EPOLLERR; // This event will return when the read end of a pipe is closed @@ -55,6 +102,12 @@ namespace EPollWatcher { } else { ++NumPipesWatched; + + if ((NumPipesWatched + NumFilesOpened) >= (MaxFDs.rlim_cur - MAX_FD_DISTANCE)) { + // We are close to the maximum FD distance + // Try to raise the limit + RaiseFDLimit(); + } } } @@ -118,6 +171,7 @@ namespace EPollWatcher { } void SetupEPoll() { + GetMaxFDs(); epoll_fd = epoll_create1(EPOLL_CLOEXEC); EPollThread = std::thread{EPollWatcher::EPollWatch}; }