From d6c861719f5bff4b6525f19a64cf8ca9239bdcc7 Mon Sep 17 00:00:00 2001 From: Carl Pearson Date: Mon, 23 Sep 2019 16:54:33 -0500 Subject: [PATCH] Squashed commit of the following: commit b96ddedf4ffbba57faaaf8bf18781a7abfb9d4c1 Author: Carl Pearson Date: Mon Sep 23 16:53:54 2019 -0500 add newline to result.hpp commit c7e9f6ff4775bf86f9af216cbe311f65bf985f63 Author: Carl Pearson Date: Mon Sep 23 16:53:09 2019 -0500 add EPERM to fs operations commit bac918fd022006cad0da899c06ac31e9db59a2fb Author: Carl Pearson Date: Mon Sep 23 16:49:25 2019 -0500 add filesystem cache interface --- README.md | 17 ++ include/perfect/cpu_set.hpp | 361 ++++++++++++++++++++++++++++++++ include/perfect/detail/fs.hpp | 36 ++++ include/perfect/drop_caches.hpp | 43 ++++ include/perfect/result.hpp | 9 + tools/CMakeLists.txt | 2 + tools/sync_drop_caches.cpp | 14 ++ 7 files changed, 482 insertions(+) create mode 100644 include/perfect/cpu_set.hpp create mode 100644 include/perfect/detail/fs.hpp create mode 100644 include/perfect/drop_caches.hpp create mode 100644 tools/sync_drop_caches.cpp diff --git a/README.md b/README.md index d2c19db..474e61e 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ CPU/GPU performance control library for benchmarking - [x] Disable GPU turbo (nvidia) - [x] Flush addresses from cache (amd64, POWER) - [x] CUDA not required (GPU functions will not be compiled) +- [x] Flush file system caches (linux) ## Installing @@ -84,6 +85,22 @@ See [examples/gpu_monitor.cu](examples/gpu_monitor.cu) * `void Monitor::pause()`: pause the monitor thread * `void Monitor::resume()`: resume the monitor thread +### Flush file system caches + +`perfect` can drop various filesystem caches + +See [tools/sync_drop_caches.cpp](tools/sync_drop_caches.cpp) + +```c++ +#include "perfect/drop_caches.hpp" +``` + +* `Result sync()`: flush filesystem caches to disk +* `Result drop_caches(DropCaches_t mode)`: remove file system caches + * `mode = PAGECACHE`: drop page caches + * `mode = ENTRIES`: drop dentries and inodes + * `mode = PAGECACHE | ENTRIES`: both + ### CPU Turbo `perfect` can enable and disable CPU boost through the Intel p-state mechanism or the ACPI cpufreq mechanism. diff --git a/include/perfect/cpu_set.hpp b/include/perfect/cpu_set.hpp new file mode 100644 index 0000000..0769d13 --- /dev/null +++ b/include/perfect/cpu_set.hpp @@ -0,0 +1,361 @@ +#pragma once + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "init.hpp" +#include "result.hpp" + +#define SUCCESS_OR_RETURN(stmt) \ +{\ + Result _ret; \ + _ret = (stmt); \ +if (_ret != Result::SUCCESS) {\ + return _ret;\ +}\ +} + +std::set operator-(const std::set &lhs, const std::set &rhs) { + std::set result; + for (auto e : lhs) { + if (0 == rhs.count(e)) { + result.insert(e); + } + } + return result; +} + +std::string remove_space(const std::string &s) { + std::string result; + + for (auto c : s) { + if (!isspace(c)) { + result += c; + } + } + return result; +} + +// like "8" or "8-10" +std::set parse_token(const std::string &token) { + // std::cerr << "parse_token: parsing '" << s << "'\n"; + std::set result; + + std::string s = token; + // ignore empty string + if (s.empty()) { + return result; + } + + // remove newline + s = remove_space(s); + + size_t pos = 0; + + int first = std::stoi(s, &pos); + // std::cerr << "parse_token: found '" << first << "'\n"; + + // single int + if (pos == s.length()) { + result.insert(first); + return result; + } + + // next char should be a "-" + assert(s[pos] == '-'); + + std::string rest = s.substr(pos + 1); + int second = std::stoi(rest, &pos); + // std::cerr << "parse_token: found '" << second << "'\n"; + + // insert first-second + // std::cerr << "parse_token: range " << first << " to " << second << "\n"; + for (int i = first; i <= second; ++i) { + result.insert(i); + } + return result; +} + +std::set parse_cpuset(const std::string &s) { + // std::cerr << "parse_cpuset: parsing '" << s << "'\n"; + std::set result; + + std::string token; + std::stringstream ss(s); + while (std::getline(ss, token, ',')) { + + if ("\n" != token) { + auto newCpus = parse_token(token); + for (auto cpu : newCpus) { + result.insert(cpu); + } + } + } + + return result; +} + +// http://man7.org/linux/man-pages/man7/cpuset.7.html +namespace perfect { +class CpuSet { +public: + std::string path_; + std::set cpus_; + std::set mems_; + CpuSet *parent_; + + // make sure cpuset is initialized + static Result init() { + + // check for "nodev cpuset" in /proc/filesystems + + // mkdir /dev/cpuset + if (mkdir("/dev/cpuset", S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH)) { + switch (errno) { + case EEXIST: { + // okay + break; + } + case EACCES: + // std::cerr << "access error in mkdir: " << strerror(errno) << "\n"; + return Result::NO_PERMISSION; + case ENOENT: + case EROFS: + default: + std::cerr << "unhandled error in mkdir: " << strerror(errno) << "\n"; + return Result::UNKNOWN; + } + + // mount -t cpuset none /dev/cpuset + if (mount("none", "/dev/cpuset", "cpuset", 0, nullptr)) { + switch (errno) { + case EEXIST: { + // okay + break; + } + case EBUSY: { + // FIXME: something is mounted here, assume it is what we want + // std::cerr << "EBUSY in mount: " << strerror(errno) << "\n"; + return Result::SUCCESS; + } + case EPERM: { + // std::cerr << "EPERM in mount: " << strerror(errno) << "\n"; + return Result::NO_PERMISSION; + } + case ENOENT: + case EROFS: + default: + std::cerr << "unhandled error in mount: " << strerror(errno) << "\n"; + return Result::UNKNOWN; + } + } + } + return Result::SUCCESS; + } + + std::string get_raw_cpus() { + std::ifstream is(path_ + "/cpuset.cpus"); + std::stringstream ss; + ss << is.rdbuf(); + return remove_space(ss.str()); + } + + std::string get_raw_mems() { + std::ifstream is(path_ + "/cpuset.mems"); + std::stringstream ss; + ss << is.rdbuf(); + return remove_space(ss.str()); + } + + std::set get_cpus() { return parse_cpuset(get_raw_cpus()); } + + std::set get_mems() { return parse_cpuset(get_raw_mems()); } + + // migrate the caller task from this cpu set to another + Result migrate_self_to(CpuSet &other) { + // enable memory migration in other + other.enable_memory_migration(); + + // get my pid + pid_t self = this_task(); + + // read this tasks and write each line to other.tasks + std::ifstream is(path_ + "/tasks"); + std::string line; + while (std::getline(is, line)) { + line = remove_space(line); + if (std::to_string(self) == line) { + // std::cerr << "migrating self task " << line << " to " << other.path + // << "\n"; + other.write_task(line); + return Result::SUCCESS; + } + } + return Result::NO_TASK; + } + + // migrate tasks in this cpu set to another + Result migrate_tasks_to(CpuSet &other) { + // enable memory migration in other + SUCCESS_OR_RETURN(other.enable_memory_migration()); + + // read this tasks and write each line to other.tasks + std::ifstream is(path_ + "/tasks"); + std::string line; + while (std::getline(is, line)) { + // std::cerr << "migrating task " << line << " to " << other.path << "\n"; + other.write_task(line); + } + + return Result::SUCCESS; + } + + Result enable_memory_migration() { + std::ofstream ofs(path_ + "/" + "cpuset.memory_migrate"); + ofs << "1"; + ofs.close(); + if (ofs.fail()) { + switch (errno) { + case EACCES: + return Result::NO_PERMISSION; + case ENOENT: + return Result::NOT_SUPPORTED; + default: + return Result::UNKNOWN; + } + } + return Result::SUCCESS; + } + + void write_task(const std::string &task) { + // write `task` to path/tasks + std::ofstream os(path_ + "/tasks"); + os << task << "\n"; + } + + // object representing the root CPU set + static Result get_root(CpuSet &root) { + SUCCESS_OR_RETURN(CpuSet::init()); + root.path_ = "/dev/cpuset"; + root.parent_ = nullptr; + return Result::SUCCESS; + } + + // the ID of this task + static pid_t this_task() { return getpid(); } + + Result make_child(CpuSet &child, const std::string &name) { + + if (mkdir((path_ + "/" + name).c_str(), + S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH)) { + switch (errno) { + case EEXIST: { + // okay + break; + } + case EACCES: + return Result::NO_PERMISSION; + case ENOENT: + case EROFS: + default: + return Result::UNKNOWN; + } + } + + child.path_ = path_ + "/" + name; + child.parent_ = this; + return Result::SUCCESS; + } + + Result enable_cpu(const int cpu) { + std::set cpus = get_cpus(); + cpus.insert(cpu); + return write_cpus(cpus); + } + + Result enable_cpus(const std::set &cpus) { + std::set finalCpus = get_cpus(); + for (auto cpu : cpus) { + finalCpus.insert(cpu); + } + return write_cpus(finalCpus); + } + + // FIXME: check error + Result write_cpus(std::set cpus) { + std::ofstream os(path_ + "/cpuset.cpus"); + bool comma = false; + for (auto cpu : cpus) { + if (comma) + os << ","; + os << cpu << "-" << cpu; + comma = true; + } + return Result::SUCCESS; + } + + // FIXME: check write + Result write_mems(std::set mems) { + std::ofstream os(path_ + "/cpuset.mems"); + bool comma = false; + for (auto mem : mems) { + if (comma) + os << ","; + os << mem << "-" << mem; + comma = true; + } + return Result::SUCCESS; + } + + Result enable_mem(const int mem) { + std::set mems = get_mems(); + mems.insert(mem); + return write_mems(mems); + } + + Result enable_mems(const std::set &mems) { + std::set finalMems = get_mems(); + for (auto mem : mems) { + finalMems.insert(mem); + } + return write_mems(finalMems); + } + + Result destroy() { + // remove all child cpu sets + + // move all attached processes back to parent + assert(parent_); + migrate_tasks_to(*parent_); + + // remove with rmdir + if (rmdir(path_.c_str())) { + switch (errno) { + default: + std::cerr << "unhandled error in rmdir: " << strerror(errno) << "\n"; + return Result::UNKNOWN; + } + } + + path_ = ""; + return Result::SUCCESS; + } + + +}; + + std::ostream &operator<<(std::ostream &s, const CpuSet &c) { + s << c.path_; + return s; + } + +} // namespace perfect diff --git a/include/perfect/detail/fs.hpp b/include/perfect/detail/fs.hpp new file mode 100644 index 0000000..0495e87 --- /dev/null +++ b/include/perfect/detail/fs.hpp @@ -0,0 +1,36 @@ +#pragma once + +#include +#include + +#include "perfect/result.hpp" + +namespace perfect { +namespace detail { +Result write_str(const std::string &path, const std::string &val) { + std::ofstream ofs(path); + if (ofs.fail()) { + return Result::NOT_SUPPORTED; + } + + ofs << val; + ofs.close(); + if (ofs.fail()) { + switch (errno) { + case EACCES: + std::cerr << "EACCES when writing to " << path << "\n"; + return Result::NO_PERMISSION; + case EPERM: + std::cerr << "EPERM when writing to " << path << "\n"; + return Result::NO_PERMISSION; + case ENOENT: + return Result::NOT_SUPPORTED; + default: + return Result::UNKNOWN; + } + } + return Result::SUCCESS; +} + +} // namespace detail +} // namespace perfect \ No newline at end of file diff --git a/include/perfect/drop_caches.hpp b/include/perfect/drop_caches.hpp new file mode 100644 index 0000000..84de9cd --- /dev/null +++ b/include/perfect/drop_caches.hpp @@ -0,0 +1,43 @@ +#pragma once + +#include + +#include +#include + +#include "result.hpp" +#include "init.hpp" +#include "detail/fs.hpp" + +namespace perfect { + +enum DropCaches_t { + PAGECACHE = 0x1, + ENTRIES = 0x2 +}; + + +// commit filesystem caches to disk +Result sync() { + // http://man7.org/linux/man-pages/man2/sync.2.html + ::sync(); // always successful + return Result::SUCCESS; +} + +Result drop_caches(const DropCaches_t mode) { + using detail::write_str; + const std::string path = "/proc/sys/vm/drop_caches"; + if (mode & PAGECACHE & ENTRIES) { + PERFECT_SUCCESS_OR_RETURN(write_str(path, "3")); + } else if (mode & PAGECACHE) { + PERFECT_SUCCESS_OR_RETURN(write_str(path, "1")); + } else if (mode & ENTRIES) { + PERFECT_SUCCESS_OR_RETURN(write_str(path, "2")); + } else { + std::cerr << "unexpected mode: " << mode << "\n"; + return Result::UNKNOWN; + } + return Result::SUCCESS; +} + +} \ No newline at end of file diff --git a/include/perfect/result.hpp b/include/perfect/result.hpp index 2c2036b..1c274ba 100644 --- a/include/perfect/result.hpp +++ b/include/perfect/result.hpp @@ -78,3 +78,12 @@ inline void check(Result result, const char *file, const int line) { } // namespace perfect #define PERFECT(stmt) check(stmt, __FILE__, __LINE__); + +#define PERFECT_SUCCESS_OR_RETURN(stmt) \ +{\ + Result _ret; \ + _ret = (stmt); \ +if (_ret != Result::SUCCESS) {\ + return _ret;\ +}\ +} diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 2e145ea..86b5b3f 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -37,3 +37,5 @@ set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} \ add_executable(enable-turbo enable_turbo.cpp) target_link_libraries(enable-turbo perfect) +add_executable(sync-drop-caches sync_drop_caches.cpp) +target_link_libraries(sync-drop-caches perfect) \ No newline at end of file diff --git a/tools/sync_drop_caches.cpp b/tools/sync_drop_caches.cpp new file mode 100644 index 0000000..a1fac66 --- /dev/null +++ b/tools/sync_drop_caches.cpp @@ -0,0 +1,14 @@ +#include + +#include "perfect/drop_caches.hpp" + +using namespace perfect; + +int main(void) { + + using namespace perfect; + + PERFECT(init()); + PERFECT(perfect::sync()); + PERFECT(drop_caches(DropCaches_t(PAGECACHE | ENTRIES))); +} \ No newline at end of file