composite_env_wrapper.h 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399
  1. // Copyright (c) 2019-present, Facebook, Inc. All rights reserved.
  2. // This source code is licensed under both the GPLv2 (found in the
  3. // COPYING file in the root directory) and Apache 2.0 License
  4. // (found in the LICENSE.Apache file in the root directory).
  5. #pragma once
  6. #include "rocksdb/env.h"
  7. #include "rocksdb/file_system.h"
  8. #include "rocksdb/system_clock.h"
  9. #ifdef _WIN32
  10. // Windows API macro interference
  11. #undef DeleteFile
  12. #undef GetCurrentTime
  13. #undef LoadLibrary
  14. #endif
  15. namespace ROCKSDB_NAMESPACE {
  16. // This class supports abstracting different types of an `Env`'s functionality
  17. // into separate interfaces. It is constructed with a `FileSystem` and a
  18. // `SystemClock` and delegates:
  19. // * File system operations to member `file_system_`.
  20. // * Time related misc operations to member `clock_`.
  21. // A subclass needs to inherit `CompositeEnv` and provide implementations for
  22. // the thread management related APIs.
  23. class CompositeEnv : public Env {
  24. public:
  25. // Initialize a CompositeEnvWrapper that delegates all thread/time related
  26. // calls to env, and all file operations to fs
  27. explicit CompositeEnv(const std::shared_ptr<FileSystem>& fs,
  28. const std::shared_ptr<SystemClock>& clock)
  29. : Env(fs, clock) {}
  30. Status RegisterDbPaths(const std::vector<std::string>& paths) override {
  31. return file_system_->RegisterDbPaths(paths);
  32. }
  33. Status UnregisterDbPaths(const std::vector<std::string>& paths) override {
  34. return file_system_->UnregisterDbPaths(paths);
  35. }
  36. // The following text is boilerplate that forwards all methods to target()
  37. Status NewSequentialFile(const std::string& f,
  38. std::unique_ptr<SequentialFile>* r,
  39. const EnvOptions& options) override;
  40. Status NewRandomAccessFile(const std::string& f,
  41. std::unique_ptr<RandomAccessFile>* r,
  42. const EnvOptions& options) override;
  43. Status NewWritableFile(const std::string& f, std::unique_ptr<WritableFile>* r,
  44. const EnvOptions& options) override;
  45. Status ReopenWritableFile(const std::string& fname,
  46. std::unique_ptr<WritableFile>* result,
  47. const EnvOptions& options) override;
  48. Status ReuseWritableFile(const std::string& fname,
  49. const std::string& old_fname,
  50. std::unique_ptr<WritableFile>* r,
  51. const EnvOptions& options) override;
  52. Status NewRandomRWFile(const std::string& fname,
  53. std::unique_ptr<RandomRWFile>* result,
  54. const EnvOptions& options) override;
  55. Status NewMemoryMappedFileBuffer(
  56. const std::string& fname,
  57. std::unique_ptr<MemoryMappedFileBuffer>* result) override {
  58. return file_system_->NewMemoryMappedFileBuffer(fname, result);
  59. }
  60. Status NewDirectory(const std::string& name,
  61. std::unique_ptr<Directory>* result) override;
  62. Status FileExists(const std::string& f) override {
  63. IOOptions io_opts;
  64. IODebugContext dbg;
  65. return file_system_->FileExists(f, io_opts, &dbg);
  66. }
  67. Status GetChildren(const std::string& dir,
  68. std::vector<std::string>* r) override {
  69. IOOptions io_opts;
  70. IODebugContext dbg;
  71. return file_system_->GetChildren(dir, io_opts, r, &dbg);
  72. }
  73. Status GetChildrenFileAttributes(
  74. const std::string& dir, std::vector<FileAttributes>* result) override {
  75. IOOptions io_opts;
  76. IODebugContext dbg;
  77. return file_system_->GetChildrenFileAttributes(dir, io_opts, result, &dbg);
  78. }
  79. Status DeleteFile(const std::string& f) override {
  80. IOOptions io_opts;
  81. IODebugContext dbg;
  82. return file_system_->DeleteFile(f, io_opts, &dbg);
  83. }
  84. Status Truncate(const std::string& fname, size_t size) override {
  85. IOOptions io_opts;
  86. IODebugContext dbg;
  87. return file_system_->Truncate(fname, size, io_opts, &dbg);
  88. }
  89. Status CreateDir(const std::string& d) override {
  90. IOOptions io_opts;
  91. IODebugContext dbg;
  92. return file_system_->CreateDir(d, io_opts, &dbg);
  93. }
  94. Status CreateDirIfMissing(const std::string& d) override {
  95. IOOptions io_opts;
  96. IODebugContext dbg;
  97. return file_system_->CreateDirIfMissing(d, io_opts, &dbg);
  98. }
  99. Status DeleteDir(const std::string& d) override {
  100. IOOptions io_opts;
  101. IODebugContext dbg;
  102. return file_system_->DeleteDir(d, io_opts, &dbg);
  103. }
  104. Status GetFileSize(const std::string& f, uint64_t* s) override {
  105. IOOptions io_opts;
  106. IODebugContext dbg;
  107. return file_system_->GetFileSize(f, io_opts, s, &dbg);
  108. }
  109. Status GetFileModificationTime(const std::string& fname,
  110. uint64_t* file_mtime) override {
  111. IOOptions io_opts;
  112. IODebugContext dbg;
  113. return file_system_->GetFileModificationTime(fname, io_opts, file_mtime,
  114. &dbg);
  115. }
  116. Status RenameFile(const std::string& s, const std::string& t) override {
  117. IOOptions io_opts;
  118. IODebugContext dbg;
  119. return file_system_->RenameFile(s, t, io_opts, &dbg);
  120. }
  121. Status LinkFile(const std::string& s, const std::string& t) override {
  122. IOOptions io_opts;
  123. IODebugContext dbg;
  124. return file_system_->LinkFile(s, t, io_opts, &dbg);
  125. }
  126. Status NumFileLinks(const std::string& fname, uint64_t* count) override {
  127. IOOptions io_opts;
  128. IODebugContext dbg;
  129. return file_system_->NumFileLinks(fname, io_opts, count, &dbg);
  130. }
  131. Status AreFilesSame(const std::string& first, const std::string& second,
  132. bool* res) override {
  133. IOOptions io_opts;
  134. IODebugContext dbg;
  135. return file_system_->AreFilesSame(first, second, io_opts, res, &dbg);
  136. }
  137. Status LockFile(const std::string& f, FileLock** l) override {
  138. IOOptions io_opts;
  139. IODebugContext dbg;
  140. return file_system_->LockFile(f, io_opts, l, &dbg);
  141. }
  142. Status UnlockFile(FileLock* l) override {
  143. IOOptions io_opts;
  144. IODebugContext dbg;
  145. return file_system_->UnlockFile(l, io_opts, &dbg);
  146. }
  147. Status GetAbsolutePath(const std::string& db_path,
  148. std::string* output_path) override {
  149. IOOptions io_opts;
  150. IODebugContext dbg;
  151. return file_system_->GetAbsolutePath(db_path, io_opts, output_path, &dbg);
  152. }
  153. Status NewLogger(const std::string& fname,
  154. std::shared_ptr<Logger>* result) override {
  155. IOOptions io_opts;
  156. IODebugContext dbg;
  157. return file_system_->NewLogger(fname, io_opts, result, &dbg);
  158. }
  159. Status IsDirectory(const std::string& path, bool* is_dir) override {
  160. IOOptions io_opts;
  161. IODebugContext dbg;
  162. return file_system_->IsDirectory(path, io_opts, is_dir, &dbg);
  163. }
  164. Status GetTestDirectory(std::string* path) override {
  165. IOOptions io_opts;
  166. IODebugContext dbg;
  167. return file_system_->GetTestDirectory(io_opts, path, &dbg);
  168. }
  169. EnvOptions OptimizeForLogRead(const EnvOptions& env_options) const override {
  170. return file_system_->OptimizeForLogRead(FileOptions(env_options));
  171. }
  172. EnvOptions OptimizeForManifestRead(
  173. const EnvOptions& env_options) const override {
  174. return file_system_->OptimizeForManifestRead(FileOptions(env_options));
  175. }
  176. EnvOptions OptimizeForLogWrite(const EnvOptions& env_options,
  177. const DBOptions& db_options) const override {
  178. return file_system_->OptimizeForLogWrite(FileOptions(env_options),
  179. db_options);
  180. }
  181. EnvOptions OptimizeForManifestWrite(
  182. const EnvOptions& env_options) const override {
  183. return file_system_->OptimizeForManifestWrite(FileOptions(env_options));
  184. }
  185. EnvOptions OptimizeForCompactionTableWrite(
  186. const EnvOptions& env_options,
  187. const ImmutableDBOptions& immutable_ops) const override {
  188. return file_system_->OptimizeForCompactionTableWrite(
  189. FileOptions(env_options), immutable_ops);
  190. }
  191. EnvOptions OptimizeForCompactionTableRead(
  192. const EnvOptions& env_options,
  193. const ImmutableDBOptions& db_options) const override {
  194. return file_system_->OptimizeForCompactionTableRead(
  195. FileOptions(env_options), db_options);
  196. }
  197. EnvOptions OptimizeForBlobFileRead(
  198. const EnvOptions& env_options,
  199. const ImmutableDBOptions& db_options) const override {
  200. return file_system_->OptimizeForBlobFileRead(FileOptions(env_options),
  201. db_options);
  202. }
  203. // This seems to clash with a macro on Windows, so #undef it here
  204. #ifdef GetFreeSpace
  205. #undef GetFreeSpace
  206. #endif
  207. Status GetFreeSpace(const std::string& path, uint64_t* diskfree) override {
  208. IOOptions io_opts;
  209. IODebugContext dbg;
  210. return file_system_->GetFreeSpace(path, io_opts, diskfree, &dbg);
  211. }
  212. uint64_t NowMicros() override { return system_clock_->NowMicros(); }
  213. uint64_t NowNanos() override { return system_clock_->NowNanos(); }
  214. uint64_t NowCPUNanos() override { return system_clock_->CPUNanos(); }
  215. void SleepForMicroseconds(int micros) override {
  216. system_clock_->SleepForMicroseconds(micros);
  217. }
  218. Status GetCurrentTime(int64_t* unix_time) override {
  219. return system_clock_->GetCurrentTime(unix_time);
  220. }
  221. std::string TimeToString(uint64_t time) override {
  222. return system_clock_->TimeToString(time);
  223. }
  224. };
  225. // A `CompositeEnvWrapper` is constructed with a target `Env` object, an
  226. // optional `FileSystem` object and an optional `SystemClock` object.
  227. // `Env::GetFileSystem()` is a fallback file system if no such object is
  228. // explicitly provided. Similarly, `Env::GetSystemClock()` is a fallback system
  229. // clock.
  230. // Besides delegating corresponding functionality to `file_system_` and `clock_`
  231. // which is inherited from `CompositeEnv`, it also implements the thread
  232. // management APIs by delegating them to the target `Env` object.
  233. //
  234. // Effectively, this class helps to support using customized file system
  235. // implementations such as a remote file system instead of the default file
  236. // system provided by the operating system.
  237. //
  238. // Also see public API `NewCompositeEnv` in rocksdb/include/env.h
  239. class CompositeEnvWrapper : public CompositeEnv {
  240. public:
  241. // Initialize a CompositeEnvWrapper that delegates all thread/time related
  242. // calls to env, and all file operations to fs
  243. explicit CompositeEnvWrapper(Env* env)
  244. : CompositeEnvWrapper(env, env->GetFileSystem(), env->GetSystemClock()) {}
  245. explicit CompositeEnvWrapper(Env* env, const std::shared_ptr<FileSystem>& fs)
  246. : CompositeEnvWrapper(env, fs, env->GetSystemClock()) {}
  247. explicit CompositeEnvWrapper(Env* env, const std::shared_ptr<SystemClock>& sc)
  248. : CompositeEnvWrapper(env, env->GetFileSystem(), sc) {}
  249. explicit CompositeEnvWrapper(Env* env, const std::shared_ptr<FileSystem>& fs,
  250. const std::shared_ptr<SystemClock>& sc);
  251. explicit CompositeEnvWrapper(const std::shared_ptr<Env>& env,
  252. const std::shared_ptr<FileSystem>& fs)
  253. : CompositeEnvWrapper(env, fs, env->GetSystemClock()) {}
  254. explicit CompositeEnvWrapper(const std::shared_ptr<Env>& env,
  255. const std::shared_ptr<SystemClock>& sc)
  256. : CompositeEnvWrapper(env, env->GetFileSystem(), sc) {}
  257. explicit CompositeEnvWrapper(const std::shared_ptr<Env>& env,
  258. const std::shared_ptr<FileSystem>& fs,
  259. const std::shared_ptr<SystemClock>& sc);
  260. static const char* kClassName() { return "CompositeEnv"; }
  261. const char* Name() const override { return kClassName(); }
  262. bool IsInstanceOf(const std::string& name) const override {
  263. if (name == kClassName()) {
  264. return true;
  265. } else {
  266. return CompositeEnv::IsInstanceOf(name);
  267. }
  268. }
  269. const Customizable* Inner() const override { return target_.env; }
  270. Status PrepareOptions(const ConfigOptions& options) override;
  271. std::string SerializeOptions(const ConfigOptions& config_options,
  272. const std::string& header) const override;
  273. // Return the target to which this Env forwards all calls
  274. Env* env_target() const { return target_.env; }
  275. #if !defined(OS_WIN) && !defined(ROCKSDB_NO_DYNAMIC_EXTENSION)
  276. Status LoadLibrary(const std::string& lib_name,
  277. const std::string& search_path,
  278. std::shared_ptr<DynamicLibrary>* result) override {
  279. return target_.env->LoadLibrary(lib_name, search_path, result);
  280. }
  281. #endif
  282. void Schedule(void (*f)(void* arg), void* a, Priority pri,
  283. void* tag = nullptr, void (*u)(void* arg) = nullptr) override {
  284. return target_.env->Schedule(f, a, pri, tag, u);
  285. }
  286. int UnSchedule(void* tag, Priority pri) override {
  287. return target_.env->UnSchedule(tag, pri);
  288. }
  289. void StartThread(void (*f)(void*), void* a) override {
  290. return target_.env->StartThread(f, a);
  291. }
  292. void WaitForJoin() override { return target_.env->WaitForJoin(); }
  293. unsigned int GetThreadPoolQueueLen(Priority pri = LOW) const override {
  294. return target_.env->GetThreadPoolQueueLen(pri);
  295. }
  296. int ReserveThreads(int threads_to_be_reserved, Priority pri) override {
  297. return target_.env->ReserveThreads(threads_to_be_reserved, pri);
  298. }
  299. int ReleaseThreads(int threads_to_be_released, Priority pri) override {
  300. return target_.env->ReleaseThreads(threads_to_be_released, pri);
  301. }
  302. Status GetHostName(char* name, uint64_t len) override {
  303. return target_.env->GetHostName(name, len);
  304. }
  305. void SetBackgroundThreads(int num, Priority pri) override {
  306. return target_.env->SetBackgroundThreads(num, pri);
  307. }
  308. int GetBackgroundThreads(Priority pri) override {
  309. return target_.env->GetBackgroundThreads(pri);
  310. }
  311. Status SetAllowNonOwnerAccess(bool allow_non_owner_access) override {
  312. return target_.env->SetAllowNonOwnerAccess(allow_non_owner_access);
  313. }
  314. void IncBackgroundThreadsIfNeeded(int num, Priority pri) override {
  315. return target_.env->IncBackgroundThreadsIfNeeded(num, pri);
  316. }
  317. void LowerThreadPoolIOPriority(Priority pool) override {
  318. target_.env->LowerThreadPoolIOPriority(pool);
  319. }
  320. void LowerThreadPoolCPUPriority(Priority pool) override {
  321. target_.env->LowerThreadPoolCPUPriority(pool);
  322. }
  323. Status LowerThreadPoolCPUPriority(Priority pool, CpuPriority pri) override {
  324. return target_.env->LowerThreadPoolCPUPriority(pool, pri);
  325. }
  326. Status GetThreadList(std::vector<ThreadStatus>* thread_list) override {
  327. return target_.env->GetThreadList(thread_list);
  328. }
  329. ThreadStatusUpdater* GetThreadStatusUpdater() const override {
  330. return target_.env->GetThreadStatusUpdater();
  331. }
  332. uint64_t GetThreadID() const override { return target_.env->GetThreadID(); }
  333. std::string GenerateUniqueId() override {
  334. return target_.env->GenerateUniqueId();
  335. }
  336. private:
  337. EnvWrapper::Target target_;
  338. };
  339. } // namespace ROCKSDB_NAMESPACE