db_options.cc 54 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144
  1. // Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
  2. // This source code is licensed under both the GPLv2 (found in the
  3. // COPYING file in the root directory) and Apache 2.0 License
  4. // (found in the LICENSE.Apache file in the root directory).
  5. #include "options/db_options.h"
  6. #include <cinttypes>
  7. #include "logging/logging.h"
  8. #include "options/configurable_helper.h"
  9. #include "options/options_helper.h"
  10. #include "options/options_parser.h"
  11. #include "port/port.h"
  12. #include "rocksdb/advanced_cache.h"
  13. #include "rocksdb/configurable.h"
  14. #include "rocksdb/env.h"
  15. #include "rocksdb/file_system.h"
  16. #include "rocksdb/listener.h"
  17. #include "rocksdb/rate_limiter.h"
  18. #include "rocksdb/sst_file_manager.h"
  19. #include "rocksdb/statistics.h"
  20. #include "rocksdb/system_clock.h"
  21. #include "rocksdb/utilities/options_type.h"
  22. #include "rocksdb/wal_filter.h"
  23. #include "util/string_util.h"
  24. namespace ROCKSDB_NAMESPACE {
  25. static std::unordered_map<std::string, WALRecoveryMode>
  26. wal_recovery_mode_string_map = {
  27. {"kTolerateCorruptedTailRecords",
  28. WALRecoveryMode::kTolerateCorruptedTailRecords},
  29. {"kAbsoluteConsistency", WALRecoveryMode::kAbsoluteConsistency},
  30. {"kPointInTimeRecovery", WALRecoveryMode::kPointInTimeRecovery},
  31. {"kSkipAnyCorruptedRecords",
  32. WALRecoveryMode::kSkipAnyCorruptedRecords}};
  33. static std::unordered_map<std::string, CacheTier> cache_tier_string_map = {
  34. {"kVolatileTier", CacheTier::kVolatileTier},
  35. {"kVolatileCompressedTier", CacheTier::kVolatileCompressedTier},
  36. {"kNonVolatileBlockTier", CacheTier::kNonVolatileBlockTier}};
  37. static std::unordered_map<std::string, InfoLogLevel> info_log_level_string_map =
  38. {{"DEBUG_LEVEL", InfoLogLevel::DEBUG_LEVEL},
  39. {"INFO_LEVEL", InfoLogLevel::INFO_LEVEL},
  40. {"WARN_LEVEL", InfoLogLevel::WARN_LEVEL},
  41. {"ERROR_LEVEL", InfoLogLevel::ERROR_LEVEL},
  42. {"FATAL_LEVEL", InfoLogLevel::FATAL_LEVEL},
  43. {"HEADER_LEVEL", InfoLogLevel::HEADER_LEVEL}};
  44. static std::unordered_map<std::string, OptionTypeInfo>
  45. db_mutable_options_type_info = {
  46. {"allow_os_buffer",
  47. {0, OptionType::kBoolean, OptionVerificationType::kDeprecated,
  48. OptionTypeFlags::kMutable}},
  49. {"base_background_compactions",
  50. {0, OptionType::kInt, OptionVerificationType::kDeprecated,
  51. OptionTypeFlags::kMutable}},
  52. {"max_background_jobs",
  53. {offsetof(struct MutableDBOptions, max_background_jobs),
  54. OptionType::kInt, OptionVerificationType::kNormal,
  55. OptionTypeFlags::kMutable}},
  56. {"max_background_compactions",
  57. {offsetof(struct MutableDBOptions, max_background_compactions),
  58. OptionType::kInt, OptionVerificationType::kNormal,
  59. OptionTypeFlags::kMutable}},
  60. {"max_subcompactions",
  61. {offsetof(struct MutableDBOptions, max_subcompactions),
  62. OptionType::kUInt32T, OptionVerificationType::kNormal,
  63. OptionTypeFlags::kMutable}},
  64. {"avoid_flush_during_shutdown",
  65. {offsetof(struct MutableDBOptions, avoid_flush_during_shutdown),
  66. OptionType::kBoolean, OptionVerificationType::kNormal,
  67. OptionTypeFlags::kMutable}},
  68. {"writable_file_max_buffer_size",
  69. {offsetof(struct MutableDBOptions, writable_file_max_buffer_size),
  70. OptionType::kSizeT, OptionVerificationType::kNormal,
  71. OptionTypeFlags::kMutable}},
  72. {"delayed_write_rate",
  73. {offsetof(struct MutableDBOptions, delayed_write_rate),
  74. OptionType::kUInt64T, OptionVerificationType::kNormal,
  75. OptionTypeFlags::kMutable}},
  76. {"max_total_wal_size",
  77. {offsetof(struct MutableDBOptions, max_total_wal_size),
  78. OptionType::kUInt64T, OptionVerificationType::kNormal,
  79. OptionTypeFlags::kMutable}},
  80. {"delete_obsolete_files_period_micros",
  81. {offsetof(struct MutableDBOptions,
  82. delete_obsolete_files_period_micros),
  83. OptionType::kUInt64T, OptionVerificationType::kNormal,
  84. OptionTypeFlags::kMutable}},
  85. {"stats_dump_period_sec",
  86. {offsetof(struct MutableDBOptions, stats_dump_period_sec),
  87. OptionType::kUInt, OptionVerificationType::kNormal,
  88. OptionTypeFlags::kMutable}},
  89. {"stats_persist_period_sec",
  90. {offsetof(struct MutableDBOptions, stats_persist_period_sec),
  91. OptionType::kUInt, OptionVerificationType::kNormal,
  92. OptionTypeFlags::kMutable}},
  93. {"stats_history_buffer_size",
  94. {offsetof(struct MutableDBOptions, stats_history_buffer_size),
  95. OptionType::kSizeT, OptionVerificationType::kNormal,
  96. OptionTypeFlags::kMutable}},
  97. {"max_open_files",
  98. {offsetof(struct MutableDBOptions, max_open_files), OptionType::kInt,
  99. OptionVerificationType::kNormal, OptionTypeFlags::kMutable}},
  100. {"bytes_per_sync",
  101. {offsetof(struct MutableDBOptions, bytes_per_sync),
  102. OptionType::kUInt64T, OptionVerificationType::kNormal,
  103. OptionTypeFlags::kMutable}},
  104. {"wal_bytes_per_sync",
  105. {offsetof(struct MutableDBOptions, wal_bytes_per_sync),
  106. OptionType::kUInt64T, OptionVerificationType::kNormal,
  107. OptionTypeFlags::kMutable}},
  108. {"strict_bytes_per_sync",
  109. {offsetof(struct MutableDBOptions, strict_bytes_per_sync),
  110. OptionType::kBoolean, OptionVerificationType::kNormal,
  111. OptionTypeFlags::kMutable}},
  112. {"compaction_readahead_size",
  113. {offsetof(struct MutableDBOptions, compaction_readahead_size),
  114. OptionType::kSizeT, OptionVerificationType::kNormal,
  115. OptionTypeFlags::kMutable}},
  116. {"max_background_flushes",
  117. {offsetof(struct MutableDBOptions, max_background_flushes),
  118. OptionType::kInt, OptionVerificationType::kNormal,
  119. OptionTypeFlags::kMutable}},
  120. {"daily_offpeak_time_utc",
  121. {offsetof(struct MutableDBOptions, daily_offpeak_time_utc),
  122. OptionType::kString, OptionVerificationType::kNormal,
  123. OptionTypeFlags::kMutable}},
  124. };
  125. static std::unordered_map<std::string, OptionTypeInfo>
  126. db_immutable_options_type_info = {
  127. /*
  128. // not yet supported
  129. std::shared_ptr<Cache> row_cache;
  130. std::shared_ptr<DeleteScheduler> delete_scheduler;
  131. std::shared_ptr<Logger> info_log;
  132. std::shared_ptr<RateLimiter> rate_limiter;
  133. std::shared_ptr<Statistics> statistics;
  134. std::vector<DbPath> db_paths;
  135. FileTypeSet checksum_handoff_file_types;
  136. CompactionStyleSet calculate_sst_write_lifetime_hint_set;
  137. */
  138. {"advise_random_on_open",
  139. {offsetof(struct ImmutableDBOptions, advise_random_on_open),
  140. OptionType::kBoolean, OptionVerificationType::kNormal,
  141. OptionTypeFlags::kNone}},
  142. {"allow_mmap_reads",
  143. {offsetof(struct ImmutableDBOptions, allow_mmap_reads),
  144. OptionType::kBoolean, OptionVerificationType::kNormal,
  145. OptionTypeFlags::kNone}},
  146. {"allow_fallocate",
  147. {offsetof(struct ImmutableDBOptions, allow_fallocate),
  148. OptionType::kBoolean, OptionVerificationType::kNormal,
  149. OptionTypeFlags::kNone}},
  150. {"allow_mmap_writes",
  151. {offsetof(struct ImmutableDBOptions, allow_mmap_writes),
  152. OptionType::kBoolean, OptionVerificationType::kNormal,
  153. OptionTypeFlags::kNone}},
  154. {"use_direct_reads",
  155. {offsetof(struct ImmutableDBOptions, use_direct_reads),
  156. OptionType::kBoolean, OptionVerificationType::kNormal,
  157. OptionTypeFlags::kNone}},
  158. {"use_direct_writes",
  159. {0, OptionType::kBoolean, OptionVerificationType::kDeprecated,
  160. OptionTypeFlags::kNone}},
  161. {"use_direct_io_for_flush_and_compaction",
  162. {offsetof(struct ImmutableDBOptions,
  163. use_direct_io_for_flush_and_compaction),
  164. OptionType::kBoolean, OptionVerificationType::kNormal,
  165. OptionTypeFlags::kNone}},
  166. {"allow_2pc",
  167. {offsetof(struct ImmutableDBOptions, allow_2pc), OptionType::kBoolean,
  168. OptionVerificationType::kNormal, OptionTypeFlags::kNone}},
  169. {"wal_filter",
  170. OptionTypeInfo::AsCustomRawPtr<WalFilter>(
  171. offsetof(struct ImmutableDBOptions, wal_filter),
  172. OptionVerificationType::kByName,
  173. (OptionTypeFlags::kAllowNull | OptionTypeFlags::kCompareNever))},
  174. {"create_if_missing",
  175. {offsetof(struct ImmutableDBOptions, create_if_missing),
  176. OptionType::kBoolean, OptionVerificationType::kNormal,
  177. OptionTypeFlags::kNone}},
  178. {"create_missing_column_families",
  179. {offsetof(struct ImmutableDBOptions, create_missing_column_families),
  180. OptionType::kBoolean, OptionVerificationType::kNormal,
  181. OptionTypeFlags::kNone}},
  182. {"disableDataSync",
  183. {0, OptionType::kBoolean, OptionVerificationType::kDeprecated,
  184. OptionTypeFlags::kNone}},
  185. {"disable_data_sync", // for compatibility
  186. {0, OptionType::kBoolean, OptionVerificationType::kDeprecated,
  187. OptionTypeFlags::kNone}},
  188. {"enable_thread_tracking",
  189. {offsetof(struct ImmutableDBOptions, enable_thread_tracking),
  190. OptionType::kBoolean, OptionVerificationType::kNormal,
  191. OptionTypeFlags::kNone}},
  192. {"error_if_exists",
  193. {offsetof(struct ImmutableDBOptions, error_if_exists),
  194. OptionType::kBoolean, OptionVerificationType::kNormal,
  195. OptionTypeFlags::kNone}},
  196. {"experimental_allow_mempurge",
  197. {0, OptionType::kBoolean, OptionVerificationType::kDeprecated,
  198. OptionTypeFlags::kNone}},
  199. {"experimental_mempurge_policy",
  200. {0, OptionType::kString, OptionVerificationType::kDeprecated,
  201. OptionTypeFlags::kNone}},
  202. {"experimental_mempurge_threshold",
  203. {0, OptionType::kDouble, OptionVerificationType::kDeprecated,
  204. OptionTypeFlags::kNone}},
  205. {"is_fd_close_on_exec",
  206. {offsetof(struct ImmutableDBOptions, is_fd_close_on_exec),
  207. OptionType::kBoolean, OptionVerificationType::kNormal,
  208. OptionTypeFlags::kNone}},
  209. {"paranoid_checks",
  210. {offsetof(struct ImmutableDBOptions, paranoid_checks),
  211. OptionType::kBoolean, OptionVerificationType::kNormal,
  212. OptionTypeFlags::kNone}},
  213. {"flush_verify_memtable_count",
  214. {offsetof(struct ImmutableDBOptions, flush_verify_memtable_count),
  215. OptionType::kBoolean, OptionVerificationType::kNormal,
  216. OptionTypeFlags::kNone}},
  217. {"compaction_verify_record_count",
  218. {offsetof(struct ImmutableDBOptions, compaction_verify_record_count),
  219. OptionType::kBoolean, OptionVerificationType::kNormal,
  220. OptionTypeFlags::kNone}},
  221. {"track_and_verify_wals_in_manifest",
  222. {offsetof(struct ImmutableDBOptions,
  223. track_and_verify_wals_in_manifest),
  224. OptionType::kBoolean, OptionVerificationType::kNormal,
  225. OptionTypeFlags::kNone}},
  226. {"track_and_verify_wals",
  227. {offsetof(struct ImmutableDBOptions, track_and_verify_wals),
  228. OptionType::kBoolean, OptionVerificationType::kNormal,
  229. OptionTypeFlags::kNone}},
  230. {"verify_sst_unique_id_in_manifest",
  231. {offsetof(struct ImmutableDBOptions, verify_sst_unique_id_in_manifest),
  232. OptionType::kBoolean, OptionVerificationType::kNormal,
  233. OptionTypeFlags::kNone}},
  234. {"skip_log_error_on_recovery",
  235. {0, OptionType::kBoolean, OptionVerificationType::kDeprecated,
  236. OptionTypeFlags::kNone}},
  237. {"skip_stats_update_on_db_open",
  238. {offsetof(struct ImmutableDBOptions, skip_stats_update_on_db_open),
  239. OptionType::kBoolean, OptionVerificationType::kNormal,
  240. OptionTypeFlags::kNone}},
  241. {"skip_checking_sst_file_sizes_on_db_open",
  242. {offsetof(struct ImmutableDBOptions,
  243. skip_checking_sst_file_sizes_on_db_open),
  244. OptionType::kBoolean, OptionVerificationType::kNormal,
  245. OptionTypeFlags::kNone}},
  246. {"new_table_reader_for_compaction_inputs",
  247. {0, OptionType::kBoolean, OptionVerificationType::kDeprecated,
  248. OptionTypeFlags::kNone}},
  249. {"random_access_max_buffer_size",
  250. {0, OptionType::kSizeT, OptionVerificationType::kDeprecated,
  251. OptionTypeFlags::kNone}},
  252. {"use_adaptive_mutex",
  253. {offsetof(struct ImmutableDBOptions, use_adaptive_mutex),
  254. OptionType::kBoolean, OptionVerificationType::kNormal,
  255. OptionTypeFlags::kNone}},
  256. {"use_fsync",
  257. {offsetof(struct ImmutableDBOptions, use_fsync), OptionType::kBoolean,
  258. OptionVerificationType::kNormal, OptionTypeFlags::kNone}},
  259. {"max_file_opening_threads",
  260. {offsetof(struct ImmutableDBOptions, max_file_opening_threads),
  261. OptionType::kInt, OptionVerificationType::kNormal,
  262. OptionTypeFlags::kNone}},
  263. {"table_cache_numshardbits",
  264. {offsetof(struct ImmutableDBOptions, table_cache_numshardbits),
  265. OptionType::kInt, OptionVerificationType::kNormal,
  266. OptionTypeFlags::kNone}},
  267. {"db_write_buffer_size",
  268. {offsetof(struct ImmutableDBOptions, db_write_buffer_size),
  269. OptionType::kSizeT, OptionVerificationType::kNormal,
  270. OptionTypeFlags::kNone}},
  271. {"keep_log_file_num",
  272. {offsetof(struct ImmutableDBOptions, keep_log_file_num),
  273. OptionType::kSizeT, OptionVerificationType::kNormal,
  274. OptionTypeFlags::kNone}},
  275. {"recycle_log_file_num",
  276. {offsetof(struct ImmutableDBOptions, recycle_log_file_num),
  277. OptionType::kSizeT, OptionVerificationType::kNormal,
  278. OptionTypeFlags::kNone}},
  279. {"log_file_time_to_roll",
  280. {offsetof(struct ImmutableDBOptions, log_file_time_to_roll),
  281. OptionType::kSizeT, OptionVerificationType::kNormal,
  282. OptionTypeFlags::kNone}},
  283. {"manifest_preallocation_size",
  284. {offsetof(struct ImmutableDBOptions, manifest_preallocation_size),
  285. OptionType::kSizeT, OptionVerificationType::kNormal,
  286. OptionTypeFlags::kNone}},
  287. {"max_log_file_size",
  288. {offsetof(struct ImmutableDBOptions, max_log_file_size),
  289. OptionType::kSizeT, OptionVerificationType::kNormal,
  290. OptionTypeFlags::kNone}},
  291. {"db_log_dir",
  292. {offsetof(struct ImmutableDBOptions, db_log_dir), OptionType::kString,
  293. OptionVerificationType::kNormal, OptionTypeFlags::kNone}},
  294. {"wal_dir",
  295. {offsetof(struct ImmutableDBOptions, wal_dir), OptionType::kString,
  296. OptionVerificationType::kNormal, OptionTypeFlags::kNone}},
  297. {"WAL_size_limit_MB",
  298. {offsetof(struct ImmutableDBOptions, WAL_size_limit_MB),
  299. OptionType::kUInt64T, OptionVerificationType::kNormal,
  300. OptionTypeFlags::kNone}},
  301. {"WAL_ttl_seconds",
  302. {offsetof(struct ImmutableDBOptions, WAL_ttl_seconds),
  303. OptionType::kUInt64T, OptionVerificationType::kNormal,
  304. OptionTypeFlags::kNone}},
  305. {"max_manifest_file_size",
  306. {offsetof(struct ImmutableDBOptions, max_manifest_file_size),
  307. OptionType::kUInt64T, OptionVerificationType::kNormal,
  308. OptionTypeFlags::kNone}},
  309. {"persist_stats_to_disk",
  310. {offsetof(struct ImmutableDBOptions, persist_stats_to_disk),
  311. OptionType::kBoolean, OptionVerificationType::kNormal,
  312. OptionTypeFlags::kNone}},
  313. {"fail_if_options_file_error",
  314. {0, OptionType::kBoolean, OptionVerificationType::kDeprecated,
  315. OptionTypeFlags::kNone}},
  316. {"enable_pipelined_write",
  317. {offsetof(struct ImmutableDBOptions, enable_pipelined_write),
  318. OptionType::kBoolean, OptionVerificationType::kNormal,
  319. OptionTypeFlags::kNone}},
  320. {"unordered_write",
  321. {offsetof(struct ImmutableDBOptions, unordered_write),
  322. OptionType::kBoolean, OptionVerificationType::kNormal,
  323. OptionTypeFlags::kNone}},
  324. {"allow_concurrent_memtable_write",
  325. {offsetof(struct ImmutableDBOptions, allow_concurrent_memtable_write),
  326. OptionType::kBoolean, OptionVerificationType::kNormal,
  327. OptionTypeFlags::kNone}},
  328. {"wal_recovery_mode",
  329. OptionTypeInfo::Enum<WALRecoveryMode>(
  330. offsetof(struct ImmutableDBOptions, wal_recovery_mode),
  331. &wal_recovery_mode_string_map)},
  332. {"enable_write_thread_adaptive_yield",
  333. {offsetof(struct ImmutableDBOptions,
  334. enable_write_thread_adaptive_yield),
  335. OptionType::kBoolean, OptionVerificationType::kNormal,
  336. OptionTypeFlags::kNone}},
  337. {"write_thread_slow_yield_usec",
  338. {offsetof(struct ImmutableDBOptions, write_thread_slow_yield_usec),
  339. OptionType::kUInt64T, OptionVerificationType::kNormal,
  340. OptionTypeFlags::kNone}},
  341. {"max_write_batch_group_size_bytes",
  342. {offsetof(struct ImmutableDBOptions, max_write_batch_group_size_bytes),
  343. OptionType::kUInt64T, OptionVerificationType::kNormal,
  344. OptionTypeFlags::kNone}},
  345. {"write_thread_max_yield_usec",
  346. {offsetof(struct ImmutableDBOptions, write_thread_max_yield_usec),
  347. OptionType::kUInt64T, OptionVerificationType::kNormal,
  348. OptionTypeFlags::kNone}},
  349. {"access_hint_on_compaction_start",
  350. OptionTypeInfo::Enum<bool>(0, nullptr, OptionTypeFlags::kNone,
  351. OptionVerificationType::kDeprecated)},
  352. {"info_log_level",
  353. OptionTypeInfo::Enum<InfoLogLevel>(
  354. offsetof(struct ImmutableDBOptions, info_log_level),
  355. &info_log_level_string_map)},
  356. {"dump_malloc_stats",
  357. {offsetof(struct ImmutableDBOptions, dump_malloc_stats),
  358. OptionType::kBoolean, OptionVerificationType::kNormal,
  359. OptionTypeFlags::kNone}},
  360. {"avoid_flush_during_recovery",
  361. {offsetof(struct ImmutableDBOptions, avoid_flush_during_recovery),
  362. OptionType::kBoolean, OptionVerificationType::kNormal,
  363. OptionTypeFlags::kNone}},
  364. {"allow_ingest_behind",
  365. {offsetof(struct ImmutableDBOptions, allow_ingest_behind),
  366. OptionType::kBoolean, OptionVerificationType::kNormal,
  367. OptionTypeFlags::kNone}},
  368. {"preserve_deletes",
  369. {0, OptionType::kBoolean, OptionVerificationType::kDeprecated,
  370. OptionTypeFlags::kNone}},
  371. {"concurrent_prepare", // Deprecated by two_write_queues
  372. {0, OptionType::kBoolean, OptionVerificationType::kDeprecated,
  373. OptionTypeFlags::kNone}},
  374. {"two_write_queues",
  375. {offsetof(struct ImmutableDBOptions, two_write_queues),
  376. OptionType::kBoolean, OptionVerificationType::kNormal,
  377. OptionTypeFlags::kNone}},
  378. {"manual_wal_flush",
  379. {offsetof(struct ImmutableDBOptions, manual_wal_flush),
  380. OptionType::kBoolean, OptionVerificationType::kNormal,
  381. OptionTypeFlags::kNone}},
  382. {"wal_compression",
  383. {offsetof(struct ImmutableDBOptions, wal_compression),
  384. OptionType::kCompressionType, OptionVerificationType::kNormal,
  385. OptionTypeFlags::kNone}},
  386. {"background_close_inactive_wals",
  387. {offsetof(struct ImmutableDBOptions, background_close_inactive_wals),
  388. OptionType::kBoolean, OptionVerificationType::kNormal,
  389. OptionTypeFlags::kNone}},
  390. {"seq_per_batch",
  391. {0, OptionType::kBoolean, OptionVerificationType::kDeprecated,
  392. OptionTypeFlags::kNone}},
  393. {"atomic_flush",
  394. {offsetof(struct ImmutableDBOptions, atomic_flush),
  395. OptionType::kBoolean, OptionVerificationType::kNormal,
  396. OptionTypeFlags::kNone}},
  397. {"avoid_unnecessary_blocking_io",
  398. {offsetof(struct ImmutableDBOptions, avoid_unnecessary_blocking_io),
  399. OptionType::kBoolean, OptionVerificationType::kNormal,
  400. OptionTypeFlags::kNone}},
  401. {"prefix_seek_opt_in_only",
  402. {offsetof(struct ImmutableDBOptions, prefix_seek_opt_in_only),
  403. OptionType::kBoolean, OptionVerificationType::kNormal,
  404. OptionTypeFlags::kNone}},
  405. {"write_dbid_to_manifest",
  406. {offsetof(struct ImmutableDBOptions, write_dbid_to_manifest),
  407. OptionType::kBoolean, OptionVerificationType::kNormal,
  408. OptionTypeFlags::kNone}},
  409. {"write_identity_file",
  410. {offsetof(struct ImmutableDBOptions, write_identity_file),
  411. OptionType::kBoolean, OptionVerificationType::kNormal,
  412. OptionTypeFlags::kNone}},
  413. {"log_readahead_size",
  414. {offsetof(struct ImmutableDBOptions, log_readahead_size),
  415. OptionType::kSizeT, OptionVerificationType::kNormal,
  416. OptionTypeFlags::kNone}},
  417. {"best_efforts_recovery",
  418. {offsetof(struct ImmutableDBOptions, best_efforts_recovery),
  419. OptionType::kBoolean, OptionVerificationType::kNormal,
  420. OptionTypeFlags::kNone}},
  421. {"max_bgerror_resume_count",
  422. {offsetof(struct ImmutableDBOptions, max_bgerror_resume_count),
  423. OptionType::kInt, OptionVerificationType::kNormal,
  424. OptionTypeFlags::kNone}},
  425. {"bgerror_resume_retry_interval",
  426. {offsetof(struct ImmutableDBOptions, bgerror_resume_retry_interval),
  427. OptionType::kUInt64T, OptionVerificationType::kNormal,
  428. OptionTypeFlags::kNone}},
  429. {"db_host_id",
  430. {offsetof(struct ImmutableDBOptions, db_host_id), OptionType::kString,
  431. OptionVerificationType::kNormal, OptionTypeFlags::kCompareNever}},
  432. // Temporarily deprecated due to race conditions (examples in PR 10375).
  433. {"rate_limiter",
  434. {offsetof(struct ImmutableDBOptions, rate_limiter),
  435. OptionType::kUnknown, OptionVerificationType::kDeprecated,
  436. OptionTypeFlags::kDontSerialize | OptionTypeFlags::kCompareNever}},
  437. // The following properties were handled as special cases in ParseOption
  438. // This means that the properties could be read from the options file
  439. // but never written to the file or compared to each other.
  440. {"rate_limiter_bytes_per_sec",
  441. {offsetof(struct ImmutableDBOptions, rate_limiter),
  442. OptionType::kUnknown, OptionVerificationType::kNormal,
  443. (OptionTypeFlags::kDontSerialize | OptionTypeFlags::kCompareNever),
  444. // Parse the input value as a RateLimiter
  445. [](const ConfigOptions& /*opts*/, const std::string& /*name*/,
  446. const std::string& value, void* addr) {
  447. auto limiter = static_cast<std::shared_ptr<RateLimiter>*>(addr);
  448. limiter->reset(NewGenericRateLimiter(
  449. static_cast<int64_t>(ParseUint64(value))));
  450. return Status::OK();
  451. }}},
  452. {"env", //**TODO: Should this be kCustomizable?
  453. OptionTypeInfo(
  454. offsetof(struct ImmutableDBOptions, env), OptionType::kUnknown,
  455. OptionVerificationType::kNormal,
  456. (OptionTypeFlags::kDontSerialize | OptionTypeFlags::kCompareNever))
  457. .SetParseFunc([](const ConfigOptions& opts,
  458. const std::string& /*name*/,
  459. const std::string& value, void* addr) {
  460. // Parse the input value as an Env
  461. auto old_env = static_cast<Env**>(addr); // Get the old value
  462. Env* new_env = *old_env; // Set new to old
  463. Status s = Env::CreateFromString(opts, value,
  464. &new_env); // Update new value
  465. if (s.ok()) { // It worked
  466. *old_env = new_env; // Update the old one
  467. }
  468. return s;
  469. })
  470. .SetPrepareFunc([](const ConfigOptions& opts,
  471. const std::string& /*name*/, void* addr) {
  472. auto env = static_cast<Env**>(addr);
  473. return (*env)->PrepareOptions(opts);
  474. })
  475. .SetValidateFunc([](const DBOptions& db_opts,
  476. const ColumnFamilyOptions& cf_opts,
  477. const std::string& /*name*/,
  478. const void* addr) {
  479. const auto env = static_cast<const Env* const*>(addr);
  480. return (*env)->ValidateOptions(db_opts, cf_opts);
  481. })},
  482. {"allow_data_in_errors",
  483. {offsetof(struct ImmutableDBOptions, allow_data_in_errors),
  484. OptionType::kBoolean, OptionVerificationType::kNormal,
  485. OptionTypeFlags::kNone}},
  486. {"file_checksum_gen_factory",
  487. OptionTypeInfo::AsCustomSharedPtr<FileChecksumGenFactory>(
  488. offsetof(struct ImmutableDBOptions, file_checksum_gen_factory),
  489. OptionVerificationType::kByNameAllowFromNull,
  490. OptionTypeFlags::kAllowNull)},
  491. {"statistics",
  492. OptionTypeInfo::AsCustomSharedPtr<Statistics>(
  493. // Statistics should not be compared and can be null
  494. // Statistics are maked "don't serialize" until they can be shared
  495. // between DBs
  496. offsetof(struct ImmutableDBOptions, statistics),
  497. OptionVerificationType::kNormal,
  498. OptionTypeFlags::kCompareNever | OptionTypeFlags::kDontSerialize |
  499. OptionTypeFlags::kAllowNull)},
  500. // Allow EventListeners that have a non-empty Name() to be read/written
  501. // as options Each listener will either be
  502. // - A simple name (e.g. "MyEventListener")
  503. // - A name with properties (e.g. "{id=MyListener1; timeout=60}"
  504. // Multiple listeners will be separated by a ":":
  505. // - "MyListener0;{id=MyListener1; timeout=60}
  506. {"listeners",
  507. {offsetof(struct ImmutableDBOptions, listeners), OptionType::kVector,
  508. OptionVerificationType::kByNameAllowNull,
  509. OptionTypeFlags::kCompareNever,
  510. [](const ConfigOptions& opts, const std::string& /*name*/,
  511. const std::string& value, void* addr) {
  512. ConfigOptions embedded = opts;
  513. embedded.ignore_unsupported_options = true;
  514. std::vector<std::shared_ptr<EventListener>> listeners;
  515. Status s;
  516. for (size_t start = 0, end = 0;
  517. s.ok() && start < value.size() && end != std::string::npos;
  518. start = end + 1) {
  519. std::string token;
  520. s = OptionTypeInfo::NextToken(value, ':', start, &end, &token);
  521. if (s.ok() && !token.empty()) {
  522. std::shared_ptr<EventListener> listener;
  523. s = EventListener::CreateFromString(embedded, token, &listener);
  524. if (s.ok() && listener != nullptr) {
  525. listeners.push_back(listener);
  526. }
  527. }
  528. }
  529. if (s.ok()) { // It worked
  530. *(static_cast<std::vector<std::shared_ptr<EventListener>>*>(
  531. addr)) = listeners;
  532. }
  533. return s;
  534. },
  535. [](const ConfigOptions& opts, const std::string& /*name*/,
  536. const void* addr, std::string* value) {
  537. const auto listeners =
  538. static_cast<const std::vector<std::shared_ptr<EventListener>>*>(
  539. addr);
  540. ConfigOptions embedded = opts;
  541. embedded.delimiter = ";";
  542. int printed = 0;
  543. for (const auto& listener : *listeners) {
  544. auto id = listener->GetId();
  545. if (!id.empty()) {
  546. std::string elem_str = listener->ToString(embedded, "");
  547. if (printed++ == 0) {
  548. value->append("{");
  549. } else {
  550. value->append(":");
  551. }
  552. value->append(elem_str);
  553. }
  554. }
  555. if (printed > 0) {
  556. value->append("}");
  557. }
  558. return Status::OK();
  559. },
  560. nullptr}},
  561. {"lowest_used_cache_tier",
  562. OptionTypeInfo::Enum<CacheTier>(
  563. offsetof(struct ImmutableDBOptions, lowest_used_cache_tier),
  564. &cache_tier_string_map, OptionTypeFlags::kNone)},
  565. {"enforce_single_del_contracts",
  566. {offsetof(struct ImmutableDBOptions, enforce_single_del_contracts),
  567. OptionType::kBoolean, OptionVerificationType::kNormal,
  568. OptionTypeFlags::kNone}},
  569. {"follower_refresh_catchup_period_ms",
  570. {offsetof(struct ImmutableDBOptions,
  571. follower_refresh_catchup_period_ms),
  572. OptionType::kUInt64T, OptionVerificationType::kNormal,
  573. OptionTypeFlags::kNone}},
  574. {"follower_catchup_retry_count",
  575. {offsetof(struct ImmutableDBOptions, follower_catchup_retry_count),
  576. OptionType::kUInt64T, OptionVerificationType::kNormal,
  577. OptionTypeFlags::kNone}},
  578. {"follower_catchup_retry_wait_ms",
  579. {offsetof(struct ImmutableDBOptions, follower_catchup_retry_wait_ms),
  580. OptionType::kUInt64T, OptionVerificationType::kNormal,
  581. OptionTypeFlags::kNone}},
  582. {"metadata_write_temperature",
  583. {offsetof(struct ImmutableDBOptions, metadata_write_temperature),
  584. OptionType::kTemperature, OptionVerificationType::kNormal,
  585. OptionTypeFlags::kNone}},
  586. {"wal_write_temperature",
  587. {offsetof(struct ImmutableDBOptions, wal_write_temperature),
  588. OptionType::kTemperature, OptionVerificationType::kNormal,
  589. OptionTypeFlags::kNone}},
  590. };
  591. const std::string OptionsHelper::kDBOptionsName = "DBOptions";
  592. class MutableDBConfigurable : public Configurable {
  593. public:
  594. explicit MutableDBConfigurable(
  595. const MutableDBOptions& mdb,
  596. const std::unordered_map<std::string, std::string>* map = nullptr)
  597. : mutable_(mdb), opt_map_(map) {
  598. RegisterOptions(&mutable_, &db_mutable_options_type_info);
  599. }
  600. bool OptionsAreEqual(const ConfigOptions& config_options,
  601. const OptionTypeInfo& opt_info,
  602. const std::string& opt_name, const void* const this_ptr,
  603. const void* const that_ptr,
  604. std::string* mismatch) const override {
  605. bool equals = opt_info.AreEqual(config_options, opt_name, this_ptr,
  606. that_ptr, mismatch);
  607. if (!equals && opt_info.IsByName()) {
  608. if (opt_map_ == nullptr) {
  609. equals = true;
  610. } else {
  611. const auto& iter = opt_map_->find(opt_name);
  612. if (iter == opt_map_->end()) {
  613. equals = true;
  614. } else {
  615. equals = opt_info.AreEqualByName(config_options, opt_name, this_ptr,
  616. iter->second);
  617. }
  618. }
  619. if (equals) { // False alarm, clear mismatch
  620. *mismatch = "";
  621. }
  622. }
  623. if (equals && opt_info.IsConfigurable() && opt_map_ != nullptr) {
  624. const auto* this_config = opt_info.AsRawPointer<Configurable>(this_ptr);
  625. if (this_config == nullptr) {
  626. const auto& iter = opt_map_->find(opt_name);
  627. // If the name exists in the map and is not empty/null,
  628. // then the this_config should be set.
  629. if (iter != opt_map_->end() && !iter->second.empty() &&
  630. iter->second != kNullptrString) {
  631. *mismatch = opt_name;
  632. equals = false;
  633. }
  634. }
  635. }
  636. return equals;
  637. }
  638. protected:
  639. MutableDBOptions mutable_;
  640. const std::unordered_map<std::string, std::string>* opt_map_;
  641. };
  642. class DBOptionsConfigurable : public MutableDBConfigurable {
  643. public:
  644. explicit DBOptionsConfigurable(
  645. const DBOptions& opts,
  646. const std::unordered_map<std::string, std::string>* map = nullptr)
  647. : MutableDBConfigurable(MutableDBOptions(opts), map), db_options_(opts) {
  648. // The ImmutableDBOptions currently requires the env to be non-null. Make
  649. // sure it is
  650. if (opts.env != nullptr) {
  651. immutable_ = ImmutableDBOptions(opts);
  652. } else {
  653. DBOptions copy = opts;
  654. copy.env = Env::Default();
  655. immutable_ = ImmutableDBOptions(copy);
  656. }
  657. RegisterOptions(&immutable_, &db_immutable_options_type_info);
  658. }
  659. protected:
  660. Status ConfigureOptions(
  661. const ConfigOptions& config_options,
  662. const std::unordered_map<std::string, std::string>& opts_map,
  663. std::unordered_map<std::string, std::string>* unused) override {
  664. Status s = Configurable::ConfigureOptions(config_options, opts_map, unused);
  665. if (s.ok()) {
  666. db_options_ = BuildDBOptions(immutable_, mutable_);
  667. s = PrepareOptions(config_options);
  668. }
  669. return s;
  670. }
  671. const void* GetOptionsPtr(const std::string& name) const override {
  672. if (name == OptionsHelper::kDBOptionsName) {
  673. return &db_options_;
  674. } else {
  675. return MutableDBConfigurable::GetOptionsPtr(name);
  676. }
  677. }
  678. private:
  679. ImmutableDBOptions immutable_;
  680. DBOptions db_options_;
  681. };
  682. std::unique_ptr<Configurable> DBOptionsAsConfigurable(
  683. const MutableDBOptions& opts) {
  684. std::unique_ptr<Configurable> ptr(new MutableDBConfigurable(opts));
  685. return ptr;
  686. }
  687. std::unique_ptr<Configurable> DBOptionsAsConfigurable(
  688. const DBOptions& opts,
  689. const std::unordered_map<std::string, std::string>* opt_map) {
  690. std::unique_ptr<Configurable> ptr(new DBOptionsConfigurable(opts, opt_map));
  691. return ptr;
  692. }
  693. ImmutableDBOptions::ImmutableDBOptions() : ImmutableDBOptions(Options()) {}
  694. ImmutableDBOptions::ImmutableDBOptions(const DBOptions& options)
  695. : create_if_missing(options.create_if_missing),
  696. create_missing_column_families(options.create_missing_column_families),
  697. error_if_exists(options.error_if_exists),
  698. paranoid_checks(options.paranoid_checks),
  699. flush_verify_memtable_count(options.flush_verify_memtable_count),
  700. compaction_verify_record_count(options.compaction_verify_record_count),
  701. track_and_verify_wals_in_manifest(
  702. options.track_and_verify_wals_in_manifest),
  703. track_and_verify_wals(options.track_and_verify_wals),
  704. verify_sst_unique_id_in_manifest(
  705. options.verify_sst_unique_id_in_manifest),
  706. env(options.env),
  707. rate_limiter(options.rate_limiter),
  708. sst_file_manager(options.sst_file_manager),
  709. info_log(options.info_log),
  710. info_log_level(options.info_log_level),
  711. max_file_opening_threads(options.max_file_opening_threads),
  712. statistics(options.statistics),
  713. use_fsync(options.use_fsync),
  714. db_paths(options.db_paths),
  715. db_log_dir(options.db_log_dir),
  716. wal_dir(options.wal_dir),
  717. max_log_file_size(options.max_log_file_size),
  718. log_file_time_to_roll(options.log_file_time_to_roll),
  719. keep_log_file_num(options.keep_log_file_num),
  720. recycle_log_file_num(options.recycle_log_file_num),
  721. max_manifest_file_size(options.max_manifest_file_size),
  722. table_cache_numshardbits(options.table_cache_numshardbits),
  723. WAL_ttl_seconds(options.WAL_ttl_seconds),
  724. WAL_size_limit_MB(options.WAL_size_limit_MB),
  725. max_write_batch_group_size_bytes(
  726. options.max_write_batch_group_size_bytes),
  727. manifest_preallocation_size(options.manifest_preallocation_size),
  728. allow_mmap_reads(options.allow_mmap_reads),
  729. allow_mmap_writes(options.allow_mmap_writes),
  730. use_direct_reads(options.use_direct_reads),
  731. use_direct_io_for_flush_and_compaction(
  732. options.use_direct_io_for_flush_and_compaction),
  733. allow_fallocate(options.allow_fallocate),
  734. is_fd_close_on_exec(options.is_fd_close_on_exec),
  735. advise_random_on_open(options.advise_random_on_open),
  736. db_write_buffer_size(options.db_write_buffer_size),
  737. write_buffer_manager(options.write_buffer_manager),
  738. use_adaptive_mutex(options.use_adaptive_mutex),
  739. listeners(options.listeners),
  740. enable_thread_tracking(options.enable_thread_tracking),
  741. enable_pipelined_write(options.enable_pipelined_write),
  742. unordered_write(options.unordered_write),
  743. allow_concurrent_memtable_write(options.allow_concurrent_memtable_write),
  744. enable_write_thread_adaptive_yield(
  745. options.enable_write_thread_adaptive_yield),
  746. write_thread_max_yield_usec(options.write_thread_max_yield_usec),
  747. write_thread_slow_yield_usec(options.write_thread_slow_yield_usec),
  748. skip_stats_update_on_db_open(options.skip_stats_update_on_db_open),
  749. skip_checking_sst_file_sizes_on_db_open(
  750. options.skip_checking_sst_file_sizes_on_db_open),
  751. wal_recovery_mode(options.wal_recovery_mode),
  752. allow_2pc(options.allow_2pc),
  753. row_cache(options.row_cache),
  754. wal_filter(options.wal_filter),
  755. dump_malloc_stats(options.dump_malloc_stats),
  756. avoid_flush_during_recovery(options.avoid_flush_during_recovery),
  757. allow_ingest_behind(options.allow_ingest_behind),
  758. two_write_queues(options.two_write_queues),
  759. manual_wal_flush(options.manual_wal_flush),
  760. wal_compression(options.wal_compression),
  761. background_close_inactive_wals(options.background_close_inactive_wals),
  762. atomic_flush(options.atomic_flush),
  763. avoid_unnecessary_blocking_io(options.avoid_unnecessary_blocking_io),
  764. prefix_seek_opt_in_only(options.prefix_seek_opt_in_only),
  765. persist_stats_to_disk(options.persist_stats_to_disk),
  766. write_dbid_to_manifest(options.write_dbid_to_manifest),
  767. write_identity_file(options.write_identity_file),
  768. log_readahead_size(options.log_readahead_size),
  769. file_checksum_gen_factory(options.file_checksum_gen_factory),
  770. best_efforts_recovery(options.best_efforts_recovery),
  771. max_bgerror_resume_count(options.max_bgerror_resume_count),
  772. bgerror_resume_retry_interval(options.bgerror_resume_retry_interval),
  773. allow_data_in_errors(options.allow_data_in_errors),
  774. db_host_id(options.db_host_id),
  775. checksum_handoff_file_types(options.checksum_handoff_file_types),
  776. lowest_used_cache_tier(options.lowest_used_cache_tier),
  777. compaction_service(options.compaction_service),
  778. enforce_single_del_contracts(options.enforce_single_del_contracts),
  779. follower_refresh_catchup_period_ms(
  780. options.follower_refresh_catchup_period_ms),
  781. follower_catchup_retry_count(options.follower_catchup_retry_count),
  782. follower_catchup_retry_wait_ms(options.follower_catchup_retry_wait_ms),
  783. metadata_write_temperature(options.metadata_write_temperature),
  784. wal_write_temperature(options.wal_write_temperature),
  785. calculate_sst_write_lifetime_hint_set(
  786. options.calculate_sst_write_lifetime_hint_set) {
  787. fs = env->GetFileSystem();
  788. clock = env->GetSystemClock().get();
  789. logger = info_log.get();
  790. stats = statistics.get();
  791. }
  792. void ImmutableDBOptions::Dump(Logger* log) const {
  793. ROCKS_LOG_HEADER(log, " Options.error_if_exists: %d",
  794. error_if_exists);
  795. ROCKS_LOG_HEADER(log, " Options.create_if_missing: %d",
  796. create_if_missing);
  797. ROCKS_LOG_HEADER(log, " Options.paranoid_checks: %d",
  798. paranoid_checks);
  799. ROCKS_LOG_HEADER(log, " Options.flush_verify_memtable_count: %d",
  800. flush_verify_memtable_count);
  801. ROCKS_LOG_HEADER(log, " Options.compaction_verify_record_count: %d",
  802. compaction_verify_record_count);
  803. ROCKS_LOG_HEADER(log,
  804. " "
  805. "Options.track_and_verify_wals_in_manifest: %d",
  806. track_and_verify_wals_in_manifest);
  807. ROCKS_LOG_HEADER(log,
  808. " "
  809. "Options.track_and_verify_wals: %d",
  810. track_and_verify_wals);
  811. ROCKS_LOG_HEADER(log, " Options.verify_sst_unique_id_in_manifest: %d",
  812. verify_sst_unique_id_in_manifest);
  813. ROCKS_LOG_HEADER(log, " Options.env: %p",
  814. env);
  815. ROCKS_LOG_HEADER(log, " Options.fs: %s",
  816. fs->Name());
  817. ROCKS_LOG_HEADER(log, " Options.info_log: %p",
  818. info_log.get());
  819. ROCKS_LOG_HEADER(log, " Options.max_file_opening_threads: %d",
  820. max_file_opening_threads);
  821. ROCKS_LOG_HEADER(log, " Options.statistics: %p",
  822. stats);
  823. if (stats) {
  824. ROCKS_LOG_HEADER(
  825. log, " Options.statistics stats level: %u",
  826. stats->get_stats_level());
  827. }
  828. ROCKS_LOG_HEADER(log, " Options.use_fsync: %d",
  829. use_fsync);
  830. ROCKS_LOG_HEADER(
  831. log, " Options.max_log_file_size: %" ROCKSDB_PRIszt,
  832. max_log_file_size);
  833. ROCKS_LOG_HEADER(log,
  834. " Options.max_manifest_file_size: %" PRIu64,
  835. max_manifest_file_size);
  836. ROCKS_LOG_HEADER(
  837. log, " Options.log_file_time_to_roll: %" ROCKSDB_PRIszt,
  838. log_file_time_to_roll);
  839. ROCKS_LOG_HEADER(
  840. log, " Options.keep_log_file_num: %" ROCKSDB_PRIszt,
  841. keep_log_file_num);
  842. ROCKS_LOG_HEADER(
  843. log, " Options.recycle_log_file_num: %" ROCKSDB_PRIszt,
  844. recycle_log_file_num);
  845. ROCKS_LOG_HEADER(log, " Options.allow_fallocate: %d",
  846. allow_fallocate);
  847. ROCKS_LOG_HEADER(log, " Options.allow_mmap_reads: %d",
  848. allow_mmap_reads);
  849. ROCKS_LOG_HEADER(log, " Options.allow_mmap_writes: %d",
  850. allow_mmap_writes);
  851. ROCKS_LOG_HEADER(log, " Options.use_direct_reads: %d",
  852. use_direct_reads);
  853. ROCKS_LOG_HEADER(log,
  854. " "
  855. "Options.use_direct_io_for_flush_and_compaction: %d",
  856. use_direct_io_for_flush_and_compaction);
  857. ROCKS_LOG_HEADER(log, " Options.create_missing_column_families: %d",
  858. create_missing_column_families);
  859. ROCKS_LOG_HEADER(log, " Options.db_log_dir: %s",
  860. db_log_dir.c_str());
  861. ROCKS_LOG_HEADER(log, " Options.wal_dir: %s",
  862. wal_dir.c_str());
  863. ROCKS_LOG_HEADER(log, " Options.table_cache_numshardbits: %d",
  864. table_cache_numshardbits);
  865. ROCKS_LOG_HEADER(log,
  866. " Options.WAL_ttl_seconds: %" PRIu64,
  867. WAL_ttl_seconds);
  868. ROCKS_LOG_HEADER(log,
  869. " Options.WAL_size_limit_MB: %" PRIu64,
  870. WAL_size_limit_MB);
  871. ROCKS_LOG_HEADER(log,
  872. " "
  873. "Options.max_write_batch_group_size_bytes: %" PRIu64,
  874. max_write_batch_group_size_bytes);
  875. ROCKS_LOG_HEADER(
  876. log, " Options.manifest_preallocation_size: %" ROCKSDB_PRIszt,
  877. manifest_preallocation_size);
  878. ROCKS_LOG_HEADER(log, " Options.is_fd_close_on_exec: %d",
  879. is_fd_close_on_exec);
  880. ROCKS_LOG_HEADER(log, " Options.advise_random_on_open: %d",
  881. advise_random_on_open);
  882. ROCKS_LOG_HEADER(
  883. log, " Options.db_write_buffer_size: %" ROCKSDB_PRIszt,
  884. db_write_buffer_size);
  885. ROCKS_LOG_HEADER(log, " Options.write_buffer_manager: %p",
  886. write_buffer_manager.get());
  887. ROCKS_LOG_HEADER(log, " Options.use_adaptive_mutex: %d",
  888. use_adaptive_mutex);
  889. ROCKS_LOG_HEADER(log, " Options.rate_limiter: %p",
  890. rate_limiter.get());
  891. Header(
  892. log, " Options.sst_file_manager.rate_bytes_per_sec: %" PRIi64,
  893. sst_file_manager ? sst_file_manager->GetDeleteRateBytesPerSecond() : 0);
  894. ROCKS_LOG_HEADER(log, " Options.wal_recovery_mode: %d",
  895. static_cast<int>(wal_recovery_mode));
  896. ROCKS_LOG_HEADER(log, " Options.enable_thread_tracking: %d",
  897. enable_thread_tracking);
  898. ROCKS_LOG_HEADER(log, " Options.enable_pipelined_write: %d",
  899. enable_pipelined_write);
  900. ROCKS_LOG_HEADER(log, " Options.unordered_write: %d",
  901. unordered_write);
  902. ROCKS_LOG_HEADER(log, " Options.allow_concurrent_memtable_write: %d",
  903. allow_concurrent_memtable_write);
  904. ROCKS_LOG_HEADER(log, " Options.enable_write_thread_adaptive_yield: %d",
  905. enable_write_thread_adaptive_yield);
  906. ROCKS_LOG_HEADER(log,
  907. " Options.write_thread_max_yield_usec: %" PRIu64,
  908. write_thread_max_yield_usec);
  909. ROCKS_LOG_HEADER(log,
  910. " Options.write_thread_slow_yield_usec: %" PRIu64,
  911. write_thread_slow_yield_usec);
  912. if (row_cache) {
  913. ROCKS_LOG_HEADER(
  914. log,
  915. " Options.row_cache: %" ROCKSDB_PRIszt,
  916. row_cache->GetCapacity());
  917. } else {
  918. ROCKS_LOG_HEADER(log,
  919. " Options.row_cache: None");
  920. }
  921. ROCKS_LOG_HEADER(log, " Options.wal_filter: %s",
  922. wal_filter ? wal_filter->Name() : "None");
  923. ROCKS_LOG_HEADER(log, " Options.avoid_flush_during_recovery: %d",
  924. avoid_flush_during_recovery);
  925. ROCKS_LOG_HEADER(log, " Options.allow_ingest_behind: %d",
  926. allow_ingest_behind);
  927. ROCKS_LOG_HEADER(log, " Options.two_write_queues: %d",
  928. two_write_queues);
  929. ROCKS_LOG_HEADER(log, " Options.manual_wal_flush: %d",
  930. manual_wal_flush);
  931. ROCKS_LOG_HEADER(log, " Options.wal_compression: %d",
  932. wal_compression);
  933. ROCKS_LOG_HEADER(log,
  934. " Options.background_close_inactive_wals: %d",
  935. background_close_inactive_wals);
  936. ROCKS_LOG_HEADER(log, " Options.atomic_flush: %d", atomic_flush);
  937. ROCKS_LOG_HEADER(log, " Options.avoid_unnecessary_blocking_io: %d",
  938. avoid_unnecessary_blocking_io);
  939. ROCKS_LOG_HEADER(log, " Options.prefix_seek_opt_in_only: %d",
  940. prefix_seek_opt_in_only);
  941. ROCKS_LOG_HEADER(log, " Options.persist_stats_to_disk: %u",
  942. persist_stats_to_disk);
  943. ROCKS_LOG_HEADER(log, " Options.write_dbid_to_manifest: %d",
  944. write_dbid_to_manifest);
  945. ROCKS_LOG_HEADER(log, " Options.write_identity_file: %d",
  946. write_identity_file);
  947. ROCKS_LOG_HEADER(
  948. log, " Options.log_readahead_size: %" ROCKSDB_PRIszt,
  949. log_readahead_size);
  950. ROCKS_LOG_HEADER(log, " Options.file_checksum_gen_factory: %s",
  951. file_checksum_gen_factory ? file_checksum_gen_factory->Name()
  952. : kUnknownFileChecksumFuncName);
  953. ROCKS_LOG_HEADER(log, " Options.best_efforts_recovery: %d",
  954. static_cast<int>(best_efforts_recovery));
  955. ROCKS_LOG_HEADER(log, " Options.max_bgerror_resume_count: %d",
  956. max_bgerror_resume_count);
  957. ROCKS_LOG_HEADER(log,
  958. " Options.bgerror_resume_retry_interval: %" PRIu64,
  959. bgerror_resume_retry_interval);
  960. ROCKS_LOG_HEADER(log, " Options.allow_data_in_errors: %d",
  961. allow_data_in_errors);
  962. ROCKS_LOG_HEADER(log, " Options.db_host_id: %s",
  963. db_host_id.c_str());
  964. ROCKS_LOG_HEADER(log, " Options.enforce_single_del_contracts: %s",
  965. enforce_single_del_contracts ? "true" : "false");
  966. ROCKS_LOG_HEADER(log, " Options.metadata_write_temperature: %s",
  967. temperature_to_string[metadata_write_temperature].c_str());
  968. ROCKS_LOG_HEADER(log, " Options.wal_write_temperature: %s",
  969. temperature_to_string[wal_write_temperature].c_str());
  970. }
  971. bool ImmutableDBOptions::IsWalDirSameAsDBPath() const {
  972. assert(!db_paths.empty());
  973. return IsWalDirSameAsDBPath(db_paths[0].path);
  974. }
  975. bool ImmutableDBOptions::IsWalDirSameAsDBPath(
  976. const std::string& db_path) const {
  977. bool same = wal_dir.empty();
  978. if (!same) {
  979. Status s = env->AreFilesSame(wal_dir, db_path, &same);
  980. if (s.IsNotSupported()) {
  981. same = wal_dir == db_path;
  982. }
  983. }
  984. return same;
  985. }
  986. const std::string& ImmutableDBOptions::GetWalDir() const {
  987. if (wal_dir.empty()) {
  988. assert(!db_paths.empty());
  989. return db_paths[0].path;
  990. } else {
  991. return wal_dir;
  992. }
  993. }
  994. const std::string& ImmutableDBOptions::GetWalDir(
  995. const std::string& path) const {
  996. if (wal_dir.empty()) {
  997. return path;
  998. } else {
  999. return wal_dir;
  1000. }
  1001. }
  1002. MutableDBOptions::MutableDBOptions()
  1003. : max_background_jobs(2),
  1004. max_background_compactions(-1),
  1005. max_subcompactions(0),
  1006. avoid_flush_during_shutdown(false),
  1007. writable_file_max_buffer_size(1024 * 1024),
  1008. delayed_write_rate(2 * 1024U * 1024U),
  1009. max_total_wal_size(0),
  1010. delete_obsolete_files_period_micros(6ULL * 60 * 60 * 1000000),
  1011. stats_dump_period_sec(600),
  1012. stats_persist_period_sec(600),
  1013. stats_history_buffer_size(1024 * 1024),
  1014. max_open_files(-1),
  1015. bytes_per_sync(0),
  1016. wal_bytes_per_sync(0),
  1017. strict_bytes_per_sync(false),
  1018. compaction_readahead_size(0),
  1019. max_background_flushes(-1) {}
  1020. MutableDBOptions::MutableDBOptions(const DBOptions& options)
  1021. : max_background_jobs(options.max_background_jobs),
  1022. max_background_compactions(options.max_background_compactions),
  1023. max_subcompactions(options.max_subcompactions),
  1024. avoid_flush_during_shutdown(options.avoid_flush_during_shutdown),
  1025. writable_file_max_buffer_size(options.writable_file_max_buffer_size),
  1026. delayed_write_rate(options.delayed_write_rate),
  1027. max_total_wal_size(options.max_total_wal_size),
  1028. delete_obsolete_files_period_micros(
  1029. options.delete_obsolete_files_period_micros),
  1030. stats_dump_period_sec(options.stats_dump_period_sec),
  1031. stats_persist_period_sec(options.stats_persist_period_sec),
  1032. stats_history_buffer_size(options.stats_history_buffer_size),
  1033. max_open_files(options.max_open_files),
  1034. bytes_per_sync(options.bytes_per_sync),
  1035. wal_bytes_per_sync(options.wal_bytes_per_sync),
  1036. strict_bytes_per_sync(options.strict_bytes_per_sync),
  1037. compaction_readahead_size(options.compaction_readahead_size),
  1038. max_background_flushes(options.max_background_flushes),
  1039. daily_offpeak_time_utc(options.daily_offpeak_time_utc) {}
  1040. void MutableDBOptions::Dump(Logger* log) const {
  1041. ROCKS_LOG_HEADER(log, " Options.max_background_jobs: %d",
  1042. max_background_jobs);
  1043. ROCKS_LOG_HEADER(log, " Options.max_background_compactions: %d",
  1044. max_background_compactions);
  1045. ROCKS_LOG_HEADER(log, " Options.max_subcompactions: %" PRIu32,
  1046. max_subcompactions);
  1047. ROCKS_LOG_HEADER(log, " Options.avoid_flush_during_shutdown: %d",
  1048. avoid_flush_during_shutdown);
  1049. ROCKS_LOG_HEADER(
  1050. log, " Options.writable_file_max_buffer_size: %" ROCKSDB_PRIszt,
  1051. writable_file_max_buffer_size);
  1052. ROCKS_LOG_HEADER(log, " Options.delayed_write_rate : %" PRIu64,
  1053. delayed_write_rate);
  1054. ROCKS_LOG_HEADER(log, " Options.max_total_wal_size: %" PRIu64,
  1055. max_total_wal_size);
  1056. ROCKS_LOG_HEADER(
  1057. log, " Options.delete_obsolete_files_period_micros: %" PRIu64,
  1058. delete_obsolete_files_period_micros);
  1059. ROCKS_LOG_HEADER(log, " Options.stats_dump_period_sec: %u",
  1060. stats_dump_period_sec);
  1061. ROCKS_LOG_HEADER(log, " Options.stats_persist_period_sec: %d",
  1062. stats_persist_period_sec);
  1063. ROCKS_LOG_HEADER(
  1064. log,
  1065. " Options.stats_history_buffer_size: %" ROCKSDB_PRIszt,
  1066. stats_history_buffer_size);
  1067. ROCKS_LOG_HEADER(log, " Options.max_open_files: %d",
  1068. max_open_files);
  1069. ROCKS_LOG_HEADER(log,
  1070. " Options.bytes_per_sync: %" PRIu64,
  1071. bytes_per_sync);
  1072. ROCKS_LOG_HEADER(log,
  1073. " Options.wal_bytes_per_sync: %" PRIu64,
  1074. wal_bytes_per_sync);
  1075. ROCKS_LOG_HEADER(log, " Options.strict_bytes_per_sync: %d",
  1076. strict_bytes_per_sync);
  1077. ROCKS_LOG_HEADER(log,
  1078. " Options.compaction_readahead_size: %" ROCKSDB_PRIszt,
  1079. compaction_readahead_size);
  1080. ROCKS_LOG_HEADER(log, " Options.max_background_flushes: %d",
  1081. max_background_flushes);
  1082. ROCKS_LOG_HEADER(log, "Options.daily_offpeak_time_utc: %s",
  1083. daily_offpeak_time_utc.c_str());
  1084. }
  1085. Status GetMutableDBOptionsFromStrings(
  1086. const MutableDBOptions& base_options,
  1087. const std::unordered_map<std::string, std::string>& options_map,
  1088. MutableDBOptions* new_options) {
  1089. assert(new_options);
  1090. *new_options = base_options;
  1091. ConfigOptions config_options;
  1092. Status s = OptionTypeInfo::ParseType(
  1093. config_options, options_map, db_mutable_options_type_info, new_options);
  1094. if (!s.ok()) {
  1095. *new_options = base_options;
  1096. }
  1097. return s;
  1098. }
  1099. bool MutableDBOptionsAreEqual(const MutableDBOptions& this_options,
  1100. const MutableDBOptions& that_options) {
  1101. ConfigOptions config_options;
  1102. std::string mismatch;
  1103. return OptionTypeInfo::StructsAreEqual(
  1104. config_options, "MutableDBOptions", &db_mutable_options_type_info,
  1105. "MutableDBOptions", &this_options, &that_options, &mismatch);
  1106. }
  1107. Status GetStringFromMutableDBOptions(const ConfigOptions& config_options,
  1108. const MutableDBOptions& mutable_opts,
  1109. std::string* opt_string) {
  1110. return OptionTypeInfo::SerializeType(
  1111. config_options, db_mutable_options_type_info, &mutable_opts, opt_string);
  1112. }
  1113. } // namespace ROCKSDB_NAMESPACE