options_helper.cc 92 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124
  1. // Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
  2. // This source code is licensed under both the GPLv2 (found in the
  3. // COPYING file in the root directory) and Apache 2.0 License
  4. // (found in the LICENSE.Apache file in the root directory).
  5. #include "options/options_helper.h"
  6. #include <cassert>
  7. #include <cctype>
  8. #include <cstdlib>
  9. #include <unordered_set>
  10. #include <vector>
  11. #include "rocksdb/cache.h"
  12. #include "rocksdb/compaction_filter.h"
  13. #include "rocksdb/convenience.h"
  14. #include "rocksdb/filter_policy.h"
  15. #include "rocksdb/memtablerep.h"
  16. #include "rocksdb/merge_operator.h"
  17. #include "rocksdb/options.h"
  18. #include "rocksdb/rate_limiter.h"
  19. #include "rocksdb/slice_transform.h"
  20. #include "rocksdb/table.h"
  21. #include "rocksdb/utilities/object_registry.h"
  22. #include "table/block_based/block_based_table_factory.h"
  23. #include "table/plain/plain_table_factory.h"
  24. #include "util/cast_util.h"
  25. #include "util/string_util.h"
  26. namespace ROCKSDB_NAMESPACE {
  27. DBOptions BuildDBOptions(const ImmutableDBOptions& immutable_db_options,
  28. const MutableDBOptions& mutable_db_options) {
  29. DBOptions options;
  30. options.create_if_missing = immutable_db_options.create_if_missing;
  31. options.create_missing_column_families =
  32. immutable_db_options.create_missing_column_families;
  33. options.error_if_exists = immutable_db_options.error_if_exists;
  34. options.paranoid_checks = immutable_db_options.paranoid_checks;
  35. options.env = immutable_db_options.env;
  36. options.file_system = immutable_db_options.fs;
  37. options.rate_limiter = immutable_db_options.rate_limiter;
  38. options.sst_file_manager = immutable_db_options.sst_file_manager;
  39. options.info_log = immutable_db_options.info_log;
  40. options.info_log_level = immutable_db_options.info_log_level;
  41. options.max_open_files = mutable_db_options.max_open_files;
  42. options.max_file_opening_threads =
  43. immutable_db_options.max_file_opening_threads;
  44. options.max_total_wal_size = mutable_db_options.max_total_wal_size;
  45. options.statistics = immutable_db_options.statistics;
  46. options.use_fsync = immutable_db_options.use_fsync;
  47. options.db_paths = immutable_db_options.db_paths;
  48. options.db_log_dir = immutable_db_options.db_log_dir;
  49. options.wal_dir = immutable_db_options.wal_dir;
  50. options.delete_obsolete_files_period_micros =
  51. mutable_db_options.delete_obsolete_files_period_micros;
  52. options.max_background_jobs = mutable_db_options.max_background_jobs;
  53. options.base_background_compactions =
  54. mutable_db_options.base_background_compactions;
  55. options.max_background_compactions =
  56. mutable_db_options.max_background_compactions;
  57. options.bytes_per_sync = mutable_db_options.bytes_per_sync;
  58. options.wal_bytes_per_sync = mutable_db_options.wal_bytes_per_sync;
  59. options.strict_bytes_per_sync = mutable_db_options.strict_bytes_per_sync;
  60. options.max_subcompactions = immutable_db_options.max_subcompactions;
  61. options.max_background_flushes = immutable_db_options.max_background_flushes;
  62. options.max_log_file_size = immutable_db_options.max_log_file_size;
  63. options.log_file_time_to_roll = immutable_db_options.log_file_time_to_roll;
  64. options.keep_log_file_num = immutable_db_options.keep_log_file_num;
  65. options.recycle_log_file_num = immutable_db_options.recycle_log_file_num;
  66. options.max_manifest_file_size = immutable_db_options.max_manifest_file_size;
  67. options.table_cache_numshardbits =
  68. immutable_db_options.table_cache_numshardbits;
  69. options.WAL_ttl_seconds = immutable_db_options.wal_ttl_seconds;
  70. options.WAL_size_limit_MB = immutable_db_options.wal_size_limit_mb;
  71. options.manifest_preallocation_size =
  72. immutable_db_options.manifest_preallocation_size;
  73. options.allow_mmap_reads = immutable_db_options.allow_mmap_reads;
  74. options.allow_mmap_writes = immutable_db_options.allow_mmap_writes;
  75. options.use_direct_reads = immutable_db_options.use_direct_reads;
  76. options.use_direct_io_for_flush_and_compaction =
  77. immutable_db_options.use_direct_io_for_flush_and_compaction;
  78. options.allow_fallocate = immutable_db_options.allow_fallocate;
  79. options.is_fd_close_on_exec = immutable_db_options.is_fd_close_on_exec;
  80. options.stats_dump_period_sec = mutable_db_options.stats_dump_period_sec;
  81. options.stats_persist_period_sec =
  82. mutable_db_options.stats_persist_period_sec;
  83. options.persist_stats_to_disk = immutable_db_options.persist_stats_to_disk;
  84. options.stats_history_buffer_size =
  85. mutable_db_options.stats_history_buffer_size;
  86. options.advise_random_on_open = immutable_db_options.advise_random_on_open;
  87. options.db_write_buffer_size = immutable_db_options.db_write_buffer_size;
  88. options.write_buffer_manager = immutable_db_options.write_buffer_manager;
  89. options.access_hint_on_compaction_start =
  90. immutable_db_options.access_hint_on_compaction_start;
  91. options.new_table_reader_for_compaction_inputs =
  92. immutable_db_options.new_table_reader_for_compaction_inputs;
  93. options.compaction_readahead_size =
  94. mutable_db_options.compaction_readahead_size;
  95. options.random_access_max_buffer_size =
  96. immutable_db_options.random_access_max_buffer_size;
  97. options.writable_file_max_buffer_size =
  98. mutable_db_options.writable_file_max_buffer_size;
  99. options.use_adaptive_mutex = immutable_db_options.use_adaptive_mutex;
  100. options.listeners = immutable_db_options.listeners;
  101. options.enable_thread_tracking = immutable_db_options.enable_thread_tracking;
  102. options.delayed_write_rate = mutable_db_options.delayed_write_rate;
  103. options.enable_pipelined_write = immutable_db_options.enable_pipelined_write;
  104. options.unordered_write = immutable_db_options.unordered_write;
  105. options.allow_concurrent_memtable_write =
  106. immutable_db_options.allow_concurrent_memtable_write;
  107. options.enable_write_thread_adaptive_yield =
  108. immutable_db_options.enable_write_thread_adaptive_yield;
  109. options.max_write_batch_group_size_bytes =
  110. immutable_db_options.max_write_batch_group_size_bytes;
  111. options.write_thread_max_yield_usec =
  112. immutable_db_options.write_thread_max_yield_usec;
  113. options.write_thread_slow_yield_usec =
  114. immutable_db_options.write_thread_slow_yield_usec;
  115. options.skip_stats_update_on_db_open =
  116. immutable_db_options.skip_stats_update_on_db_open;
  117. options.skip_checking_sst_file_sizes_on_db_open =
  118. immutable_db_options.skip_checking_sst_file_sizes_on_db_open;
  119. options.wal_recovery_mode = immutable_db_options.wal_recovery_mode;
  120. options.allow_2pc = immutable_db_options.allow_2pc;
  121. options.row_cache = immutable_db_options.row_cache;
  122. #ifndef ROCKSDB_LITE
  123. options.wal_filter = immutable_db_options.wal_filter;
  124. #endif // ROCKSDB_LITE
  125. options.fail_if_options_file_error =
  126. immutable_db_options.fail_if_options_file_error;
  127. options.dump_malloc_stats = immutable_db_options.dump_malloc_stats;
  128. options.avoid_flush_during_recovery =
  129. immutable_db_options.avoid_flush_during_recovery;
  130. options.avoid_flush_during_shutdown =
  131. mutable_db_options.avoid_flush_during_shutdown;
  132. options.allow_ingest_behind =
  133. immutable_db_options.allow_ingest_behind;
  134. options.preserve_deletes =
  135. immutable_db_options.preserve_deletes;
  136. options.two_write_queues = immutable_db_options.two_write_queues;
  137. options.manual_wal_flush = immutable_db_options.manual_wal_flush;
  138. options.atomic_flush = immutable_db_options.atomic_flush;
  139. options.avoid_unnecessary_blocking_io =
  140. immutable_db_options.avoid_unnecessary_blocking_io;
  141. options.log_readahead_size = immutable_db_options.log_readahead_size;
  142. options.sst_file_checksum_func = immutable_db_options.sst_file_checksum_func;
  143. return options;
  144. }
  145. ColumnFamilyOptions BuildColumnFamilyOptions(
  146. const ColumnFamilyOptions& options,
  147. const MutableCFOptions& mutable_cf_options) {
  148. ColumnFamilyOptions cf_opts(options);
  149. // Memtable related options
  150. cf_opts.write_buffer_size = mutable_cf_options.write_buffer_size;
  151. cf_opts.max_write_buffer_number = mutable_cf_options.max_write_buffer_number;
  152. cf_opts.arena_block_size = mutable_cf_options.arena_block_size;
  153. cf_opts.memtable_prefix_bloom_size_ratio =
  154. mutable_cf_options.memtable_prefix_bloom_size_ratio;
  155. cf_opts.memtable_whole_key_filtering =
  156. mutable_cf_options.memtable_whole_key_filtering;
  157. cf_opts.memtable_huge_page_size = mutable_cf_options.memtable_huge_page_size;
  158. cf_opts.max_successive_merges = mutable_cf_options.max_successive_merges;
  159. cf_opts.inplace_update_num_locks =
  160. mutable_cf_options.inplace_update_num_locks;
  161. cf_opts.prefix_extractor = mutable_cf_options.prefix_extractor;
  162. // Compaction related options
  163. cf_opts.disable_auto_compactions =
  164. mutable_cf_options.disable_auto_compactions;
  165. cf_opts.soft_pending_compaction_bytes_limit =
  166. mutable_cf_options.soft_pending_compaction_bytes_limit;
  167. cf_opts.hard_pending_compaction_bytes_limit =
  168. mutable_cf_options.hard_pending_compaction_bytes_limit;
  169. cf_opts.level0_file_num_compaction_trigger =
  170. mutable_cf_options.level0_file_num_compaction_trigger;
  171. cf_opts.level0_slowdown_writes_trigger =
  172. mutable_cf_options.level0_slowdown_writes_trigger;
  173. cf_opts.level0_stop_writes_trigger =
  174. mutable_cf_options.level0_stop_writes_trigger;
  175. cf_opts.max_compaction_bytes = mutable_cf_options.max_compaction_bytes;
  176. cf_opts.target_file_size_base = mutable_cf_options.target_file_size_base;
  177. cf_opts.target_file_size_multiplier =
  178. mutable_cf_options.target_file_size_multiplier;
  179. cf_opts.max_bytes_for_level_base =
  180. mutable_cf_options.max_bytes_for_level_base;
  181. cf_opts.max_bytes_for_level_multiplier =
  182. mutable_cf_options.max_bytes_for_level_multiplier;
  183. cf_opts.ttl = mutable_cf_options.ttl;
  184. cf_opts.periodic_compaction_seconds =
  185. mutable_cf_options.periodic_compaction_seconds;
  186. cf_opts.max_bytes_for_level_multiplier_additional.clear();
  187. for (auto value :
  188. mutable_cf_options.max_bytes_for_level_multiplier_additional) {
  189. cf_opts.max_bytes_for_level_multiplier_additional.emplace_back(value);
  190. }
  191. cf_opts.compaction_options_fifo = mutable_cf_options.compaction_options_fifo;
  192. cf_opts.compaction_options_universal =
  193. mutable_cf_options.compaction_options_universal;
  194. // Misc options
  195. cf_opts.max_sequential_skip_in_iterations =
  196. mutable_cf_options.max_sequential_skip_in_iterations;
  197. cf_opts.paranoid_file_checks = mutable_cf_options.paranoid_file_checks;
  198. cf_opts.report_bg_io_stats = mutable_cf_options.report_bg_io_stats;
  199. cf_opts.compression = mutable_cf_options.compression;
  200. cf_opts.sample_for_compression = mutable_cf_options.sample_for_compression;
  201. cf_opts.table_factory = options.table_factory;
  202. // TODO(yhchiang): find some way to handle the following derived options
  203. // * max_file_size
  204. return cf_opts;
  205. }
  206. std::map<CompactionStyle, std::string>
  207. OptionsHelper::compaction_style_to_string = {
  208. {kCompactionStyleLevel, "kCompactionStyleLevel"},
  209. {kCompactionStyleUniversal, "kCompactionStyleUniversal"},
  210. {kCompactionStyleFIFO, "kCompactionStyleFIFO"},
  211. {kCompactionStyleNone, "kCompactionStyleNone"}};
  212. std::map<CompactionPri, std::string> OptionsHelper::compaction_pri_to_string = {
  213. {kByCompensatedSize, "kByCompensatedSize"},
  214. {kOldestLargestSeqFirst, "kOldestLargestSeqFirst"},
  215. {kOldestSmallestSeqFirst, "kOldestSmallestSeqFirst"},
  216. {kMinOverlappingRatio, "kMinOverlappingRatio"}};
  217. std::map<CompactionStopStyle, std::string>
  218. OptionsHelper::compaction_stop_style_to_string = {
  219. {kCompactionStopStyleSimilarSize, "kCompactionStopStyleSimilarSize"},
  220. {kCompactionStopStyleTotalSize, "kCompactionStopStyleTotalSize"}};
  221. std::unordered_map<std::string, ChecksumType>
  222. OptionsHelper::checksum_type_string_map = {{"kNoChecksum", kNoChecksum},
  223. {"kCRC32c", kCRC32c},
  224. {"kxxHash", kxxHash},
  225. {"kxxHash64", kxxHash64}};
  226. std::unordered_map<std::string, CompressionType>
  227. OptionsHelper::compression_type_string_map = {
  228. {"kNoCompression", kNoCompression},
  229. {"kSnappyCompression", kSnappyCompression},
  230. {"kZlibCompression", kZlibCompression},
  231. {"kBZip2Compression", kBZip2Compression},
  232. {"kLZ4Compression", kLZ4Compression},
  233. {"kLZ4HCCompression", kLZ4HCCompression},
  234. {"kXpressCompression", kXpressCompression},
  235. {"kZSTD", kZSTD},
  236. {"kZSTDNotFinalCompression", kZSTDNotFinalCompression},
  237. {"kDisableCompressionOption", kDisableCompressionOption}};
  238. #ifndef ROCKSDB_LITE
  239. const std::string kNameComparator = "comparator";
  240. const std::string kNameEnv = "env";
  241. const std::string kNameMergeOperator = "merge_operator";
  242. template <typename T>
  243. Status GetStringFromStruct(
  244. std::string* opt_string, const T& options,
  245. const std::unordered_map<std::string, OptionTypeInfo>& type_info,
  246. const std::string& delimiter);
  247. namespace {
  248. template <typename T>
  249. bool ParseEnum(const std::unordered_map<std::string, T>& type_map,
  250. const std::string& type, T* value) {
  251. auto iter = type_map.find(type);
  252. if (iter != type_map.end()) {
  253. *value = iter->second;
  254. return true;
  255. }
  256. return false;
  257. }
  258. template <typename T>
  259. bool SerializeEnum(const std::unordered_map<std::string, T>& type_map,
  260. const T& type, std::string* value) {
  261. for (const auto& pair : type_map) {
  262. if (pair.second == type) {
  263. *value = pair.first;
  264. return true;
  265. }
  266. }
  267. return false;
  268. }
  269. bool SerializeVectorCompressionType(const std::vector<CompressionType>& types,
  270. std::string* value) {
  271. std::stringstream ss;
  272. bool result;
  273. for (size_t i = 0; i < types.size(); ++i) {
  274. if (i > 0) {
  275. ss << ':';
  276. }
  277. std::string string_type;
  278. result = SerializeEnum<CompressionType>(compression_type_string_map,
  279. types[i], &string_type);
  280. if (result == false) {
  281. return result;
  282. }
  283. ss << string_type;
  284. }
  285. *value = ss.str();
  286. return true;
  287. }
  288. bool ParseVectorCompressionType(
  289. const std::string& value,
  290. std::vector<CompressionType>* compression_per_level) {
  291. compression_per_level->clear();
  292. size_t start = 0;
  293. while (start < value.size()) {
  294. size_t end = value.find(':', start);
  295. bool is_ok;
  296. CompressionType type;
  297. if (end == std::string::npos) {
  298. is_ok = ParseEnum<CompressionType>(compression_type_string_map,
  299. value.substr(start), &type);
  300. if (!is_ok) {
  301. return false;
  302. }
  303. compression_per_level->emplace_back(type);
  304. break;
  305. } else {
  306. is_ok = ParseEnum<CompressionType>(
  307. compression_type_string_map, value.substr(start, end - start), &type);
  308. if (!is_ok) {
  309. return false;
  310. }
  311. compression_per_level->emplace_back(type);
  312. start = end + 1;
  313. }
  314. }
  315. return true;
  316. }
  317. // This is to handle backward compatibility, where compaction_options_fifo
  318. // could be assigned a single scalar value, say, like "23", which would be
  319. // assigned to max_table_files_size.
  320. bool FIFOCompactionOptionsSpecialCase(const std::string& opt_str,
  321. CompactionOptionsFIFO* options) {
  322. if (opt_str.find("=") != std::string::npos) {
  323. // New format. Go do your new parsing using ParseStructOptions.
  324. return false;
  325. }
  326. // Old format. Parse just a single uint64_t value.
  327. options->max_table_files_size = ParseUint64(opt_str);
  328. return true;
  329. }
  330. template <typename T>
  331. bool SerializeStruct(
  332. const T& options, std::string* value,
  333. const std::unordered_map<std::string, OptionTypeInfo>& type_info_map) {
  334. std::string opt_str;
  335. Status s = GetStringFromStruct(&opt_str, options, type_info_map, ";");
  336. if (!s.ok()) {
  337. return false;
  338. }
  339. *value = "{" + opt_str + "}";
  340. return true;
  341. }
  342. template <typename T>
  343. bool ParseSingleStructOption(
  344. const std::string& opt_val_str, T* options,
  345. const std::unordered_map<std::string, OptionTypeInfo>& type_info_map) {
  346. size_t end = opt_val_str.find('=');
  347. std::string key = opt_val_str.substr(0, end);
  348. std::string value = opt_val_str.substr(end + 1);
  349. auto iter = type_info_map.find(key);
  350. if (iter == type_info_map.end()) {
  351. return false;
  352. }
  353. const auto& opt_info = iter->second;
  354. if (opt_info.verification == OptionVerificationType::kDeprecated) {
  355. // Should also skip deprecated sub-options such as
  356. // fifo_compaction_options_type_info.ttl
  357. return true;
  358. }
  359. return ParseOptionHelper(
  360. reinterpret_cast<char*>(options) + opt_info.mutable_offset, opt_info.type,
  361. value);
  362. }
  363. template <typename T>
  364. bool ParseStructOptions(
  365. const std::string& opt_str, T* options,
  366. const std::unordered_map<std::string, OptionTypeInfo>& type_info_map) {
  367. assert(!opt_str.empty());
  368. size_t start = 0;
  369. if (opt_str[0] == '{') {
  370. start++;
  371. }
  372. while ((start != std::string::npos) && (start < opt_str.size())) {
  373. if (opt_str[start] == '}') {
  374. break;
  375. }
  376. size_t end = opt_str.find(';', start);
  377. size_t len = (end == std::string::npos) ? end : end - start;
  378. if (!ParseSingleStructOption(opt_str.substr(start, len), options,
  379. type_info_map)) {
  380. return false;
  381. }
  382. start = (end == std::string::npos) ? end : end + 1;
  383. }
  384. return true;
  385. }
  386. } // anonymouse namespace
  387. bool ParseSliceTransformHelper(
  388. const std::string& kFixedPrefixName, const std::string& kCappedPrefixName,
  389. const std::string& value,
  390. std::shared_ptr<const SliceTransform>* slice_transform) {
  391. const char* no_op_name = "rocksdb.Noop";
  392. size_t no_op_length = strlen(no_op_name);
  393. auto& pe_value = value;
  394. if (pe_value.size() > kFixedPrefixName.size() &&
  395. pe_value.compare(0, kFixedPrefixName.size(), kFixedPrefixName) == 0) {
  396. int prefix_length = ParseInt(trim(value.substr(kFixedPrefixName.size())));
  397. slice_transform->reset(NewFixedPrefixTransform(prefix_length));
  398. } else if (pe_value.size() > kCappedPrefixName.size() &&
  399. pe_value.compare(0, kCappedPrefixName.size(), kCappedPrefixName) ==
  400. 0) {
  401. int prefix_length =
  402. ParseInt(trim(pe_value.substr(kCappedPrefixName.size())));
  403. slice_transform->reset(NewCappedPrefixTransform(prefix_length));
  404. } else if (pe_value.size() == no_op_length &&
  405. pe_value.compare(0, no_op_length, no_op_name) == 0) {
  406. const SliceTransform* no_op_transform = NewNoopTransform();
  407. slice_transform->reset(no_op_transform);
  408. } else if (value == kNullptrString) {
  409. slice_transform->reset();
  410. } else {
  411. return false;
  412. }
  413. return true;
  414. }
  415. bool ParseSliceTransform(
  416. const std::string& value,
  417. std::shared_ptr<const SliceTransform>* slice_transform) {
  418. // While we normally don't convert the string representation of a
  419. // pointer-typed option into its instance, here we do so for backward
  420. // compatibility as we allow this action in SetOption().
  421. // TODO(yhchiang): A possible better place for these serialization /
  422. // deserialization is inside the class definition of pointer-typed
  423. // option itself, but this requires a bigger change of public API.
  424. bool result =
  425. ParseSliceTransformHelper("fixed:", "capped:", value, slice_transform);
  426. if (result) {
  427. return result;
  428. }
  429. result = ParseSliceTransformHelper(
  430. "rocksdb.FixedPrefix.", "rocksdb.CappedPrefix.", value, slice_transform);
  431. if (result) {
  432. return result;
  433. }
  434. // TODO(yhchiang): we can further support other default
  435. // SliceTransforms here.
  436. return false;
  437. }
  438. bool ParseOptionHelper(char* opt_address, const OptionType& opt_type,
  439. const std::string& value) {
  440. switch (opt_type) {
  441. case OptionType::kBoolean:
  442. *reinterpret_cast<bool*>(opt_address) = ParseBoolean("", value);
  443. break;
  444. case OptionType::kInt:
  445. *reinterpret_cast<int*>(opt_address) = ParseInt(value);
  446. break;
  447. case OptionType::kInt32T:
  448. *reinterpret_cast<int32_t*>(opt_address) = ParseInt32(value);
  449. break;
  450. case OptionType::kInt64T:
  451. PutUnaligned(reinterpret_cast<int64_t*>(opt_address), ParseInt64(value));
  452. break;
  453. case OptionType::kVectorInt:
  454. *reinterpret_cast<std::vector<int>*>(opt_address) = ParseVectorInt(value);
  455. break;
  456. case OptionType::kUInt:
  457. *reinterpret_cast<unsigned int*>(opt_address) = ParseUint32(value);
  458. break;
  459. case OptionType::kUInt32T:
  460. *reinterpret_cast<uint32_t*>(opt_address) = ParseUint32(value);
  461. break;
  462. case OptionType::kUInt64T:
  463. PutUnaligned(reinterpret_cast<uint64_t*>(opt_address), ParseUint64(value));
  464. break;
  465. case OptionType::kSizeT:
  466. PutUnaligned(reinterpret_cast<size_t*>(opt_address), ParseSizeT(value));
  467. break;
  468. case OptionType::kString:
  469. *reinterpret_cast<std::string*>(opt_address) = value;
  470. break;
  471. case OptionType::kDouble:
  472. *reinterpret_cast<double*>(opt_address) = ParseDouble(value);
  473. break;
  474. case OptionType::kCompactionStyle:
  475. return ParseEnum<CompactionStyle>(
  476. compaction_style_string_map, value,
  477. reinterpret_cast<CompactionStyle*>(opt_address));
  478. case OptionType::kCompactionPri:
  479. return ParseEnum<CompactionPri>(
  480. compaction_pri_string_map, value,
  481. reinterpret_cast<CompactionPri*>(opt_address));
  482. case OptionType::kCompressionType:
  483. return ParseEnum<CompressionType>(
  484. compression_type_string_map, value,
  485. reinterpret_cast<CompressionType*>(opt_address));
  486. case OptionType::kVectorCompressionType:
  487. return ParseVectorCompressionType(
  488. value, reinterpret_cast<std::vector<CompressionType>*>(opt_address));
  489. case OptionType::kSliceTransform:
  490. return ParseSliceTransform(
  491. value, reinterpret_cast<std::shared_ptr<const SliceTransform>*>(
  492. opt_address));
  493. case OptionType::kChecksumType:
  494. return ParseEnum<ChecksumType>(
  495. checksum_type_string_map, value,
  496. reinterpret_cast<ChecksumType*>(opt_address));
  497. case OptionType::kBlockBasedTableIndexType:
  498. return ParseEnum<BlockBasedTableOptions::IndexType>(
  499. block_base_table_index_type_string_map, value,
  500. reinterpret_cast<BlockBasedTableOptions::IndexType*>(opt_address));
  501. case OptionType::kBlockBasedTableDataBlockIndexType:
  502. return ParseEnum<BlockBasedTableOptions::DataBlockIndexType>(
  503. block_base_table_data_block_index_type_string_map, value,
  504. reinterpret_cast<BlockBasedTableOptions::DataBlockIndexType*>(
  505. opt_address));
  506. case OptionType::kBlockBasedTableIndexShorteningMode:
  507. return ParseEnum<BlockBasedTableOptions::IndexShorteningMode>(
  508. block_base_table_index_shortening_mode_string_map, value,
  509. reinterpret_cast<BlockBasedTableOptions::IndexShorteningMode*>(
  510. opt_address));
  511. case OptionType::kEncodingType:
  512. return ParseEnum<EncodingType>(
  513. encoding_type_string_map, value,
  514. reinterpret_cast<EncodingType*>(opt_address));
  515. case OptionType::kWALRecoveryMode:
  516. return ParseEnum<WALRecoveryMode>(
  517. wal_recovery_mode_string_map, value,
  518. reinterpret_cast<WALRecoveryMode*>(opt_address));
  519. case OptionType::kAccessHint:
  520. return ParseEnum<DBOptions::AccessHint>(
  521. access_hint_string_map, value,
  522. reinterpret_cast<DBOptions::AccessHint*>(opt_address));
  523. case OptionType::kInfoLogLevel:
  524. return ParseEnum<InfoLogLevel>(
  525. info_log_level_string_map, value,
  526. reinterpret_cast<InfoLogLevel*>(opt_address));
  527. case OptionType::kCompactionOptionsFIFO: {
  528. if (!FIFOCompactionOptionsSpecialCase(
  529. value, reinterpret_cast<CompactionOptionsFIFO*>(opt_address))) {
  530. return ParseStructOptions<CompactionOptionsFIFO>(
  531. value, reinterpret_cast<CompactionOptionsFIFO*>(opt_address),
  532. fifo_compaction_options_type_info);
  533. }
  534. return true;
  535. }
  536. case OptionType::kLRUCacheOptions: {
  537. return ParseStructOptions<LRUCacheOptions>(value,
  538. reinterpret_cast<LRUCacheOptions*>(opt_address),
  539. lru_cache_options_type_info);
  540. }
  541. case OptionType::kCompactionOptionsUniversal:
  542. return ParseStructOptions<CompactionOptionsUniversal>(
  543. value, reinterpret_cast<CompactionOptionsUniversal*>(opt_address),
  544. universal_compaction_options_type_info);
  545. case OptionType::kCompactionStopStyle:
  546. return ParseEnum<CompactionStopStyle>(
  547. compaction_stop_style_string_map, value,
  548. reinterpret_cast<CompactionStopStyle*>(opt_address));
  549. default:
  550. return false;
  551. }
  552. return true;
  553. }
  554. bool SerializeSingleOptionHelper(const char* opt_address,
  555. const OptionType opt_type,
  556. std::string* value) {
  557. assert(value);
  558. switch (opt_type) {
  559. case OptionType::kBoolean:
  560. *value = *(reinterpret_cast<const bool*>(opt_address)) ? "true" : "false";
  561. break;
  562. case OptionType::kInt:
  563. *value = ToString(*(reinterpret_cast<const int*>(opt_address)));
  564. break;
  565. case OptionType::kInt32T:
  566. *value = ToString(*(reinterpret_cast<const int32_t*>(opt_address)));
  567. break;
  568. case OptionType::kInt64T:
  569. {
  570. int64_t v;
  571. GetUnaligned(reinterpret_cast<const int64_t*>(opt_address), &v);
  572. *value = ToString(v);
  573. }
  574. break;
  575. case OptionType::kVectorInt:
  576. return SerializeIntVector(
  577. *reinterpret_cast<const std::vector<int>*>(opt_address), value);
  578. case OptionType::kUInt:
  579. *value = ToString(*(reinterpret_cast<const unsigned int*>(opt_address)));
  580. break;
  581. case OptionType::kUInt32T:
  582. *value = ToString(*(reinterpret_cast<const uint32_t*>(opt_address)));
  583. break;
  584. case OptionType::kUInt64T:
  585. {
  586. uint64_t v;
  587. GetUnaligned(reinterpret_cast<const uint64_t*>(opt_address), &v);
  588. *value = ToString(v);
  589. }
  590. break;
  591. case OptionType::kSizeT:
  592. {
  593. size_t v;
  594. GetUnaligned(reinterpret_cast<const size_t*>(opt_address), &v);
  595. *value = ToString(v);
  596. }
  597. break;
  598. case OptionType::kDouble:
  599. *value = ToString(*(reinterpret_cast<const double*>(opt_address)));
  600. break;
  601. case OptionType::kString:
  602. *value = EscapeOptionString(
  603. *(reinterpret_cast<const std::string*>(opt_address)));
  604. break;
  605. case OptionType::kCompactionStyle:
  606. return SerializeEnum<CompactionStyle>(
  607. compaction_style_string_map,
  608. *(reinterpret_cast<const CompactionStyle*>(opt_address)), value);
  609. case OptionType::kCompactionPri:
  610. return SerializeEnum<CompactionPri>(
  611. compaction_pri_string_map,
  612. *(reinterpret_cast<const CompactionPri*>(opt_address)), value);
  613. case OptionType::kCompressionType:
  614. return SerializeEnum<CompressionType>(
  615. compression_type_string_map,
  616. *(reinterpret_cast<const CompressionType*>(opt_address)), value);
  617. case OptionType::kVectorCompressionType:
  618. return SerializeVectorCompressionType(
  619. *(reinterpret_cast<const std::vector<CompressionType>*>(opt_address)),
  620. value);
  621. break;
  622. case OptionType::kSliceTransform: {
  623. const auto* slice_transform_ptr =
  624. reinterpret_cast<const std::shared_ptr<const SliceTransform>*>(
  625. opt_address);
  626. *value = slice_transform_ptr->get() ? slice_transform_ptr->get()->Name()
  627. : kNullptrString;
  628. break;
  629. }
  630. case OptionType::kTableFactory: {
  631. const auto* table_factory_ptr =
  632. reinterpret_cast<const std::shared_ptr<const TableFactory>*>(
  633. opt_address);
  634. *value = table_factory_ptr->get() ? table_factory_ptr->get()->Name()
  635. : kNullptrString;
  636. break;
  637. }
  638. case OptionType::kComparator: {
  639. // it's a const pointer of const Comparator*
  640. const auto* ptr = reinterpret_cast<const Comparator* const*>(opt_address);
  641. // Since the user-specified comparator will be wrapped by
  642. // InternalKeyComparator, we should persist the user-specified one
  643. // instead of InternalKeyComparator.
  644. if (*ptr == nullptr) {
  645. *value = kNullptrString;
  646. } else {
  647. const Comparator* root_comp = (*ptr)->GetRootComparator();
  648. if (root_comp == nullptr) {
  649. root_comp = (*ptr);
  650. }
  651. *value = root_comp->Name();
  652. }
  653. break;
  654. }
  655. case OptionType::kCompactionFilter: {
  656. // it's a const pointer of const CompactionFilter*
  657. const auto* ptr =
  658. reinterpret_cast<const CompactionFilter* const*>(opt_address);
  659. *value = *ptr ? (*ptr)->Name() : kNullptrString;
  660. break;
  661. }
  662. case OptionType::kCompactionFilterFactory: {
  663. const auto* ptr =
  664. reinterpret_cast<const std::shared_ptr<CompactionFilterFactory>*>(
  665. opt_address);
  666. *value = ptr->get() ? ptr->get()->Name() : kNullptrString;
  667. break;
  668. }
  669. case OptionType::kMemTableRepFactory: {
  670. const auto* ptr =
  671. reinterpret_cast<const std::shared_ptr<MemTableRepFactory>*>(
  672. opt_address);
  673. *value = ptr->get() ? ptr->get()->Name() : kNullptrString;
  674. break;
  675. }
  676. case OptionType::kMergeOperator: {
  677. const auto* ptr =
  678. reinterpret_cast<const std::shared_ptr<MergeOperator>*>(opt_address);
  679. *value = ptr->get() ? ptr->get()->Name() : kNullptrString;
  680. break;
  681. }
  682. case OptionType::kFilterPolicy: {
  683. const auto* ptr =
  684. reinterpret_cast<const std::shared_ptr<FilterPolicy>*>(opt_address);
  685. *value = ptr->get() ? ptr->get()->Name() : kNullptrString;
  686. break;
  687. }
  688. case OptionType::kChecksumType:
  689. return SerializeEnum<ChecksumType>(
  690. checksum_type_string_map,
  691. *reinterpret_cast<const ChecksumType*>(opt_address), value);
  692. case OptionType::kBlockBasedTableIndexType:
  693. return SerializeEnum<BlockBasedTableOptions::IndexType>(
  694. block_base_table_index_type_string_map,
  695. *reinterpret_cast<const BlockBasedTableOptions::IndexType*>(
  696. opt_address),
  697. value);
  698. case OptionType::kBlockBasedTableDataBlockIndexType:
  699. return SerializeEnum<BlockBasedTableOptions::DataBlockIndexType>(
  700. block_base_table_data_block_index_type_string_map,
  701. *reinterpret_cast<const BlockBasedTableOptions::DataBlockIndexType*>(
  702. opt_address),
  703. value);
  704. case OptionType::kBlockBasedTableIndexShorteningMode:
  705. return SerializeEnum<BlockBasedTableOptions::IndexShorteningMode>(
  706. block_base_table_index_shortening_mode_string_map,
  707. *reinterpret_cast<const BlockBasedTableOptions::IndexShorteningMode*>(
  708. opt_address),
  709. value);
  710. case OptionType::kFlushBlockPolicyFactory: {
  711. const auto* ptr =
  712. reinterpret_cast<const std::shared_ptr<FlushBlockPolicyFactory>*>(
  713. opt_address);
  714. *value = ptr->get() ? ptr->get()->Name() : kNullptrString;
  715. break;
  716. }
  717. case OptionType::kEncodingType:
  718. return SerializeEnum<EncodingType>(
  719. encoding_type_string_map,
  720. *reinterpret_cast<const EncodingType*>(opt_address), value);
  721. case OptionType::kWALRecoveryMode:
  722. return SerializeEnum<WALRecoveryMode>(
  723. wal_recovery_mode_string_map,
  724. *reinterpret_cast<const WALRecoveryMode*>(opt_address), value);
  725. case OptionType::kAccessHint:
  726. return SerializeEnum<DBOptions::AccessHint>(
  727. access_hint_string_map,
  728. *reinterpret_cast<const DBOptions::AccessHint*>(opt_address), value);
  729. case OptionType::kInfoLogLevel:
  730. return SerializeEnum<InfoLogLevel>(
  731. info_log_level_string_map,
  732. *reinterpret_cast<const InfoLogLevel*>(opt_address), value);
  733. case OptionType::kCompactionOptionsFIFO:
  734. return SerializeStruct<CompactionOptionsFIFO>(
  735. *reinterpret_cast<const CompactionOptionsFIFO*>(opt_address), value,
  736. fifo_compaction_options_type_info);
  737. case OptionType::kCompactionOptionsUniversal:
  738. return SerializeStruct<CompactionOptionsUniversal>(
  739. *reinterpret_cast<const CompactionOptionsUniversal*>(opt_address),
  740. value, universal_compaction_options_type_info);
  741. case OptionType::kCompactionStopStyle:
  742. return SerializeEnum<CompactionStopStyle>(
  743. compaction_stop_style_string_map,
  744. *reinterpret_cast<const CompactionStopStyle*>(opt_address), value);
  745. default:
  746. return false;
  747. }
  748. return true;
  749. }
  750. Status GetMutableOptionsFromStrings(
  751. const MutableCFOptions& base_options,
  752. const std::unordered_map<std::string, std::string>& options_map,
  753. Logger* info_log, MutableCFOptions* new_options) {
  754. assert(new_options);
  755. *new_options = base_options;
  756. for (const auto& o : options_map) {
  757. try {
  758. auto iter = cf_options_type_info.find(o.first);
  759. if (iter == cf_options_type_info.end()) {
  760. return Status::InvalidArgument("Unrecognized option: " + o.first);
  761. }
  762. const auto& opt_info = iter->second;
  763. if (!opt_info.is_mutable) {
  764. return Status::InvalidArgument("Option not changeable: " + o.first);
  765. }
  766. if (opt_info.verification == OptionVerificationType::kDeprecated) {
  767. // log warning when user tries to set a deprecated option but don't fail
  768. // the call for compatibility.
  769. ROCKS_LOG_WARN(info_log, "%s is a deprecated option and cannot be set",
  770. o.first.c_str());
  771. continue;
  772. }
  773. bool is_ok = ParseOptionHelper(
  774. reinterpret_cast<char*>(new_options) + opt_info.mutable_offset,
  775. opt_info.type, o.second);
  776. if (!is_ok) {
  777. return Status::InvalidArgument("Error parsing " + o.first);
  778. }
  779. } catch (std::exception& e) {
  780. return Status::InvalidArgument("Error parsing " + o.first + ":" +
  781. std::string(e.what()));
  782. }
  783. }
  784. return Status::OK();
  785. }
  786. Status GetMutableDBOptionsFromStrings(
  787. const MutableDBOptions& base_options,
  788. const std::unordered_map<std::string, std::string>& options_map,
  789. MutableDBOptions* new_options) {
  790. assert(new_options);
  791. *new_options = base_options;
  792. for (const auto& o : options_map) {
  793. try {
  794. auto iter = db_options_type_info.find(o.first);
  795. if (iter == db_options_type_info.end()) {
  796. return Status::InvalidArgument("Unrecognized option: " + o.first);
  797. }
  798. const auto& opt_info = iter->second;
  799. if (!opt_info.is_mutable) {
  800. return Status::InvalidArgument("Option not changeable: " + o.first);
  801. }
  802. bool is_ok = ParseOptionHelper(
  803. reinterpret_cast<char*>(new_options) + opt_info.mutable_offset,
  804. opt_info.type, o.second);
  805. if (!is_ok) {
  806. return Status::InvalidArgument("Error parsing " + o.first);
  807. }
  808. } catch (std::exception& e) {
  809. return Status::InvalidArgument("Error parsing " + o.first + ":" +
  810. std::string(e.what()));
  811. }
  812. }
  813. return Status::OK();
  814. }
  815. Status StringToMap(const std::string& opts_str,
  816. std::unordered_map<std::string, std::string>* opts_map) {
  817. assert(opts_map);
  818. // Example:
  819. // opts_str = "write_buffer_size=1024;max_write_buffer_number=2;"
  820. // "nested_opt={opt1=1;opt2=2};max_bytes_for_level_base=100"
  821. size_t pos = 0;
  822. std::string opts = trim(opts_str);
  823. while (pos < opts.size()) {
  824. size_t eq_pos = opts.find('=', pos);
  825. if (eq_pos == std::string::npos) {
  826. return Status::InvalidArgument("Mismatched key value pair, '=' expected");
  827. }
  828. std::string key = trim(opts.substr(pos, eq_pos - pos));
  829. if (key.empty()) {
  830. return Status::InvalidArgument("Empty key found");
  831. }
  832. // skip space after '=' and look for '{' for possible nested options
  833. pos = eq_pos + 1;
  834. while (pos < opts.size() && isspace(opts[pos])) {
  835. ++pos;
  836. }
  837. // Empty value at the end
  838. if (pos >= opts.size()) {
  839. (*opts_map)[key] = "";
  840. break;
  841. }
  842. if (opts[pos] == '{') {
  843. int count = 1;
  844. size_t brace_pos = pos + 1;
  845. while (brace_pos < opts.size()) {
  846. if (opts[brace_pos] == '{') {
  847. ++count;
  848. } else if (opts[brace_pos] == '}') {
  849. --count;
  850. if (count == 0) {
  851. break;
  852. }
  853. }
  854. ++brace_pos;
  855. }
  856. // found the matching closing brace
  857. if (count == 0) {
  858. (*opts_map)[key] = trim(opts.substr(pos + 1, brace_pos - pos - 1));
  859. // skip all whitespace and move to the next ';'
  860. // brace_pos points to the next position after the matching '}'
  861. pos = brace_pos + 1;
  862. while (pos < opts.size() && isspace(opts[pos])) {
  863. ++pos;
  864. }
  865. if (pos < opts.size() && opts[pos] != ';') {
  866. return Status::InvalidArgument(
  867. "Unexpected chars after nested options");
  868. }
  869. ++pos;
  870. } else {
  871. return Status::InvalidArgument(
  872. "Mismatched curly braces for nested options");
  873. }
  874. } else {
  875. size_t sc_pos = opts.find(';', pos);
  876. if (sc_pos == std::string::npos) {
  877. (*opts_map)[key] = trim(opts.substr(pos));
  878. // It either ends with a trailing semi-colon or the last key-value pair
  879. break;
  880. } else {
  881. (*opts_map)[key] = trim(opts.substr(pos, sc_pos - pos));
  882. }
  883. pos = sc_pos + 1;
  884. }
  885. }
  886. return Status::OK();
  887. }
  888. Status ParseCompressionOptions(const std::string& value, const std::string& name,
  889. CompressionOptions& compression_opts) {
  890. size_t start = 0;
  891. size_t end = value.find(':');
  892. if (end == std::string::npos) {
  893. return Status::InvalidArgument("unable to parse the specified CF option " +
  894. name);
  895. }
  896. compression_opts.window_bits = ParseInt(value.substr(start, end - start));
  897. start = end + 1;
  898. end = value.find(':', start);
  899. if (end == std::string::npos) {
  900. return Status::InvalidArgument("unable to parse the specified CF option " +
  901. name);
  902. }
  903. compression_opts.level = ParseInt(value.substr(start, end - start));
  904. start = end + 1;
  905. if (start >= value.size()) {
  906. return Status::InvalidArgument("unable to parse the specified CF option " +
  907. name);
  908. }
  909. end = value.find(':', start);
  910. compression_opts.strategy =
  911. ParseInt(value.substr(start, value.size() - start));
  912. // max_dict_bytes is optional for backwards compatibility
  913. if (end != std::string::npos) {
  914. start = end + 1;
  915. if (start >= value.size()) {
  916. return Status::InvalidArgument(
  917. "unable to parse the specified CF option " + name);
  918. }
  919. compression_opts.max_dict_bytes =
  920. ParseInt(value.substr(start, value.size() - start));
  921. end = value.find(':', start);
  922. }
  923. // zstd_max_train_bytes is optional for backwards compatibility
  924. if (end != std::string::npos) {
  925. start = end + 1;
  926. if (start >= value.size()) {
  927. return Status::InvalidArgument(
  928. "unable to parse the specified CF option " + name);
  929. }
  930. compression_opts.zstd_max_train_bytes =
  931. ParseInt(value.substr(start, value.size() - start));
  932. end = value.find(':', start);
  933. }
  934. // enabled is optional for backwards compatibility
  935. if (end != std::string::npos) {
  936. start = end + 1;
  937. if (start >= value.size()) {
  938. return Status::InvalidArgument(
  939. "unable to parse the specified CF option " + name);
  940. }
  941. compression_opts.enabled =
  942. ParseBoolean("", value.substr(start, value.size() - start));
  943. }
  944. return Status::OK();
  945. }
  946. Status ParseColumnFamilyOption(const std::string& name,
  947. const std::string& org_value,
  948. ColumnFamilyOptions* new_options,
  949. bool input_strings_escaped = false) {
  950. const std::string& value =
  951. input_strings_escaped ? UnescapeOptionString(org_value) : org_value;
  952. try {
  953. if (name == "block_based_table_factory") {
  954. // Nested options
  955. BlockBasedTableOptions table_opt, base_table_options;
  956. BlockBasedTableFactory* block_based_table_factory =
  957. static_cast_with_check<BlockBasedTableFactory, TableFactory>(
  958. new_options->table_factory.get());
  959. if (block_based_table_factory != nullptr) {
  960. base_table_options = block_based_table_factory->table_options();
  961. }
  962. Status table_opt_s = GetBlockBasedTableOptionsFromString(
  963. base_table_options, value, &table_opt);
  964. if (!table_opt_s.ok()) {
  965. return Status::InvalidArgument(
  966. "unable to parse the specified CF option " + name);
  967. }
  968. new_options->table_factory.reset(NewBlockBasedTableFactory(table_opt));
  969. } else if (name == "plain_table_factory") {
  970. // Nested options
  971. PlainTableOptions table_opt, base_table_options;
  972. PlainTableFactory* plain_table_factory =
  973. static_cast_with_check<PlainTableFactory, TableFactory>(
  974. new_options->table_factory.get());
  975. if (plain_table_factory != nullptr) {
  976. base_table_options = plain_table_factory->table_options();
  977. }
  978. Status table_opt_s = GetPlainTableOptionsFromString(
  979. base_table_options, value, &table_opt);
  980. if (!table_opt_s.ok()) {
  981. return Status::InvalidArgument(
  982. "unable to parse the specified CF option " + name);
  983. }
  984. new_options->table_factory.reset(NewPlainTableFactory(table_opt));
  985. } else if (name == "memtable") {
  986. std::unique_ptr<MemTableRepFactory> new_mem_factory;
  987. Status mem_factory_s =
  988. GetMemTableRepFactoryFromString(value, &new_mem_factory);
  989. if (!mem_factory_s.ok()) {
  990. return Status::InvalidArgument(
  991. "unable to parse the specified CF option " + name);
  992. }
  993. new_options->memtable_factory.reset(new_mem_factory.release());
  994. } else if (name == "bottommost_compression_opts") {
  995. Status s = ParseCompressionOptions(
  996. value, name, new_options->bottommost_compression_opts);
  997. if (!s.ok()) {
  998. return s;
  999. }
  1000. } else if (name == "compression_opts") {
  1001. Status s =
  1002. ParseCompressionOptions(value, name, new_options->compression_opts);
  1003. if (!s.ok()) {
  1004. return s;
  1005. }
  1006. } else {
  1007. if (name == kNameComparator) {
  1008. // Try to get comparator from object registry first.
  1009. // Only support static comparator for now.
  1010. Status status = ObjectRegistry::NewInstance()->NewStaticObject(
  1011. value, &new_options->comparator);
  1012. if (status.ok()) {
  1013. return status;
  1014. }
  1015. } else if (name == kNameMergeOperator) {
  1016. // Try to get merge operator from object registry first.
  1017. std::shared_ptr<MergeOperator> mo;
  1018. Status status =
  1019. ObjectRegistry::NewInstance()->NewSharedObject<MergeOperator>(
  1020. value, &new_options->merge_operator);
  1021. // Only support static comparator for now.
  1022. if (status.ok()) {
  1023. return status;
  1024. }
  1025. }
  1026. auto iter = cf_options_type_info.find(name);
  1027. if (iter == cf_options_type_info.end()) {
  1028. return Status::InvalidArgument(
  1029. "Unable to parse the specified CF option " + name);
  1030. }
  1031. const auto& opt_info = iter->second;
  1032. if (opt_info.verification != OptionVerificationType::kDeprecated &&
  1033. ParseOptionHelper(
  1034. reinterpret_cast<char*>(new_options) + opt_info.offset,
  1035. opt_info.type, value)) {
  1036. return Status::OK();
  1037. }
  1038. switch (opt_info.verification) {
  1039. case OptionVerificationType::kByName:
  1040. case OptionVerificationType::kByNameAllowNull:
  1041. case OptionVerificationType::kByNameAllowFromNull:
  1042. return Status::NotSupported(
  1043. "Deserializing the specified CF option " + name +
  1044. " is not supported");
  1045. case OptionVerificationType::kDeprecated:
  1046. return Status::OK();
  1047. default:
  1048. return Status::InvalidArgument(
  1049. "Unable to parse the specified CF option " + name);
  1050. }
  1051. }
  1052. } catch (const std::exception&) {
  1053. return Status::InvalidArgument(
  1054. "unable to parse the specified option " + name);
  1055. }
  1056. return Status::OK();
  1057. }
  1058. template <typename T>
  1059. bool SerializeSingleStructOption(
  1060. std::string* opt_string, const T& options,
  1061. const std::unordered_map<std::string, OptionTypeInfo>& type_info,
  1062. const std::string& name, const std::string& delimiter) {
  1063. auto iter = type_info.find(name);
  1064. if (iter == type_info.end()) {
  1065. return false;
  1066. }
  1067. auto& opt_info = iter->second;
  1068. const char* opt_address =
  1069. reinterpret_cast<const char*>(&options) + opt_info.offset;
  1070. std::string value;
  1071. bool result = SerializeSingleOptionHelper(opt_address, opt_info.type, &value);
  1072. if (result) {
  1073. *opt_string = name + "=" + value + delimiter;
  1074. }
  1075. return result;
  1076. }
  1077. template <typename T>
  1078. Status GetStringFromStruct(
  1079. std::string* opt_string, const T& options,
  1080. const std::unordered_map<std::string, OptionTypeInfo>& type_info,
  1081. const std::string& delimiter) {
  1082. assert(opt_string);
  1083. opt_string->clear();
  1084. for (auto iter = type_info.begin(); iter != type_info.end(); ++iter) {
  1085. if (iter->second.verification == OptionVerificationType::kDeprecated) {
  1086. // If the option is no longer used in rocksdb and marked as deprecated,
  1087. // we skip it in the serialization.
  1088. continue;
  1089. }
  1090. std::string single_output;
  1091. bool result = SerializeSingleStructOption<T>(
  1092. &single_output, options, type_info, iter->first, delimiter);
  1093. if (result) {
  1094. opt_string->append(single_output);
  1095. } else {
  1096. return Status::InvalidArgument("failed to serialize %s\n",
  1097. iter->first.c_str());
  1098. }
  1099. assert(result);
  1100. }
  1101. return Status::OK();
  1102. }
  1103. Status GetStringFromDBOptions(std::string* opt_string,
  1104. const DBOptions& db_options,
  1105. const std::string& delimiter) {
  1106. return GetStringFromStruct<DBOptions>(opt_string, db_options,
  1107. db_options_type_info, delimiter);
  1108. }
  1109. Status GetStringFromColumnFamilyOptions(std::string* opt_string,
  1110. const ColumnFamilyOptions& cf_options,
  1111. const std::string& delimiter) {
  1112. return GetStringFromStruct<ColumnFamilyOptions>(
  1113. opt_string, cf_options, cf_options_type_info, delimiter);
  1114. }
  1115. Status GetStringFromCompressionType(std::string* compression_str,
  1116. CompressionType compression_type) {
  1117. bool ok = SerializeEnum<CompressionType>(compression_type_string_map,
  1118. compression_type, compression_str);
  1119. if (ok) {
  1120. return Status::OK();
  1121. } else {
  1122. return Status::InvalidArgument("Invalid compression types");
  1123. }
  1124. }
  1125. std::vector<CompressionType> GetSupportedCompressions() {
  1126. std::vector<CompressionType> supported_compressions;
  1127. for (const auto& comp_to_name : compression_type_string_map) {
  1128. CompressionType t = comp_to_name.second;
  1129. if (t != kDisableCompressionOption && CompressionTypeSupported(t)) {
  1130. supported_compressions.push_back(t);
  1131. }
  1132. }
  1133. return supported_compressions;
  1134. }
  1135. Status ParseDBOption(const std::string& name,
  1136. const std::string& org_value,
  1137. DBOptions* new_options,
  1138. bool input_strings_escaped = false) {
  1139. const std::string& value =
  1140. input_strings_escaped ? UnescapeOptionString(org_value) : org_value;
  1141. try {
  1142. if (name == "rate_limiter_bytes_per_sec") {
  1143. new_options->rate_limiter.reset(
  1144. NewGenericRateLimiter(static_cast<int64_t>(ParseUint64(value))));
  1145. } else if (name == kNameEnv) {
  1146. // Currently `Env` can be deserialized from object registry only.
  1147. Env* env = new_options->env;
  1148. Status status = Env::LoadEnv(value, &env);
  1149. // Only support static env for now.
  1150. if (status.ok()) {
  1151. new_options->env = env;
  1152. }
  1153. } else {
  1154. auto iter = db_options_type_info.find(name);
  1155. if (iter == db_options_type_info.end()) {
  1156. return Status::InvalidArgument("Unrecognized option DBOptions:", name);
  1157. }
  1158. const auto& opt_info = iter->second;
  1159. if (opt_info.verification != OptionVerificationType::kDeprecated &&
  1160. ParseOptionHelper(
  1161. reinterpret_cast<char*>(new_options) + opt_info.offset,
  1162. opt_info.type, value)) {
  1163. return Status::OK();
  1164. }
  1165. switch (opt_info.verification) {
  1166. case OptionVerificationType::kByName:
  1167. case OptionVerificationType::kByNameAllowNull:
  1168. return Status::NotSupported(
  1169. "Deserializing the specified DB option " + name +
  1170. " is not supported");
  1171. case OptionVerificationType::kDeprecated:
  1172. return Status::OK();
  1173. default:
  1174. return Status::InvalidArgument(
  1175. "Unable to parse the specified DB option " + name);
  1176. }
  1177. }
  1178. } catch (const std::exception&) {
  1179. return Status::InvalidArgument("Unable to parse DBOptions:", name);
  1180. }
  1181. return Status::OK();
  1182. }
  1183. Status GetColumnFamilyOptionsFromMap(
  1184. const ColumnFamilyOptions& base_options,
  1185. const std::unordered_map<std::string, std::string>& opts_map,
  1186. ColumnFamilyOptions* new_options, bool input_strings_escaped,
  1187. bool ignore_unknown_options) {
  1188. return GetColumnFamilyOptionsFromMapInternal(
  1189. base_options, opts_map, new_options, input_strings_escaped, nullptr,
  1190. ignore_unknown_options);
  1191. }
  1192. Status GetColumnFamilyOptionsFromMapInternal(
  1193. const ColumnFamilyOptions& base_options,
  1194. const std::unordered_map<std::string, std::string>& opts_map,
  1195. ColumnFamilyOptions* new_options, bool input_strings_escaped,
  1196. std::vector<std::string>* unsupported_options_names,
  1197. bool ignore_unknown_options) {
  1198. assert(new_options);
  1199. *new_options = base_options;
  1200. if (unsupported_options_names) {
  1201. unsupported_options_names->clear();
  1202. }
  1203. for (const auto& o : opts_map) {
  1204. auto s = ParseColumnFamilyOption(o.first, o.second, new_options,
  1205. input_strings_escaped);
  1206. if (!s.ok()) {
  1207. if (s.IsNotSupported()) {
  1208. // If the deserialization of the specified option is not supported
  1209. // and an output vector of unsupported_options is provided, then
  1210. // we log the name of the unsupported option and proceed.
  1211. if (unsupported_options_names != nullptr) {
  1212. unsupported_options_names->push_back(o.first);
  1213. }
  1214. // Note that we still return Status::OK in such case to maintain
  1215. // the backward compatibility in the old public API defined in
  1216. // rocksdb/convenience.h
  1217. } else if (s.IsInvalidArgument() && ignore_unknown_options) {
  1218. continue;
  1219. } else {
  1220. // Restore "new_options" to the default "base_options".
  1221. *new_options = base_options;
  1222. return s;
  1223. }
  1224. }
  1225. }
  1226. return Status::OK();
  1227. }
  1228. Status GetColumnFamilyOptionsFromString(
  1229. const ColumnFamilyOptions& base_options,
  1230. const std::string& opts_str,
  1231. ColumnFamilyOptions* new_options) {
  1232. std::unordered_map<std::string, std::string> opts_map;
  1233. Status s = StringToMap(opts_str, &opts_map);
  1234. if (!s.ok()) {
  1235. *new_options = base_options;
  1236. return s;
  1237. }
  1238. return GetColumnFamilyOptionsFromMap(base_options, opts_map, new_options);
  1239. }
  1240. Status GetDBOptionsFromMap(
  1241. const DBOptions& base_options,
  1242. const std::unordered_map<std::string, std::string>& opts_map,
  1243. DBOptions* new_options, bool input_strings_escaped,
  1244. bool ignore_unknown_options) {
  1245. return GetDBOptionsFromMapInternal(base_options, opts_map, new_options,
  1246. input_strings_escaped, nullptr,
  1247. ignore_unknown_options);
  1248. }
  1249. Status GetDBOptionsFromMapInternal(
  1250. const DBOptions& base_options,
  1251. const std::unordered_map<std::string, std::string>& opts_map,
  1252. DBOptions* new_options, bool input_strings_escaped,
  1253. std::vector<std::string>* unsupported_options_names,
  1254. bool ignore_unknown_options) {
  1255. assert(new_options);
  1256. *new_options = base_options;
  1257. if (unsupported_options_names) {
  1258. unsupported_options_names->clear();
  1259. }
  1260. for (const auto& o : opts_map) {
  1261. auto s = ParseDBOption(o.first, o.second,
  1262. new_options, input_strings_escaped);
  1263. if (!s.ok()) {
  1264. if (s.IsNotSupported()) {
  1265. // If the deserialization of the specified option is not supported
  1266. // and an output vector of unsupported_options is provided, then
  1267. // we log the name of the unsupported option and proceed.
  1268. if (unsupported_options_names != nullptr) {
  1269. unsupported_options_names->push_back(o.first);
  1270. }
  1271. // Note that we still return Status::OK in such case to maintain
  1272. // the backward compatibility in the old public API defined in
  1273. // rocksdb/convenience.h
  1274. } else if (s.IsInvalidArgument() && ignore_unknown_options) {
  1275. continue;
  1276. } else {
  1277. // Restore "new_options" to the default "base_options".
  1278. *new_options = base_options;
  1279. return s;
  1280. }
  1281. }
  1282. }
  1283. return Status::OK();
  1284. }
  1285. Status GetDBOptionsFromString(
  1286. const DBOptions& base_options,
  1287. const std::string& opts_str,
  1288. DBOptions* new_options) {
  1289. std::unordered_map<std::string, std::string> opts_map;
  1290. Status s = StringToMap(opts_str, &opts_map);
  1291. if (!s.ok()) {
  1292. *new_options = base_options;
  1293. return s;
  1294. }
  1295. return GetDBOptionsFromMap(base_options, opts_map, new_options);
  1296. }
  1297. Status GetOptionsFromString(const Options& base_options,
  1298. const std::string& opts_str, Options* new_options) {
  1299. std::unordered_map<std::string, std::string> opts_map;
  1300. Status s = StringToMap(opts_str, &opts_map);
  1301. if (!s.ok()) {
  1302. return s;
  1303. }
  1304. DBOptions new_db_options(base_options);
  1305. ColumnFamilyOptions new_cf_options(base_options);
  1306. for (const auto& o : opts_map) {
  1307. if (ParseDBOption(o.first, o.second, &new_db_options).ok()) {
  1308. } else if (ParseColumnFamilyOption(
  1309. o.first, o.second, &new_cf_options).ok()) {
  1310. } else {
  1311. return Status::InvalidArgument("Can't parse option " + o.first);
  1312. }
  1313. }
  1314. *new_options = Options(new_db_options, new_cf_options);
  1315. return Status::OK();
  1316. }
  1317. Status GetTableFactoryFromMap(
  1318. const std::string& factory_name,
  1319. const std::unordered_map<std::string, std::string>& opt_map,
  1320. std::shared_ptr<TableFactory>* table_factory, bool ignore_unknown_options) {
  1321. Status s;
  1322. if (factory_name == BlockBasedTableFactory().Name()) {
  1323. BlockBasedTableOptions bbt_opt;
  1324. s = GetBlockBasedTableOptionsFromMap(BlockBasedTableOptions(), opt_map,
  1325. &bbt_opt,
  1326. true, /* input_strings_escaped */
  1327. ignore_unknown_options);
  1328. if (!s.ok()) {
  1329. return s;
  1330. }
  1331. table_factory->reset(new BlockBasedTableFactory(bbt_opt));
  1332. return Status::OK();
  1333. } else if (factory_name == PlainTableFactory().Name()) {
  1334. PlainTableOptions pt_opt;
  1335. s = GetPlainTableOptionsFromMap(PlainTableOptions(), opt_map, &pt_opt,
  1336. true, /* input_strings_escaped */
  1337. ignore_unknown_options);
  1338. if (!s.ok()) {
  1339. return s;
  1340. }
  1341. table_factory->reset(new PlainTableFactory(pt_opt));
  1342. return Status::OK();
  1343. }
  1344. // Return OK for not supported table factories as TableFactory
  1345. // Deserialization is optional.
  1346. table_factory->reset();
  1347. return Status::OK();
  1348. }
  1349. std::unordered_map<std::string, OptionTypeInfo>
  1350. OptionsHelper::db_options_type_info = {
  1351. /*
  1352. // not yet supported
  1353. std::shared_ptr<Cache> row_cache;
  1354. std::shared_ptr<DeleteScheduler> delete_scheduler;
  1355. std::shared_ptr<Logger> info_log;
  1356. std::shared_ptr<RateLimiter> rate_limiter;
  1357. std::shared_ptr<Statistics> statistics;
  1358. std::vector<DbPath> db_paths;
  1359. std::vector<std::shared_ptr<EventListener>> listeners;
  1360. */
  1361. {"advise_random_on_open",
  1362. {offsetof(struct DBOptions, advise_random_on_open),
  1363. OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}},
  1364. {"allow_mmap_reads",
  1365. {offsetof(struct DBOptions, allow_mmap_reads), OptionType::kBoolean,
  1366. OptionVerificationType::kNormal, false, 0}},
  1367. {"allow_fallocate",
  1368. {offsetof(struct DBOptions, allow_fallocate), OptionType::kBoolean,
  1369. OptionVerificationType::kNormal, false, 0}},
  1370. {"allow_mmap_writes",
  1371. {offsetof(struct DBOptions, allow_mmap_writes), OptionType::kBoolean,
  1372. OptionVerificationType::kNormal, false, 0}},
  1373. {"use_direct_reads",
  1374. {offsetof(struct DBOptions, use_direct_reads), OptionType::kBoolean,
  1375. OptionVerificationType::kNormal, false, 0}},
  1376. {"use_direct_writes",
  1377. {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, false,
  1378. 0}},
  1379. {"use_direct_io_for_flush_and_compaction",
  1380. {offsetof(struct DBOptions, use_direct_io_for_flush_and_compaction),
  1381. OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}},
  1382. {"allow_2pc",
  1383. {offsetof(struct DBOptions, allow_2pc), OptionType::kBoolean,
  1384. OptionVerificationType::kNormal, false, 0}},
  1385. {"allow_os_buffer",
  1386. {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, true,
  1387. 0}},
  1388. {"create_if_missing",
  1389. {offsetof(struct DBOptions, create_if_missing), OptionType::kBoolean,
  1390. OptionVerificationType::kNormal, false, 0}},
  1391. {"create_missing_column_families",
  1392. {offsetof(struct DBOptions, create_missing_column_families),
  1393. OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}},
  1394. {"disableDataSync",
  1395. {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, false,
  1396. 0}},
  1397. {"disable_data_sync", // for compatibility
  1398. {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, false,
  1399. 0}},
  1400. {"enable_thread_tracking",
  1401. {offsetof(struct DBOptions, enable_thread_tracking),
  1402. OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}},
  1403. {"error_if_exists",
  1404. {offsetof(struct DBOptions, error_if_exists), OptionType::kBoolean,
  1405. OptionVerificationType::kNormal, false, 0}},
  1406. {"is_fd_close_on_exec",
  1407. {offsetof(struct DBOptions, is_fd_close_on_exec), OptionType::kBoolean,
  1408. OptionVerificationType::kNormal, false, 0}},
  1409. {"paranoid_checks",
  1410. {offsetof(struct DBOptions, paranoid_checks), OptionType::kBoolean,
  1411. OptionVerificationType::kNormal, false, 0}},
  1412. {"skip_log_error_on_recovery",
  1413. {offsetof(struct DBOptions, skip_log_error_on_recovery),
  1414. OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}},
  1415. {"skip_stats_update_on_db_open",
  1416. {offsetof(struct DBOptions, skip_stats_update_on_db_open),
  1417. OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}},
  1418. {"skip_checking_sst_file_sizes_on_db_open",
  1419. {offsetof(struct DBOptions, skip_checking_sst_file_sizes_on_db_open),
  1420. OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}},
  1421. {"new_table_reader_for_compaction_inputs",
  1422. {offsetof(struct DBOptions, new_table_reader_for_compaction_inputs),
  1423. OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}},
  1424. {"compaction_readahead_size",
  1425. {offsetof(struct DBOptions, compaction_readahead_size),
  1426. OptionType::kSizeT, OptionVerificationType::kNormal, true,
  1427. offsetof(struct MutableDBOptions, compaction_readahead_size)}},
  1428. {"random_access_max_buffer_size",
  1429. {offsetof(struct DBOptions, random_access_max_buffer_size),
  1430. OptionType::kSizeT, OptionVerificationType::kNormal, false, 0}},
  1431. {"use_adaptive_mutex",
  1432. {offsetof(struct DBOptions, use_adaptive_mutex), OptionType::kBoolean,
  1433. OptionVerificationType::kNormal, false, 0}},
  1434. {"use_fsync",
  1435. {offsetof(struct DBOptions, use_fsync), OptionType::kBoolean,
  1436. OptionVerificationType::kNormal, false, 0}},
  1437. {"max_background_jobs",
  1438. {offsetof(struct DBOptions, max_background_jobs), OptionType::kInt,
  1439. OptionVerificationType::kNormal, true,
  1440. offsetof(struct MutableDBOptions, max_background_jobs)}},
  1441. {"max_background_compactions",
  1442. {offsetof(struct DBOptions, max_background_compactions),
  1443. OptionType::kInt, OptionVerificationType::kNormal, true,
  1444. offsetof(struct MutableDBOptions, max_background_compactions)}},
  1445. {"base_background_compactions",
  1446. {offsetof(struct DBOptions, base_background_compactions),
  1447. OptionType::kInt, OptionVerificationType::kNormal, true,
  1448. offsetof(struct MutableDBOptions, base_background_compactions)}},
  1449. {"max_background_flushes",
  1450. {offsetof(struct DBOptions, max_background_flushes), OptionType::kInt,
  1451. OptionVerificationType::kNormal, false, 0}},
  1452. {"max_file_opening_threads",
  1453. {offsetof(struct DBOptions, max_file_opening_threads),
  1454. OptionType::kInt, OptionVerificationType::kNormal, false, 0}},
  1455. {"max_open_files",
  1456. {offsetof(struct DBOptions, max_open_files), OptionType::kInt,
  1457. OptionVerificationType::kNormal, true,
  1458. offsetof(struct MutableDBOptions, max_open_files)}},
  1459. {"table_cache_numshardbits",
  1460. {offsetof(struct DBOptions, table_cache_numshardbits),
  1461. OptionType::kInt, OptionVerificationType::kNormal, false, 0}},
  1462. {"db_write_buffer_size",
  1463. {offsetof(struct DBOptions, db_write_buffer_size), OptionType::kSizeT,
  1464. OptionVerificationType::kNormal, false, 0}},
  1465. {"keep_log_file_num",
  1466. {offsetof(struct DBOptions, keep_log_file_num), OptionType::kSizeT,
  1467. OptionVerificationType::kNormal, false, 0}},
  1468. {"recycle_log_file_num",
  1469. {offsetof(struct DBOptions, recycle_log_file_num), OptionType::kSizeT,
  1470. OptionVerificationType::kNormal, false, 0}},
  1471. {"log_file_time_to_roll",
  1472. {offsetof(struct DBOptions, log_file_time_to_roll), OptionType::kSizeT,
  1473. OptionVerificationType::kNormal, false, 0}},
  1474. {"manifest_preallocation_size",
  1475. {offsetof(struct DBOptions, manifest_preallocation_size),
  1476. OptionType::kSizeT, OptionVerificationType::kNormal, false, 0}},
  1477. {"max_log_file_size",
  1478. {offsetof(struct DBOptions, max_log_file_size), OptionType::kSizeT,
  1479. OptionVerificationType::kNormal, false, 0}},
  1480. {"db_log_dir",
  1481. {offsetof(struct DBOptions, db_log_dir), OptionType::kString,
  1482. OptionVerificationType::kNormal, false, 0}},
  1483. {"wal_dir",
  1484. {offsetof(struct DBOptions, wal_dir), OptionType::kString,
  1485. OptionVerificationType::kNormal, false, 0}},
  1486. {"max_subcompactions",
  1487. {offsetof(struct DBOptions, max_subcompactions), OptionType::kUInt32T,
  1488. OptionVerificationType::kNormal, false, 0}},
  1489. {"WAL_size_limit_MB",
  1490. {offsetof(struct DBOptions, WAL_size_limit_MB), OptionType::kUInt64T,
  1491. OptionVerificationType::kNormal, false, 0}},
  1492. {"WAL_ttl_seconds",
  1493. {offsetof(struct DBOptions, WAL_ttl_seconds), OptionType::kUInt64T,
  1494. OptionVerificationType::kNormal, false, 0}},
  1495. {"bytes_per_sync",
  1496. {offsetof(struct DBOptions, bytes_per_sync), OptionType::kUInt64T,
  1497. OptionVerificationType::kNormal, true,
  1498. offsetof(struct MutableDBOptions, bytes_per_sync)}},
  1499. {"delayed_write_rate",
  1500. {offsetof(struct DBOptions, delayed_write_rate), OptionType::kUInt64T,
  1501. OptionVerificationType::kNormal, true,
  1502. offsetof(struct MutableDBOptions, delayed_write_rate)}},
  1503. {"delete_obsolete_files_period_micros",
  1504. {offsetof(struct DBOptions, delete_obsolete_files_period_micros),
  1505. OptionType::kUInt64T, OptionVerificationType::kNormal, true,
  1506. offsetof(struct MutableDBOptions,
  1507. delete_obsolete_files_period_micros)}},
  1508. {"max_manifest_file_size",
  1509. {offsetof(struct DBOptions, max_manifest_file_size),
  1510. OptionType::kUInt64T, OptionVerificationType::kNormal, false, 0}},
  1511. {"max_total_wal_size",
  1512. {offsetof(struct DBOptions, max_total_wal_size), OptionType::kUInt64T,
  1513. OptionVerificationType::kNormal, true,
  1514. offsetof(struct MutableDBOptions, max_total_wal_size)}},
  1515. {"wal_bytes_per_sync",
  1516. {offsetof(struct DBOptions, wal_bytes_per_sync), OptionType::kUInt64T,
  1517. OptionVerificationType::kNormal, true,
  1518. offsetof(struct MutableDBOptions, wal_bytes_per_sync)}},
  1519. {"strict_bytes_per_sync",
  1520. {offsetof(struct DBOptions, strict_bytes_per_sync),
  1521. OptionType::kBoolean, OptionVerificationType::kNormal, true,
  1522. offsetof(struct MutableDBOptions, strict_bytes_per_sync)}},
  1523. {"stats_dump_period_sec",
  1524. {offsetof(struct DBOptions, stats_dump_period_sec), OptionType::kUInt,
  1525. OptionVerificationType::kNormal, true,
  1526. offsetof(struct MutableDBOptions, stats_dump_period_sec)}},
  1527. {"stats_persist_period_sec",
  1528. {offsetof(struct DBOptions, stats_persist_period_sec),
  1529. OptionType::kUInt, OptionVerificationType::kNormal, true,
  1530. offsetof(struct MutableDBOptions, stats_persist_period_sec)}},
  1531. {"persist_stats_to_disk",
  1532. {offsetof(struct DBOptions, persist_stats_to_disk),
  1533. OptionType::kBoolean, OptionVerificationType::kNormal, false,
  1534. offsetof(struct ImmutableDBOptions, persist_stats_to_disk)}},
  1535. {"stats_history_buffer_size",
  1536. {offsetof(struct DBOptions, stats_history_buffer_size),
  1537. OptionType::kSizeT, OptionVerificationType::kNormal, true,
  1538. offsetof(struct MutableDBOptions, stats_history_buffer_size)}},
  1539. {"fail_if_options_file_error",
  1540. {offsetof(struct DBOptions, fail_if_options_file_error),
  1541. OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}},
  1542. {"enable_pipelined_write",
  1543. {offsetof(struct DBOptions, enable_pipelined_write),
  1544. OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}},
  1545. {"unordered_write",
  1546. {offsetof(struct DBOptions, unordered_write), OptionType::kBoolean,
  1547. OptionVerificationType::kNormal, false, 0}},
  1548. {"allow_concurrent_memtable_write",
  1549. {offsetof(struct DBOptions, allow_concurrent_memtable_write),
  1550. OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}},
  1551. {"wal_recovery_mode",
  1552. {offsetof(struct DBOptions, wal_recovery_mode),
  1553. OptionType::kWALRecoveryMode, OptionVerificationType::kNormal, false,
  1554. 0}},
  1555. {"enable_write_thread_adaptive_yield",
  1556. {offsetof(struct DBOptions, enable_write_thread_adaptive_yield),
  1557. OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}},
  1558. {"write_thread_slow_yield_usec",
  1559. {offsetof(struct DBOptions, write_thread_slow_yield_usec),
  1560. OptionType::kUInt64T, OptionVerificationType::kNormal, false, 0}},
  1561. {"max_write_batch_group_size_bytes",
  1562. {offsetof(struct DBOptions, max_write_batch_group_size_bytes),
  1563. OptionType::kUInt64T, OptionVerificationType::kNormal, false, 0}},
  1564. {"write_thread_max_yield_usec",
  1565. {offsetof(struct DBOptions, write_thread_max_yield_usec),
  1566. OptionType::kUInt64T, OptionVerificationType::kNormal, false, 0}},
  1567. {"access_hint_on_compaction_start",
  1568. {offsetof(struct DBOptions, access_hint_on_compaction_start),
  1569. OptionType::kAccessHint, OptionVerificationType::kNormal, false, 0}},
  1570. {"info_log_level",
  1571. {offsetof(struct DBOptions, info_log_level), OptionType::kInfoLogLevel,
  1572. OptionVerificationType::kNormal, false, 0}},
  1573. {"dump_malloc_stats",
  1574. {offsetof(struct DBOptions, dump_malloc_stats), OptionType::kBoolean,
  1575. OptionVerificationType::kNormal, false, 0}},
  1576. {"avoid_flush_during_recovery",
  1577. {offsetof(struct DBOptions, avoid_flush_during_recovery),
  1578. OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}},
  1579. {"avoid_flush_during_shutdown",
  1580. {offsetof(struct DBOptions, avoid_flush_during_shutdown),
  1581. OptionType::kBoolean, OptionVerificationType::kNormal, true,
  1582. offsetof(struct MutableDBOptions, avoid_flush_during_shutdown)}},
  1583. {"writable_file_max_buffer_size",
  1584. {offsetof(struct DBOptions, writable_file_max_buffer_size),
  1585. OptionType::kSizeT, OptionVerificationType::kNormal, true,
  1586. offsetof(struct MutableDBOptions, writable_file_max_buffer_size)}},
  1587. {"allow_ingest_behind",
  1588. {offsetof(struct DBOptions, allow_ingest_behind), OptionType::kBoolean,
  1589. OptionVerificationType::kNormal, false,
  1590. offsetof(struct ImmutableDBOptions, allow_ingest_behind)}},
  1591. {"preserve_deletes",
  1592. {offsetof(struct DBOptions, preserve_deletes), OptionType::kBoolean,
  1593. OptionVerificationType::kNormal, false,
  1594. offsetof(struct ImmutableDBOptions, preserve_deletes)}},
  1595. {"concurrent_prepare", // Deprecated by two_write_queues
  1596. {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, false,
  1597. 0}},
  1598. {"two_write_queues",
  1599. {offsetof(struct DBOptions, two_write_queues), OptionType::kBoolean,
  1600. OptionVerificationType::kNormal, false,
  1601. offsetof(struct ImmutableDBOptions, two_write_queues)}},
  1602. {"manual_wal_flush",
  1603. {offsetof(struct DBOptions, manual_wal_flush), OptionType::kBoolean,
  1604. OptionVerificationType::kNormal, false,
  1605. offsetof(struct ImmutableDBOptions, manual_wal_flush)}},
  1606. {"seq_per_batch",
  1607. {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, false,
  1608. 0}},
  1609. {"atomic_flush",
  1610. {offsetof(struct DBOptions, atomic_flush), OptionType::kBoolean,
  1611. OptionVerificationType::kNormal, false,
  1612. offsetof(struct ImmutableDBOptions, atomic_flush)}},
  1613. {"avoid_unnecessary_blocking_io",
  1614. {offsetof(struct DBOptions, avoid_unnecessary_blocking_io),
  1615. OptionType::kBoolean, OptionVerificationType::kNormal, false,
  1616. offsetof(struct ImmutableDBOptions, avoid_unnecessary_blocking_io)}},
  1617. {"write_dbid_to_manifest",
  1618. {offsetof(struct DBOptions, write_dbid_to_manifest),
  1619. OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}},
  1620. {"log_readahead_size",
  1621. {offsetof(struct DBOptions, log_readahead_size), OptionType::kSizeT,
  1622. OptionVerificationType::kNormal, false, 0}},
  1623. };
  1624. std::unordered_map<std::string, BlockBasedTableOptions::IndexType>
  1625. OptionsHelper::block_base_table_index_type_string_map = {
  1626. {"kBinarySearch", BlockBasedTableOptions::IndexType::kBinarySearch},
  1627. {"kHashSearch", BlockBasedTableOptions::IndexType::kHashSearch},
  1628. {"kTwoLevelIndexSearch",
  1629. BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch},
  1630. {"kBinarySearchWithFirstKey",
  1631. BlockBasedTableOptions::IndexType::kBinarySearchWithFirstKey}};
  1632. std::unordered_map<std::string, BlockBasedTableOptions::DataBlockIndexType>
  1633. OptionsHelper::block_base_table_data_block_index_type_string_map = {
  1634. {"kDataBlockBinarySearch",
  1635. BlockBasedTableOptions::DataBlockIndexType::kDataBlockBinarySearch},
  1636. {"kDataBlockBinaryAndHash",
  1637. BlockBasedTableOptions::DataBlockIndexType::kDataBlockBinaryAndHash}};
  1638. std::unordered_map<std::string, BlockBasedTableOptions::IndexShorteningMode>
  1639. OptionsHelper::block_base_table_index_shortening_mode_string_map = {
  1640. {"kNoShortening",
  1641. BlockBasedTableOptions::IndexShorteningMode::kNoShortening},
  1642. {"kShortenSeparators",
  1643. BlockBasedTableOptions::IndexShorteningMode::kShortenSeparators},
  1644. {"kShortenSeparatorsAndSuccessor",
  1645. BlockBasedTableOptions::IndexShorteningMode::
  1646. kShortenSeparatorsAndSuccessor}};
  1647. std::unordered_map<std::string, EncodingType>
  1648. OptionsHelper::encoding_type_string_map = {{"kPlain", kPlain},
  1649. {"kPrefix", kPrefix}};
  1650. std::unordered_map<std::string, CompactionStyle>
  1651. OptionsHelper::compaction_style_string_map = {
  1652. {"kCompactionStyleLevel", kCompactionStyleLevel},
  1653. {"kCompactionStyleUniversal", kCompactionStyleUniversal},
  1654. {"kCompactionStyleFIFO", kCompactionStyleFIFO},
  1655. {"kCompactionStyleNone", kCompactionStyleNone}};
  1656. std::unordered_map<std::string, CompactionPri>
  1657. OptionsHelper::compaction_pri_string_map = {
  1658. {"kByCompensatedSize", kByCompensatedSize},
  1659. {"kOldestLargestSeqFirst", kOldestLargestSeqFirst},
  1660. {"kOldestSmallestSeqFirst", kOldestSmallestSeqFirst},
  1661. {"kMinOverlappingRatio", kMinOverlappingRatio}};
  1662. std::unordered_map<std::string, WALRecoveryMode>
  1663. OptionsHelper::wal_recovery_mode_string_map = {
  1664. {"kTolerateCorruptedTailRecords",
  1665. WALRecoveryMode::kTolerateCorruptedTailRecords},
  1666. {"kAbsoluteConsistency", WALRecoveryMode::kAbsoluteConsistency},
  1667. {"kPointInTimeRecovery", WALRecoveryMode::kPointInTimeRecovery},
  1668. {"kSkipAnyCorruptedRecords",
  1669. WALRecoveryMode::kSkipAnyCorruptedRecords}};
  1670. std::unordered_map<std::string, DBOptions::AccessHint>
  1671. OptionsHelper::access_hint_string_map = {
  1672. {"NONE", DBOptions::AccessHint::NONE},
  1673. {"NORMAL", DBOptions::AccessHint::NORMAL},
  1674. {"SEQUENTIAL", DBOptions::AccessHint::SEQUENTIAL},
  1675. {"WILLNEED", DBOptions::AccessHint::WILLNEED}};
  1676. std::unordered_map<std::string, InfoLogLevel>
  1677. OptionsHelper::info_log_level_string_map = {
  1678. {"DEBUG_LEVEL", InfoLogLevel::DEBUG_LEVEL},
  1679. {"INFO_LEVEL", InfoLogLevel::INFO_LEVEL},
  1680. {"WARN_LEVEL", InfoLogLevel::WARN_LEVEL},
  1681. {"ERROR_LEVEL", InfoLogLevel::ERROR_LEVEL},
  1682. {"FATAL_LEVEL", InfoLogLevel::FATAL_LEVEL},
  1683. {"HEADER_LEVEL", InfoLogLevel::HEADER_LEVEL}};
  1684. ColumnFamilyOptions OptionsHelper::dummy_cf_options;
  1685. CompactionOptionsFIFO OptionsHelper::dummy_comp_options;
  1686. LRUCacheOptions OptionsHelper::dummy_lru_cache_options;
  1687. CompactionOptionsUniversal OptionsHelper::dummy_comp_options_universal;
  1688. // offset_of is used to get the offset of a class data member
  1689. // ex: offset_of(&ColumnFamilyOptions::num_levels)
  1690. // This call will return the offset of num_levels in ColumnFamilyOptions class
  1691. //
  1692. // This is the same as offsetof() but allow us to work with non standard-layout
  1693. // classes and structures
  1694. // refs:
  1695. // http://en.cppreference.com/w/cpp/concept/StandardLayoutType
  1696. // https://gist.github.com/graphitemaster/494f21190bb2c63c5516
  1697. template <typename T1>
  1698. int offset_of(T1 ColumnFamilyOptions::*member) {
  1699. return int(size_t(&(OptionsHelper::dummy_cf_options.*member)) -
  1700. size_t(&OptionsHelper::dummy_cf_options));
  1701. }
  1702. template <typename T1>
  1703. int offset_of(T1 AdvancedColumnFamilyOptions::*member) {
  1704. return int(size_t(&(OptionsHelper::dummy_cf_options.*member)) -
  1705. size_t(&OptionsHelper::dummy_cf_options));
  1706. }
  1707. template <typename T1>
  1708. int offset_of(T1 CompactionOptionsFIFO::*member) {
  1709. return int(size_t(&(OptionsHelper::dummy_comp_options.*member)) -
  1710. size_t(&OptionsHelper::dummy_comp_options));
  1711. }
  1712. template <typename T1>
  1713. int offset_of(T1 LRUCacheOptions::*member) {
  1714. return int(size_t(&(OptionsHelper::dummy_lru_cache_options.*member)) -
  1715. size_t(&OptionsHelper::dummy_lru_cache_options));
  1716. }
  1717. template <typename T1>
  1718. int offset_of(T1 CompactionOptionsUniversal::*member) {
  1719. return int(size_t(&(OptionsHelper::dummy_comp_options_universal.*member)) -
  1720. size_t(&OptionsHelper::dummy_comp_options_universal));
  1721. }
  1722. std::unordered_map<std::string, OptionTypeInfo>
  1723. OptionsHelper::cf_options_type_info = {
  1724. /* not yet supported
  1725. CompressionOptions compression_opts;
  1726. TablePropertiesCollectorFactories table_properties_collector_factories;
  1727. typedef std::vector<std::shared_ptr<TablePropertiesCollectorFactory>>
  1728. TablePropertiesCollectorFactories;
  1729. UpdateStatus (*inplace_callback)(char* existing_value,
  1730. uint34_t* existing_value_size,
  1731. Slice delta_value,
  1732. std::string* merged_value);
  1733. std::vector<DbPath> cf_paths;
  1734. */
  1735. {"report_bg_io_stats",
  1736. {offset_of(&ColumnFamilyOptions::report_bg_io_stats),
  1737. OptionType::kBoolean, OptionVerificationType::kNormal, true,
  1738. offsetof(struct MutableCFOptions, report_bg_io_stats)}},
  1739. {"compaction_measure_io_stats",
  1740. {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, false,
  1741. 0}},
  1742. {"disable_auto_compactions",
  1743. {offset_of(&ColumnFamilyOptions::disable_auto_compactions),
  1744. OptionType::kBoolean, OptionVerificationType::kNormal, true,
  1745. offsetof(struct MutableCFOptions, disable_auto_compactions)}},
  1746. {"filter_deletes",
  1747. {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, true,
  1748. 0}},
  1749. {"inplace_update_support",
  1750. {offset_of(&ColumnFamilyOptions::inplace_update_support),
  1751. OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}},
  1752. {"level_compaction_dynamic_level_bytes",
  1753. {offset_of(&ColumnFamilyOptions::level_compaction_dynamic_level_bytes),
  1754. OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}},
  1755. {"optimize_filters_for_hits",
  1756. {offset_of(&ColumnFamilyOptions::optimize_filters_for_hits),
  1757. OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}},
  1758. {"paranoid_file_checks",
  1759. {offset_of(&ColumnFamilyOptions::paranoid_file_checks),
  1760. OptionType::kBoolean, OptionVerificationType::kNormal, true,
  1761. offsetof(struct MutableCFOptions, paranoid_file_checks)}},
  1762. {"force_consistency_checks",
  1763. {offset_of(&ColumnFamilyOptions::force_consistency_checks),
  1764. OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}},
  1765. {"purge_redundant_kvs_while_flush",
  1766. {offset_of(&ColumnFamilyOptions::purge_redundant_kvs_while_flush),
  1767. OptionType::kBoolean, OptionVerificationType::kDeprecated, false, 0}},
  1768. {"verify_checksums_in_compaction",
  1769. {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, true,
  1770. 0}},
  1771. {"soft_pending_compaction_bytes_limit",
  1772. {offset_of(&ColumnFamilyOptions::soft_pending_compaction_bytes_limit),
  1773. OptionType::kUInt64T, OptionVerificationType::kNormal, true,
  1774. offsetof(struct MutableCFOptions,
  1775. soft_pending_compaction_bytes_limit)}},
  1776. {"hard_pending_compaction_bytes_limit",
  1777. {offset_of(&ColumnFamilyOptions::hard_pending_compaction_bytes_limit),
  1778. OptionType::kUInt64T, OptionVerificationType::kNormal, true,
  1779. offsetof(struct MutableCFOptions,
  1780. hard_pending_compaction_bytes_limit)}},
  1781. {"hard_rate_limit",
  1782. {0, OptionType::kDouble, OptionVerificationType::kDeprecated, true,
  1783. 0}},
  1784. {"soft_rate_limit",
  1785. {0, OptionType::kDouble, OptionVerificationType::kDeprecated, true,
  1786. 0}},
  1787. {"max_compaction_bytes",
  1788. {offset_of(&ColumnFamilyOptions::max_compaction_bytes),
  1789. OptionType::kUInt64T, OptionVerificationType::kNormal, true,
  1790. offsetof(struct MutableCFOptions, max_compaction_bytes)}},
  1791. {"expanded_compaction_factor",
  1792. {0, OptionType::kInt, OptionVerificationType::kDeprecated, true, 0}},
  1793. {"level0_file_num_compaction_trigger",
  1794. {offset_of(&ColumnFamilyOptions::level0_file_num_compaction_trigger),
  1795. OptionType::kInt, OptionVerificationType::kNormal, true,
  1796. offsetof(struct MutableCFOptions,
  1797. level0_file_num_compaction_trigger)}},
  1798. {"level0_slowdown_writes_trigger",
  1799. {offset_of(&ColumnFamilyOptions::level0_slowdown_writes_trigger),
  1800. OptionType::kInt, OptionVerificationType::kNormal, true,
  1801. offsetof(struct MutableCFOptions, level0_slowdown_writes_trigger)}},
  1802. {"level0_stop_writes_trigger",
  1803. {offset_of(&ColumnFamilyOptions::level0_stop_writes_trigger),
  1804. OptionType::kInt, OptionVerificationType::kNormal, true,
  1805. offsetof(struct MutableCFOptions, level0_stop_writes_trigger)}},
  1806. {"max_grandparent_overlap_factor",
  1807. {0, OptionType::kInt, OptionVerificationType::kDeprecated, true, 0}},
  1808. {"max_mem_compaction_level",
  1809. {0, OptionType::kInt, OptionVerificationType::kDeprecated, false, 0}},
  1810. {"max_write_buffer_number",
  1811. {offset_of(&ColumnFamilyOptions::max_write_buffer_number),
  1812. OptionType::kInt, OptionVerificationType::kNormal, true,
  1813. offsetof(struct MutableCFOptions, max_write_buffer_number)}},
  1814. {"max_write_buffer_number_to_maintain",
  1815. {offset_of(&ColumnFamilyOptions::max_write_buffer_number_to_maintain),
  1816. OptionType::kInt, OptionVerificationType::kNormal, false, 0}},
  1817. {"max_write_buffer_size_to_maintain",
  1818. {offset_of(&ColumnFamilyOptions::max_write_buffer_size_to_maintain),
  1819. OptionType::kInt64T, OptionVerificationType::kNormal, false, 0}},
  1820. {"min_write_buffer_number_to_merge",
  1821. {offset_of(&ColumnFamilyOptions::min_write_buffer_number_to_merge),
  1822. OptionType::kInt, OptionVerificationType::kNormal, false, 0}},
  1823. {"num_levels",
  1824. {offset_of(&ColumnFamilyOptions::num_levels), OptionType::kInt,
  1825. OptionVerificationType::kNormal, false, 0}},
  1826. {"source_compaction_factor",
  1827. {0, OptionType::kInt, OptionVerificationType::kDeprecated, true, 0}},
  1828. {"target_file_size_multiplier",
  1829. {offset_of(&ColumnFamilyOptions::target_file_size_multiplier),
  1830. OptionType::kInt, OptionVerificationType::kNormal, true,
  1831. offsetof(struct MutableCFOptions, target_file_size_multiplier)}},
  1832. {"arena_block_size",
  1833. {offset_of(&ColumnFamilyOptions::arena_block_size), OptionType::kSizeT,
  1834. OptionVerificationType::kNormal, true,
  1835. offsetof(struct MutableCFOptions, arena_block_size)}},
  1836. {"inplace_update_num_locks",
  1837. {offset_of(&ColumnFamilyOptions::inplace_update_num_locks),
  1838. OptionType::kSizeT, OptionVerificationType::kNormal, true,
  1839. offsetof(struct MutableCFOptions, inplace_update_num_locks)}},
  1840. {"max_successive_merges",
  1841. {offset_of(&ColumnFamilyOptions::max_successive_merges),
  1842. OptionType::kSizeT, OptionVerificationType::kNormal, true,
  1843. offsetof(struct MutableCFOptions, max_successive_merges)}},
  1844. {"memtable_huge_page_size",
  1845. {offset_of(&ColumnFamilyOptions::memtable_huge_page_size),
  1846. OptionType::kSizeT, OptionVerificationType::kNormal, true,
  1847. offsetof(struct MutableCFOptions, memtable_huge_page_size)}},
  1848. {"memtable_prefix_bloom_huge_page_tlb_size",
  1849. {0, OptionType::kSizeT, OptionVerificationType::kDeprecated, true, 0}},
  1850. {"write_buffer_size",
  1851. {offset_of(&ColumnFamilyOptions::write_buffer_size),
  1852. OptionType::kSizeT, OptionVerificationType::kNormal, true,
  1853. offsetof(struct MutableCFOptions, write_buffer_size)}},
  1854. {"bloom_locality",
  1855. {offset_of(&ColumnFamilyOptions::bloom_locality), OptionType::kUInt32T,
  1856. OptionVerificationType::kNormal, false, 0}},
  1857. {"memtable_prefix_bloom_bits",
  1858. {0, OptionType::kUInt32T, OptionVerificationType::kDeprecated, true,
  1859. 0}},
  1860. {"memtable_prefix_bloom_size_ratio",
  1861. {offset_of(&ColumnFamilyOptions::memtable_prefix_bloom_size_ratio),
  1862. OptionType::kDouble, OptionVerificationType::kNormal, true,
  1863. offsetof(struct MutableCFOptions, memtable_prefix_bloom_size_ratio)}},
  1864. {"memtable_prefix_bloom_probes",
  1865. {0, OptionType::kUInt32T, OptionVerificationType::kDeprecated, true,
  1866. 0}},
  1867. {"memtable_whole_key_filtering",
  1868. {offset_of(&ColumnFamilyOptions::memtable_whole_key_filtering),
  1869. OptionType::kBoolean, OptionVerificationType::kNormal, true,
  1870. offsetof(struct MutableCFOptions, memtable_whole_key_filtering)}},
  1871. {"min_partial_merge_operands",
  1872. {0, OptionType::kUInt32T, OptionVerificationType::kDeprecated, true,
  1873. 0}},
  1874. {"max_bytes_for_level_base",
  1875. {offset_of(&ColumnFamilyOptions::max_bytes_for_level_base),
  1876. OptionType::kUInt64T, OptionVerificationType::kNormal, true,
  1877. offsetof(struct MutableCFOptions, max_bytes_for_level_base)}},
  1878. {"snap_refresh_nanos",
  1879. {0, OptionType::kUInt64T, OptionVerificationType::kDeprecated, true,
  1880. 0}},
  1881. {"max_bytes_for_level_multiplier",
  1882. {offset_of(&ColumnFamilyOptions::max_bytes_for_level_multiplier),
  1883. OptionType::kDouble, OptionVerificationType::kNormal, true,
  1884. offsetof(struct MutableCFOptions, max_bytes_for_level_multiplier)}},
  1885. {"max_bytes_for_level_multiplier_additional",
  1886. {offset_of(
  1887. &ColumnFamilyOptions::max_bytes_for_level_multiplier_additional),
  1888. OptionType::kVectorInt, OptionVerificationType::kNormal, true,
  1889. offsetof(struct MutableCFOptions,
  1890. max_bytes_for_level_multiplier_additional)}},
  1891. {"max_sequential_skip_in_iterations",
  1892. {offset_of(&ColumnFamilyOptions::max_sequential_skip_in_iterations),
  1893. OptionType::kUInt64T, OptionVerificationType::kNormal, true,
  1894. offsetof(struct MutableCFOptions,
  1895. max_sequential_skip_in_iterations)}},
  1896. {"target_file_size_base",
  1897. {offset_of(&ColumnFamilyOptions::target_file_size_base),
  1898. OptionType::kUInt64T, OptionVerificationType::kNormal, true,
  1899. offsetof(struct MutableCFOptions, target_file_size_base)}},
  1900. {"rate_limit_delay_max_milliseconds",
  1901. {0, OptionType::kUInt, OptionVerificationType::kDeprecated, false, 0}},
  1902. {"compression",
  1903. {offset_of(&ColumnFamilyOptions::compression),
  1904. OptionType::kCompressionType, OptionVerificationType::kNormal, true,
  1905. offsetof(struct MutableCFOptions, compression)}},
  1906. {"compression_per_level",
  1907. {offset_of(&ColumnFamilyOptions::compression_per_level),
  1908. OptionType::kVectorCompressionType, OptionVerificationType::kNormal,
  1909. false, 0}},
  1910. {"bottommost_compression",
  1911. {offset_of(&ColumnFamilyOptions::bottommost_compression),
  1912. OptionType::kCompressionType, OptionVerificationType::kNormal, false,
  1913. 0}},
  1914. {kNameComparator,
  1915. {offset_of(&ColumnFamilyOptions::comparator), OptionType::kComparator,
  1916. OptionVerificationType::kByName, false, 0}},
  1917. {"prefix_extractor",
  1918. {offset_of(&ColumnFamilyOptions::prefix_extractor),
  1919. OptionType::kSliceTransform, OptionVerificationType::kByNameAllowNull,
  1920. true, offsetof(struct MutableCFOptions, prefix_extractor)}},
  1921. {"memtable_insert_with_hint_prefix_extractor",
  1922. {offset_of(
  1923. &ColumnFamilyOptions::memtable_insert_with_hint_prefix_extractor),
  1924. OptionType::kSliceTransform, OptionVerificationType::kByNameAllowNull,
  1925. false, 0}},
  1926. {"memtable_factory",
  1927. {offset_of(&ColumnFamilyOptions::memtable_factory),
  1928. OptionType::kMemTableRepFactory, OptionVerificationType::kByName,
  1929. false, 0}},
  1930. {"table_factory",
  1931. {offset_of(&ColumnFamilyOptions::table_factory),
  1932. OptionType::kTableFactory, OptionVerificationType::kByName, false,
  1933. 0}},
  1934. {"compaction_filter",
  1935. {offset_of(&ColumnFamilyOptions::compaction_filter),
  1936. OptionType::kCompactionFilter, OptionVerificationType::kByName, false,
  1937. 0}},
  1938. {"compaction_filter_factory",
  1939. {offset_of(&ColumnFamilyOptions::compaction_filter_factory),
  1940. OptionType::kCompactionFilterFactory, OptionVerificationType::kByName,
  1941. false, 0}},
  1942. {kNameMergeOperator,
  1943. {offset_of(&ColumnFamilyOptions::merge_operator),
  1944. OptionType::kMergeOperator,
  1945. OptionVerificationType::kByNameAllowFromNull, false, 0}},
  1946. {"compaction_style",
  1947. {offset_of(&ColumnFamilyOptions::compaction_style),
  1948. OptionType::kCompactionStyle, OptionVerificationType::kNormal, false,
  1949. 0}},
  1950. {"compaction_pri",
  1951. {offset_of(&ColumnFamilyOptions::compaction_pri),
  1952. OptionType::kCompactionPri, OptionVerificationType::kNormal, false,
  1953. 0}},
  1954. {"compaction_options_fifo",
  1955. {offset_of(&ColumnFamilyOptions::compaction_options_fifo),
  1956. OptionType::kCompactionOptionsFIFO, OptionVerificationType::kNormal,
  1957. true, offsetof(struct MutableCFOptions, compaction_options_fifo)}},
  1958. {"compaction_options_universal",
  1959. {offset_of(&ColumnFamilyOptions::compaction_options_universal),
  1960. OptionType::kCompactionOptionsUniversal,
  1961. OptionVerificationType::kNormal, true,
  1962. offsetof(struct MutableCFOptions, compaction_options_universal)}},
  1963. {"ttl",
  1964. {offset_of(&ColumnFamilyOptions::ttl), OptionType::kUInt64T,
  1965. OptionVerificationType::kNormal, true,
  1966. offsetof(struct MutableCFOptions, ttl)}},
  1967. {"periodic_compaction_seconds",
  1968. {offset_of(&ColumnFamilyOptions::periodic_compaction_seconds),
  1969. OptionType::kUInt64T, OptionVerificationType::kNormal, true,
  1970. offsetof(struct MutableCFOptions, periodic_compaction_seconds)}},
  1971. {"sample_for_compression",
  1972. {offset_of(&ColumnFamilyOptions::sample_for_compression),
  1973. OptionType::kUInt64T, OptionVerificationType::kNormal, true,
  1974. offsetof(struct MutableCFOptions, sample_for_compression)}}};
  1975. std::unordered_map<std::string, OptionTypeInfo>
  1976. OptionsHelper::fifo_compaction_options_type_info = {
  1977. {"max_table_files_size",
  1978. {offset_of(&CompactionOptionsFIFO::max_table_files_size),
  1979. OptionType::kUInt64T, OptionVerificationType::kNormal, true,
  1980. offsetof(struct CompactionOptionsFIFO, max_table_files_size)}},
  1981. {"ttl",
  1982. {0, OptionType::kUInt64T,
  1983. OptionVerificationType::kDeprecated, false,
  1984. 0}},
  1985. {"allow_compaction",
  1986. {offset_of(&CompactionOptionsFIFO::allow_compaction),
  1987. OptionType::kBoolean, OptionVerificationType::kNormal, true,
  1988. offsetof(struct CompactionOptionsFIFO, allow_compaction)}}};
  1989. std::unordered_map<std::string, OptionTypeInfo>
  1990. OptionsHelper::universal_compaction_options_type_info = {
  1991. {"size_ratio",
  1992. {offset_of(&CompactionOptionsUniversal::size_ratio), OptionType::kUInt,
  1993. OptionVerificationType::kNormal, true,
  1994. offsetof(class CompactionOptionsUniversal, size_ratio)}},
  1995. {"min_merge_width",
  1996. {offset_of(&CompactionOptionsUniversal::min_merge_width),
  1997. OptionType::kUInt, OptionVerificationType::kNormal, true,
  1998. offsetof(class CompactionOptionsUniversal, min_merge_width)}},
  1999. {"max_merge_width",
  2000. {offset_of(&CompactionOptionsUniversal::max_merge_width),
  2001. OptionType::kUInt, OptionVerificationType::kNormal, true,
  2002. offsetof(class CompactionOptionsUniversal, max_merge_width)}},
  2003. {"max_size_amplification_percent",
  2004. {offset_of(
  2005. &CompactionOptionsUniversal::max_size_amplification_percent),
  2006. OptionType::kUInt, OptionVerificationType::kNormal, true,
  2007. offsetof(class CompactionOptionsUniversal,
  2008. max_size_amplification_percent)}},
  2009. {"compression_size_percent",
  2010. {offset_of(&CompactionOptionsUniversal::compression_size_percent),
  2011. OptionType::kInt, OptionVerificationType::kNormal, true,
  2012. offsetof(class CompactionOptionsUniversal,
  2013. compression_size_percent)}},
  2014. {"stop_style",
  2015. {offset_of(&CompactionOptionsUniversal::stop_style),
  2016. OptionType::kCompactionStopStyle, OptionVerificationType::kNormal,
  2017. true, offsetof(class CompactionOptionsUniversal, stop_style)}},
  2018. {"allow_trivial_move",
  2019. {offset_of(&CompactionOptionsUniversal::allow_trivial_move),
  2020. OptionType::kBoolean, OptionVerificationType::kNormal, true,
  2021. offsetof(class CompactionOptionsUniversal, allow_trivial_move)}}};
  2022. std::unordered_map<std::string, CompactionStopStyle>
  2023. OptionsHelper::compaction_stop_style_string_map = {
  2024. {"kCompactionStopStyleSimilarSize", kCompactionStopStyleSimilarSize},
  2025. {"kCompactionStopStyleTotalSize", kCompactionStopStyleTotalSize}};
  2026. std::unordered_map<std::string, OptionTypeInfo>
  2027. OptionsHelper::lru_cache_options_type_info = {
  2028. {"capacity",
  2029. {offset_of(&LRUCacheOptions::capacity), OptionType::kSizeT,
  2030. OptionVerificationType::kNormal, true,
  2031. offsetof(struct LRUCacheOptions, capacity)}},
  2032. {"num_shard_bits",
  2033. {offset_of(&LRUCacheOptions::num_shard_bits), OptionType::kInt,
  2034. OptionVerificationType::kNormal, true,
  2035. offsetof(struct LRUCacheOptions, num_shard_bits)}},
  2036. {"strict_capacity_limit",
  2037. {offset_of(&LRUCacheOptions::strict_capacity_limit),
  2038. OptionType::kBoolean, OptionVerificationType::kNormal, true,
  2039. offsetof(struct LRUCacheOptions, strict_capacity_limit)}},
  2040. {"high_pri_pool_ratio",
  2041. {offset_of(&LRUCacheOptions::high_pri_pool_ratio), OptionType::kDouble,
  2042. OptionVerificationType::kNormal, true,
  2043. offsetof(struct LRUCacheOptions, high_pri_pool_ratio)}}};
  2044. #endif // !ROCKSDB_LITE
  2045. } // namespace ROCKSDB_NAMESPACE