options_parser.cc 27 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765
  1. // Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
  2. // This source code is licensed under both the GPLv2 (found in the
  3. // COPYING file in the root directory) and Apache 2.0 License
  4. // (found in the LICENSE.Apache file in the root directory).
  5. #include "options/options_parser.h"
  6. #include <cmath>
  7. #include <map>
  8. #include <string>
  9. #include <utility>
  10. #include <vector>
  11. #include "file/file_util.h"
  12. #include "file/line_file_reader.h"
  13. #include "file/writable_file_writer.h"
  14. #include "options/cf_options.h"
  15. #include "options/db_options.h"
  16. #include "options/options_helper.h"
  17. #include "port/port.h"
  18. #include "rocksdb/convenience.h"
  19. #include "rocksdb/db.h"
  20. #include "rocksdb/utilities/options_type.h"
  21. #include "test_util/sync_point.h"
  22. #include "util/cast_util.h"
  23. #include "util/string_util.h"
  24. namespace ROCKSDB_NAMESPACE {
  25. static const std::string option_file_header =
  26. "# This is a RocksDB option file.\n"
  27. "#\n"
  28. "# For detailed file format spec, please refer to the example file\n"
  29. "# in examples/rocksdb_option_file_example.ini\n"
  30. "#\n"
  31. "\n";
  32. Status PersistRocksDBOptions(const WriteOptions& write_options,
  33. const DBOptions& db_opt,
  34. const std::vector<std::string>& cf_names,
  35. const std::vector<ColumnFamilyOptions>& cf_opts,
  36. const std::string& file_name, FileSystem* fs) {
  37. ConfigOptions
  38. config_options; // Use default for escaped(true) and check (exact)
  39. config_options.delimiter = "\n ";
  40. // Do not invoke PrepareOptions when we are doing validation.
  41. config_options.invoke_prepare_options = false;
  42. // If a readahead size was set in the input options, use it
  43. if (db_opt.log_readahead_size > 0) {
  44. config_options.file_readahead_size = db_opt.log_readahead_size;
  45. }
  46. return PersistRocksDBOptions(write_options, config_options, db_opt, cf_names,
  47. cf_opts, file_name, fs);
  48. }
  49. Status PersistRocksDBOptions(const WriteOptions& write_options,
  50. const ConfigOptions& config_options_in,
  51. const DBOptions& db_opt,
  52. const std::vector<std::string>& cf_names,
  53. const std::vector<ColumnFamilyOptions>& cf_opts,
  54. const std::string& file_name, FileSystem* fs) {
  55. ConfigOptions config_options = config_options_in;
  56. config_options.delimiter = "\n "; // Override the default to nl
  57. TEST_SYNC_POINT("PersistRocksDBOptions:start");
  58. if (cf_names.size() != cf_opts.size()) {
  59. return Status::InvalidArgument(
  60. "cf_names.size() and cf_opts.size() must be the same");
  61. }
  62. std::unique_ptr<FSWritableFile> wf;
  63. FileOptions file_options;
  64. file_options.temperature = db_opt.metadata_write_temperature;
  65. Status s = fs->NewWritableFile(file_name, file_options, &wf, nullptr);
  66. if (!s.ok()) {
  67. return s;
  68. }
  69. std::unique_ptr<WritableFileWriter> writable;
  70. writable.reset(new WritableFileWriter(std::move(wf), file_name, EnvOptions(),
  71. nullptr /* statistics */));
  72. TEST_SYNC_POINT("PersistRocksDBOptions:create");
  73. std::string options_file_content;
  74. IOOptions opts;
  75. s = WritableFileWriter::PrepareIOOptions(write_options, opts);
  76. if (s.ok()) {
  77. s = writable->Append(opts, option_file_header + "[" +
  78. opt_section_titles[kOptionSectionVersion] +
  79. "]\n"
  80. " rocksdb_version=" +
  81. std::to_string(ROCKSDB_MAJOR) + "." +
  82. std::to_string(ROCKSDB_MINOR) + "." +
  83. std::to_string(ROCKSDB_PATCH) + "\n");
  84. }
  85. if (s.ok()) {
  86. s = writable->Append(
  87. opts,
  88. " options_file_version=" + std::to_string(ROCKSDB_OPTION_FILE_MAJOR) +
  89. "." + std::to_string(ROCKSDB_OPTION_FILE_MINOR) + "\n");
  90. }
  91. if (s.ok()) {
  92. s = writable->Append(
  93. opts, "\n[" + opt_section_titles[kOptionSectionDBOptions] + "]\n ");
  94. }
  95. if (s.ok()) {
  96. s = GetStringFromDBOptions(config_options, db_opt, &options_file_content);
  97. }
  98. if (s.ok()) {
  99. s = writable->Append(opts, options_file_content + "\n");
  100. }
  101. for (size_t i = 0; s.ok() && i < cf_opts.size(); ++i) {
  102. // CFOptions section
  103. s = writable->Append(
  104. opts, "\n[" + opt_section_titles[kOptionSectionCFOptions] + " \"" +
  105. EscapeOptionString(cf_names[i]) + "\"]\n ");
  106. if (s.ok()) {
  107. s = GetStringFromColumnFamilyOptions(config_options, cf_opts[i],
  108. &options_file_content);
  109. }
  110. if (s.ok()) {
  111. s = writable->Append(opts, options_file_content + "\n");
  112. }
  113. // TableOptions section
  114. auto* tf = cf_opts[i].table_factory.get();
  115. if (tf != nullptr) {
  116. if (s.ok()) {
  117. s = writable->Append(
  118. opts, "[" + opt_section_titles[kOptionSectionTableOptions] +
  119. tf->Name() + " \"" + EscapeOptionString(cf_names[i]) +
  120. "\"]\n ");
  121. }
  122. if (s.ok()) {
  123. options_file_content.clear();
  124. s = tf->GetOptionString(config_options, &options_file_content);
  125. }
  126. if (s.ok()) {
  127. s = writable->Append(opts, options_file_content + "\n");
  128. }
  129. }
  130. }
  131. if (s.ok()) {
  132. s = writable->Sync(opts, true /* use_fsync */);
  133. }
  134. if (s.ok()) {
  135. s = writable->Close(opts);
  136. }
  137. TEST_SYNC_POINT("PersistRocksDBOptions:written");
  138. if (s.ok()) {
  139. return RocksDBOptionsParser::VerifyRocksDBOptionsFromFile(
  140. config_options, db_opt, cf_names, cf_opts, file_name, fs);
  141. }
  142. return s;
  143. }
  144. RocksDBOptionsParser::RocksDBOptionsParser() { Reset(); }
  145. void RocksDBOptionsParser::Reset() {
  146. db_opt_ = DBOptions();
  147. db_opt_map_.clear();
  148. cf_names_.clear();
  149. cf_opts_.clear();
  150. cf_opt_maps_.clear();
  151. has_version_section_ = false;
  152. has_db_options_ = false;
  153. has_default_cf_options_ = false;
  154. for (int i = 0; i < 3; ++i) {
  155. db_version[i] = 0;
  156. opt_file_version[i] = 0;
  157. }
  158. }
  159. bool RocksDBOptionsParser::IsSection(const std::string& line) {
  160. if (line.size() < 2) {
  161. return false;
  162. }
  163. if (line[0] != '[' || line[line.size() - 1] != ']') {
  164. return false;
  165. }
  166. return true;
  167. }
  168. Status RocksDBOptionsParser::ParseSection(OptionSection* section,
  169. std::string* title,
  170. std::string* argument,
  171. const std::string& line,
  172. const int line_num) {
  173. *section = kOptionSectionUnknown;
  174. // A section is of the form [<SectionName> "<SectionArg>"], where
  175. // "<SectionArg>" is optional.
  176. size_t arg_start_pos = line.find('\"');
  177. size_t arg_end_pos = line.rfind('\"');
  178. // The following if-then check tries to identify whether the input
  179. // section has the optional section argument.
  180. if (arg_start_pos != std::string::npos && arg_start_pos != arg_end_pos) {
  181. *title = TrimAndRemoveComment(line.substr(1, arg_start_pos - 1), true);
  182. *argument = UnescapeOptionString(
  183. line.substr(arg_start_pos + 1, arg_end_pos - arg_start_pos - 1));
  184. } else {
  185. *title = TrimAndRemoveComment(line.substr(1, line.size() - 2), true);
  186. *argument = "";
  187. }
  188. for (int i = 0; i < kOptionSectionUnknown; ++i) {
  189. if (title->find(opt_section_titles[i]) == 0) {
  190. if (i == kOptionSectionVersion || i == kOptionSectionDBOptions ||
  191. i == kOptionSectionCFOptions) {
  192. if (title->size() == opt_section_titles[i].size()) {
  193. // if true, then it indicats equal
  194. *section = static_cast<OptionSection>(i);
  195. return CheckSection(*section, *argument, line_num);
  196. }
  197. } else if (i == kOptionSectionTableOptions) {
  198. // This type of sections has a sufffix at the end of the
  199. // section title
  200. if (title->size() > opt_section_titles[i].size()) {
  201. *section = static_cast<OptionSection>(i);
  202. return CheckSection(*section, *argument, line_num);
  203. }
  204. }
  205. }
  206. }
  207. return Status::InvalidArgument(std::string("Unknown section ") + line);
  208. }
  209. Status RocksDBOptionsParser::InvalidArgument(const int line_num,
  210. const std::string& message) {
  211. return Status::InvalidArgument(
  212. "[RocksDBOptionsParser Error] ",
  213. message + " (at line " + std::to_string(line_num) + ")");
  214. }
  215. Status RocksDBOptionsParser::ParseStatement(std::string* name,
  216. std::string* value,
  217. const std::string& line,
  218. const int line_num) {
  219. size_t eq_pos = line.find('=');
  220. if (eq_pos == std::string::npos) {
  221. return InvalidArgument(line_num, "A valid statement must have a '='.");
  222. }
  223. *name = TrimAndRemoveComment(line.substr(0, eq_pos), true);
  224. *value =
  225. TrimAndRemoveComment(line.substr(eq_pos + 1, line.size() - eq_pos - 1));
  226. if (name->empty()) {
  227. return InvalidArgument(line_num,
  228. "A valid statement must have a variable name.");
  229. }
  230. return Status::OK();
  231. }
  232. Status RocksDBOptionsParser::Parse(const std::string& file_name, FileSystem* fs,
  233. bool ignore_unknown_options,
  234. size_t file_readahead_size) {
  235. ConfigOptions
  236. config_options; // Use default for escaped(true) and check (exact)
  237. config_options.ignore_unknown_options = ignore_unknown_options;
  238. if (file_readahead_size > 0) {
  239. config_options.file_readahead_size = file_readahead_size;
  240. }
  241. return Parse(config_options, file_name, fs);
  242. }
  243. Status RocksDBOptionsParser::Parse(const ConfigOptions& config_options_in,
  244. const std::string& file_name,
  245. FileSystem* fs) {
  246. Reset();
  247. ConfigOptions config_options = config_options_in;
  248. Status s;
  249. bool retry = false;
  250. do {
  251. std::unique_ptr<FSSequentialFile> seq_file;
  252. s = fs->NewSequentialFile(file_name, FileOptions(), &seq_file, nullptr);
  253. if (!s.ok()) {
  254. return s;
  255. }
  256. LineFileReader lf_reader(
  257. std::move(seq_file), file_name, config_options.file_readahead_size,
  258. nullptr, std::vector<std::shared_ptr<EventListener>>{}, nullptr, retry);
  259. OptionSection section = kOptionSectionUnknown;
  260. std::string title;
  261. std::string argument;
  262. std::unordered_map<std::string, std::string> opt_map;
  263. std::string line;
  264. // we only support single-lined statement.
  265. while (
  266. lf_reader.ReadLine(&line, Env::IO_TOTAL /* rate_limiter_priority */)) {
  267. int line_num = static_cast<int>(lf_reader.GetLineNumber());
  268. line = TrimAndRemoveComment(line);
  269. if (line.empty()) {
  270. continue;
  271. }
  272. if (IsSection(line)) {
  273. s = EndSection(config_options, section, title, argument, opt_map);
  274. opt_map.clear();
  275. if (!s.ok()) {
  276. break;
  277. }
  278. // If the option file is not generated by a higher version, unknown
  279. // option should only mean corruption.
  280. if (config_options.ignore_unknown_options &&
  281. section == kOptionSectionVersion) {
  282. using VTuple = std::tuple<int, int, int>;
  283. if (VTuple(db_version[0], db_version[1], db_version[2]) <=
  284. VTuple(ROCKSDB_MAJOR, ROCKSDB_MINOR, ROCKSDB_PATCH)) {
  285. config_options.ignore_unknown_options = false;
  286. }
  287. }
  288. s = ParseSection(&section, &title, &argument, line, line_num);
  289. if (!s.ok()) {
  290. break;
  291. }
  292. } else {
  293. std::string name;
  294. std::string value;
  295. s = ParseStatement(&name, &value, line, line_num);
  296. if (!s.ok()) {
  297. break;
  298. }
  299. opt_map.insert({name, value});
  300. }
  301. }
  302. if (s.ok()) {
  303. s = lf_reader.GetStatus();
  304. }
  305. if (s.ok()) {
  306. s = EndSection(config_options, section, title, argument, opt_map);
  307. opt_map.clear();
  308. }
  309. if (s.ok()) {
  310. s = ValidityCheck();
  311. }
  312. if (!s.ok()) {
  313. if ((s.IsCorruption() || s.IsInvalidArgument()) && !retry &&
  314. CheckFSFeatureSupport(fs,
  315. FSSupportedOps::kVerifyAndReconstructRead)) {
  316. retry = true;
  317. Reset();
  318. } else {
  319. return s;
  320. }
  321. } else {
  322. return s;
  323. }
  324. } while (retry);
  325. return s;
  326. }
  327. Status RocksDBOptionsParser::CheckSection(const OptionSection section,
  328. const std::string& section_arg,
  329. const int line_num) {
  330. if (section == kOptionSectionDBOptions) {
  331. if (has_db_options_) {
  332. return InvalidArgument(
  333. line_num,
  334. "More than one DBOption section found in the option config file");
  335. }
  336. has_db_options_ = true;
  337. } else if (section == kOptionSectionCFOptions) {
  338. bool is_default_cf = (section_arg == kDefaultColumnFamilyName);
  339. if (cf_opts_.size() == 0 && !is_default_cf) {
  340. return InvalidArgument(
  341. line_num,
  342. "Default column family must be the first CFOptions section "
  343. "in the option config file");
  344. } else if (cf_opts_.size() != 0 && is_default_cf) {
  345. return InvalidArgument(
  346. line_num,
  347. "Default column family must be the first CFOptions section "
  348. "in the optio/n config file");
  349. } else if (GetCFOptions(section_arg) != nullptr) {
  350. return InvalidArgument(
  351. line_num,
  352. "Two identical column families found in option config file");
  353. }
  354. has_default_cf_options_ |= is_default_cf;
  355. } else if (section == kOptionSectionTableOptions) {
  356. if (GetCFOptions(section_arg) == nullptr) {
  357. return InvalidArgument(
  358. line_num, std::string("Does not find a matched column family name in "
  359. "TableOptions section. Column Family Name:") +
  360. section_arg);
  361. }
  362. } else if (section == kOptionSectionVersion) {
  363. if (has_version_section_) {
  364. return InvalidArgument(
  365. line_num,
  366. "More than one Version section found in the option config file.");
  367. }
  368. has_version_section_ = true;
  369. }
  370. return Status::OK();
  371. }
  372. Status RocksDBOptionsParser::ParseVersionNumber(const std::string& ver_name,
  373. const std::string& ver_string,
  374. const int max_count,
  375. int* version) {
  376. int version_index = 0;
  377. int current_number = 0;
  378. int current_digit_count = 0;
  379. bool has_dot = false;
  380. for (int i = 0; i < max_count; ++i) {
  381. version[i] = 0;
  382. }
  383. constexpr int kBufferSize = 200;
  384. char buffer[kBufferSize];
  385. for (size_t i = 0; i < ver_string.size(); ++i) {
  386. if (ver_string[i] == '.') {
  387. if (version_index >= max_count - 1) {
  388. snprintf(buffer, sizeof(buffer) - 1,
  389. "A valid %s can only contains at most %d dots.",
  390. ver_name.c_str(), max_count - 1);
  391. return Status::InvalidArgument(buffer);
  392. }
  393. if (current_digit_count == 0) {
  394. snprintf(buffer, sizeof(buffer) - 1,
  395. "A valid %s must have at least one digit before each dot.",
  396. ver_name.c_str());
  397. return Status::InvalidArgument(buffer);
  398. }
  399. version[version_index++] = current_number;
  400. current_number = 0;
  401. current_digit_count = 0;
  402. has_dot = true;
  403. } else if (isdigit(ver_string[i])) {
  404. current_number = current_number * 10 + (ver_string[i] - '0');
  405. current_digit_count++;
  406. } else {
  407. snprintf(buffer, sizeof(buffer) - 1,
  408. "A valid %s can only contains dots and numbers.",
  409. ver_name.c_str());
  410. return Status::InvalidArgument(buffer);
  411. }
  412. }
  413. version[version_index] = current_number;
  414. if (has_dot && current_digit_count == 0) {
  415. snprintf(buffer, sizeof(buffer) - 1,
  416. "A valid %s must have at least one digit after each dot.",
  417. ver_name.c_str());
  418. return Status::InvalidArgument(buffer);
  419. }
  420. return Status::OK();
  421. }
  422. Status RocksDBOptionsParser::EndSection(
  423. const ConfigOptions& config_options, const OptionSection section,
  424. const std::string& section_title, const std::string& section_arg,
  425. const std::unordered_map<std::string, std::string>& opt_map) {
  426. Status s;
  427. if (section == kOptionSectionDBOptions) {
  428. s = GetDBOptionsFromMap(config_options, DBOptions(), opt_map, &db_opt_);
  429. if (!s.ok()) {
  430. return s;
  431. }
  432. db_opt_map_ = opt_map;
  433. } else if (section == kOptionSectionCFOptions) {
  434. // This condition should be ensured earlier in ParseSection
  435. // so we make an assertion here.
  436. assert(GetCFOptions(section_arg) == nullptr);
  437. cf_names_.emplace_back(section_arg);
  438. cf_opts_.emplace_back();
  439. s = GetColumnFamilyOptionsFromMap(config_options, ColumnFamilyOptions(),
  440. opt_map, &cf_opts_.back());
  441. if (!s.ok()) {
  442. return s;
  443. }
  444. // keep the parsed string.
  445. cf_opt_maps_.emplace_back(opt_map);
  446. } else if (section == kOptionSectionTableOptions) {
  447. assert(GetCFOptions(section_arg) != nullptr);
  448. auto* cf_opt = GetCFOptionsImpl(section_arg);
  449. if (cf_opt == nullptr) {
  450. return Status::InvalidArgument(
  451. "The specified column family must be defined before the "
  452. "TableOptions section:",
  453. section_arg);
  454. }
  455. // Ignore error as table factory deserialization is optional
  456. cf_opt->table_factory.reset();
  457. s = TableFactory::CreateFromString(
  458. config_options,
  459. section_title.substr(
  460. opt_section_titles[kOptionSectionTableOptions].size()),
  461. &(cf_opt->table_factory));
  462. if (s.ok() && cf_opt->table_factory != nullptr) {
  463. s = cf_opt->table_factory->ConfigureFromMap(config_options, opt_map);
  464. // Translate any errors (NotFound, NotSupported, to InvalidArgument
  465. if (s.ok() || s.IsInvalidArgument()) {
  466. return s;
  467. } else {
  468. return Status::InvalidArgument(s.getState());
  469. }
  470. } else {
  471. // Return OK for not supported table factories as TableFactory
  472. // Deserialization is optional.
  473. cf_opt->table_factory.reset();
  474. return Status::OK();
  475. }
  476. } else if (section == kOptionSectionVersion) {
  477. for (const auto& pair : opt_map) {
  478. if (pair.first == "rocksdb_version") {
  479. s = ParseVersionNumber(pair.first, pair.second, 3, db_version);
  480. if (!s.ok()) {
  481. return s;
  482. }
  483. } else if (pair.first == "options_file_version") {
  484. s = ParseVersionNumber(pair.first, pair.second, 2, opt_file_version);
  485. if (!s.ok()) {
  486. return s;
  487. }
  488. if (opt_file_version[0] < 1) {
  489. return Status::InvalidArgument(
  490. "A valid options_file_version must be at least 1.");
  491. }
  492. }
  493. }
  494. }
  495. return s;
  496. }
  497. Status RocksDBOptionsParser::ValidityCheck() {
  498. if (!has_db_options_) {
  499. return Status::Corruption(
  500. "A RocksDB Option file must have a single DBOptions section");
  501. }
  502. if (!has_default_cf_options_) {
  503. return Status::Corruption(
  504. "A RocksDB Option file must have a single CFOptions:default section");
  505. }
  506. return Status::OK();
  507. }
  508. std::string RocksDBOptionsParser::TrimAndRemoveComment(const std::string& line,
  509. bool trim_only) {
  510. size_t start = 0;
  511. size_t end = line.size();
  512. // we only support "#" style comment
  513. if (!trim_only) {
  514. size_t search_pos = 0;
  515. while (search_pos < line.size()) {
  516. size_t comment_pos = line.find('#', search_pos);
  517. if (comment_pos == std::string::npos) {
  518. break;
  519. }
  520. if (comment_pos == 0 || line[comment_pos - 1] != '\\') {
  521. end = comment_pos;
  522. break;
  523. }
  524. search_pos = comment_pos + 1;
  525. }
  526. }
  527. while (start < end && isspace(line[start]) != 0) {
  528. ++start;
  529. }
  530. // start < end implies end > 0.
  531. while (start < end && isspace(line[end - 1]) != 0) {
  532. --end;
  533. }
  534. if (start < end) {
  535. return line.substr(start, end - start);
  536. }
  537. return "";
  538. }
  539. Status RocksDBOptionsParser::VerifyRocksDBOptionsFromFile(
  540. const ConfigOptions& config_options_in, const DBOptions& db_opt,
  541. const std::vector<std::string>& cf_names,
  542. const std::vector<ColumnFamilyOptions>& cf_opts,
  543. const std::string& file_name, FileSystem* fs) {
  544. RocksDBOptionsParser parser;
  545. ConfigOptions config_options = config_options_in;
  546. config_options.invoke_prepare_options =
  547. false; // No need to do a prepare for verify
  548. if (config_options.sanity_level < ConfigOptions::kSanityLevelExactMatch) {
  549. // If we are not doing an exact comparison, we should ignore
  550. // unsupported options, as they may cause the Parse to fail
  551. // (if the ObjectRegistry is not initialized)
  552. config_options.ignore_unsupported_options = true;
  553. }
  554. Status s = parser.Parse(config_options, file_name, fs);
  555. if (!s.ok()) {
  556. return s;
  557. }
  558. // Verify DBOptions
  559. s = VerifyDBOptions(config_options, db_opt, *parser.db_opt(),
  560. parser.db_opt_map());
  561. if (!s.ok()) {
  562. return s;
  563. }
  564. // Verify ColumnFamily Name
  565. if (cf_names.size() != parser.cf_names()->size()) {
  566. if (config_options.sanity_level >=
  567. ConfigOptions::kSanityLevelLooselyCompatible) {
  568. return Status::InvalidArgument(
  569. "[RocksDBOptionParser Error] The persisted options does not have "
  570. "the same number of column family names as the db instance.");
  571. } else if (cf_opts.size() > parser.cf_opts()->size()) {
  572. return Status::InvalidArgument(
  573. "[RocksDBOptionsParser Error]",
  574. "The persisted options file has less number of column family "
  575. "names than that of the specified one.");
  576. }
  577. }
  578. for (size_t i = 0; i < cf_names.size(); ++i) {
  579. if (cf_names[i] != parser.cf_names()->at(i)) {
  580. return Status::InvalidArgument(
  581. "[RocksDBOptionParser Error] The persisted options and the db"
  582. "instance does not have the same name for column family ",
  583. std::to_string(i));
  584. }
  585. }
  586. // Verify Column Family Options
  587. if (cf_opts.size() != parser.cf_opts()->size()) {
  588. if (config_options.sanity_level >=
  589. ConfigOptions::kSanityLevelLooselyCompatible) {
  590. return Status::InvalidArgument(
  591. "[RocksDBOptionsParser Error]",
  592. "The persisted options does not have the same number of "
  593. "column families as the db instance.");
  594. } else if (cf_opts.size() > parser.cf_opts()->size()) {
  595. return Status::InvalidArgument(
  596. "[RocksDBOptionsParser Error]",
  597. "The persisted options file has less number of column families "
  598. "than that of the specified number.");
  599. }
  600. }
  601. for (size_t i = 0; i < cf_opts.size(); ++i) {
  602. s = VerifyCFOptions(config_options, cf_opts[i], parser.cf_opts()->at(i),
  603. &(parser.cf_opt_maps()->at(i)));
  604. if (!s.ok()) {
  605. return s;
  606. }
  607. s = VerifyTableFactory(config_options, cf_opts[i].table_factory.get(),
  608. parser.cf_opts()->at(i).table_factory.get());
  609. if (!s.ok()) {
  610. return s;
  611. }
  612. }
  613. return Status::OK();
  614. }
  615. Status RocksDBOptionsParser::VerifyDBOptions(
  616. const ConfigOptions& config_options, const DBOptions& base_opt,
  617. const DBOptions& file_opt,
  618. const std::unordered_map<std::string, std::string>* opt_map) {
  619. auto base_config = DBOptionsAsConfigurable(base_opt, opt_map);
  620. auto file_config = DBOptionsAsConfigurable(file_opt, opt_map);
  621. std::string mismatch;
  622. if (!base_config->AreEquivalent(config_options, file_config.get(),
  623. &mismatch)) {
  624. const size_t kBufferSize = 2048;
  625. char buffer[kBufferSize];
  626. std::string base_value;
  627. std::string file_value;
  628. int offset = snprintf(buffer, sizeof(buffer),
  629. "[RocksDBOptionsParser]: "
  630. "failed the verification on DBOptions::%s -- ",
  631. mismatch.c_str());
  632. Status s = base_config->GetOption(config_options, mismatch, &base_value);
  633. if (s.ok()) {
  634. s = file_config->GetOption(config_options, mismatch, &file_value);
  635. }
  636. assert(offset >= 0);
  637. assert(static_cast<size_t>(offset) < sizeof(buffer));
  638. if (s.ok()) {
  639. snprintf(buffer + offset, sizeof(buffer) - static_cast<size_t>(offset),
  640. "-- The specified one is %s while the persisted one is %s.\n",
  641. base_value.c_str(), file_value.c_str());
  642. } else {
  643. snprintf(buffer + offset, sizeof(buffer) - static_cast<size_t>(offset),
  644. "-- Unable to re-serialize an option: %s.\n",
  645. s.ToString().c_str());
  646. }
  647. return Status::InvalidArgument(Slice(buffer, strlen(buffer)));
  648. }
  649. return Status::OK();
  650. }
  651. Status RocksDBOptionsParser::VerifyCFOptions(
  652. const ConfigOptions& config_options, const ColumnFamilyOptions& base_opt,
  653. const ColumnFamilyOptions& file_opt,
  654. const std::unordered_map<std::string, std::string>* opt_map) {
  655. auto base_config = CFOptionsAsConfigurable(base_opt, opt_map);
  656. auto file_config = CFOptionsAsConfigurable(file_opt, opt_map);
  657. std::string mismatch;
  658. if (!base_config->AreEquivalent(config_options, file_config.get(),
  659. &mismatch)) {
  660. std::string base_value;
  661. std::string file_value;
  662. // The options do not match
  663. const size_t kBufferSize = 2048;
  664. char buffer[kBufferSize];
  665. Status s = base_config->GetOption(config_options, mismatch, &base_value);
  666. if (s.ok()) {
  667. s = file_config->GetOption(config_options, mismatch, &file_value);
  668. // In file_opt, certain options like MergeOperator may be nullptr due to
  669. // factor methods not available. So we use opt_map to get
  670. // option value to use in the error message below.
  671. if (s.ok() && file_value == kNullptrString && opt_map) {
  672. auto const& opt_val_str = (opt_map->find(mismatch));
  673. if (opt_val_str != opt_map->end()) {
  674. file_value = opt_val_str->second;
  675. }
  676. }
  677. }
  678. int offset = snprintf(buffer, sizeof(buffer),
  679. "[RocksDBOptionsParser]: "
  680. "failed the verification on ColumnFamilyOptions::%s",
  681. mismatch.c_str());
  682. assert(offset >= 0);
  683. assert(static_cast<size_t>(offset) < sizeof(buffer));
  684. if (s.ok()) {
  685. snprintf(buffer + offset, sizeof(buffer) - static_cast<size_t>(offset),
  686. "--- The specified one is %s while the persisted one is %s.\n",
  687. base_value.c_str(), file_value.c_str());
  688. } else {
  689. snprintf(buffer + offset, sizeof(buffer) - static_cast<size_t>(offset),
  690. "--- Unable to re-serialize an option: %s.\n",
  691. s.ToString().c_str());
  692. }
  693. return Status::InvalidArgument(Slice(buffer, sizeof(buffer)));
  694. } // For each option
  695. return Status::OK();
  696. }
  697. Status RocksDBOptionsParser::VerifyTableFactory(
  698. const ConfigOptions& config_options, const TableFactory* base_tf,
  699. const TableFactory* file_tf) {
  700. std::string mismatch;
  701. if (base_tf && file_tf) {
  702. if (config_options.sanity_level > ConfigOptions::kSanityLevelNone &&
  703. std::string(base_tf->Name()) != std::string(file_tf->Name())) {
  704. return Status::Corruption(
  705. "[RocksDBOptionsParser]: "
  706. "failed the verification on TableFactory->Name()");
  707. } else if (!base_tf->AreEquivalent(config_options, file_tf, &mismatch)) {
  708. return Status::Corruption(std::string("[RocksDBOptionsParser]:"
  709. "failed the verification on ") +
  710. base_tf->Name() + "::",
  711. mismatch);
  712. }
  713. } else {
  714. // TODO(yhchiang): further support sanity check here
  715. }
  716. return Status::OK();
  717. }
  718. } // namespace ROCKSDB_NAMESPACE