block_based_table_factory.cc 26 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649
  1. // Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
  2. // This source code is licensed under both the GPLv2 (found in the
  3. // COPYING file in the root directory) and Apache 2.0 License
  4. // (found in the LICENSE.Apache file in the root directory).
  5. //
  6. // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
  7. // Use of this source code is governed by a BSD-style license that can be
  8. // found in the LICENSE file. See the AUTHORS file for names of contributors.
  9. #include <stdint.h>
  10. #include <cinttypes>
  11. #include <memory>
  12. #include <string>
  13. #include "options/options_helper.h"
  14. #include "port/port.h"
  15. #include "rocksdb/cache.h"
  16. #include "rocksdb/convenience.h"
  17. #include "rocksdb/flush_block_policy.h"
  18. #include "table/block_based/block_based_table_builder.h"
  19. #include "table/block_based/block_based_table_factory.h"
  20. #include "table/block_based/block_based_table_reader.h"
  21. #include "table/format.h"
  22. #include "util/mutexlock.h"
  23. #include "util/string_util.h"
  24. namespace ROCKSDB_NAMESPACE {
  25. void TailPrefetchStats::RecordEffectiveSize(size_t len) {
  26. MutexLock l(&mutex_);
  27. if (num_records_ < kNumTracked) {
  28. num_records_++;
  29. }
  30. records_[next_++] = len;
  31. if (next_ == kNumTracked) {
  32. next_ = 0;
  33. }
  34. }
  35. size_t TailPrefetchStats::GetSuggestedPrefetchSize() {
  36. std::vector<size_t> sorted;
  37. {
  38. MutexLock l(&mutex_);
  39. if (num_records_ == 0) {
  40. return 0;
  41. }
  42. sorted.assign(records_, records_ + num_records_);
  43. }
  44. // Of the historic size, we find the maximum one that satisifis the condtiion
  45. // that if prefetching all, less than 1/8 will be wasted.
  46. std::sort(sorted.begin(), sorted.end());
  47. // Assuming we have 5 data points, and after sorting it looks like this:
  48. //
  49. // +---+
  50. // +---+ | |
  51. // | | | |
  52. // | | | |
  53. // | | | |
  54. // | | | |
  55. // +---+ | | | |
  56. // | | | | | |
  57. // +---+ | | | | | |
  58. // | | | | | | | |
  59. // +---+ | | | | | | | |
  60. // | | | | | | | | | |
  61. // | | | | | | | | | |
  62. // | | | | | | | | | |
  63. // | | | | | | | | | |
  64. // | | | | | | | | | |
  65. // +---+ +---+ +---+ +---+ +---+
  66. //
  67. // and we use every of the value as a candidate, and estimate how much we
  68. // wasted, compared to read. For example, when we use the 3rd record
  69. // as candiate. This area is what we read:
  70. // +---+
  71. // +---+ | |
  72. // | | | |
  73. // | | | |
  74. // | | | |
  75. // | | | |
  76. // *** *** *** ***+ *** *** *** *** **
  77. // * | | | | | |
  78. // +---+ | | | | | *
  79. // * | | | | | | | |
  80. // +---+ | | | | | | | *
  81. // * | | | | X | | | | |
  82. // | | | | | | | | | *
  83. // * | | | | | | | | |
  84. // | | | | | | | | | *
  85. // * | | | | | | | | |
  86. // *** *** ***-*** ***--*** ***--*** +****
  87. // which is (size of the record) X (number of records).
  88. //
  89. // While wasted is this area:
  90. // +---+
  91. // +---+ | |
  92. // | | | |
  93. // | | | |
  94. // | | | |
  95. // | | | |
  96. // *** *** *** ****---+ | | | |
  97. // * * | | | | |
  98. // * *-*** *** | | | | |
  99. // * * | | | | | | |
  100. // *--** *** | | | | | | |
  101. // | | | | | X | | | | |
  102. // | | | | | | | | | |
  103. // | | | | | | | | | |
  104. // | | | | | | | | | |
  105. // | | | | | | | | | |
  106. // +---+ +---+ +---+ +---+ +---+
  107. //
  108. // Which can be calculated iteratively.
  109. // The difference between wasted using 4st and 3rd record, will
  110. // be following area:
  111. // +---+
  112. // +--+ +-+ ++ +-+ +-+ +---+ | |
  113. // + xxxxxxxxxxxxxxxxxxxxxxxx | | | |
  114. // xxxxxxxxxxxxxxxxxxxxxxxx | | | |
  115. // + xxxxxxxxxxxxxxxxxxxxxxxx | | | |
  116. // | xxxxxxxxxxxxxxxxxxxxxxxx | | | |
  117. // +-+ +-+ +-+ ++ +---+ +--+ | | |
  118. // | | | | | | |
  119. // +---+ ++ | | | | | |
  120. // | | | | | | X | | |
  121. // +---+ ++ | | | | | | | |
  122. // | | | | | | | | | |
  123. // | | | | | | | | | |
  124. // | | | | | | | | | |
  125. // | | | | | | | | | |
  126. // | | | | | | | | | |
  127. // +---+ +---+ +---+ +---+ +---+
  128. //
  129. // which will be the size difference between 4st and 3rd record,
  130. // times 3, which is number of records before the 4st.
  131. // Here we assume that all data within the prefetch range will be useful. In
  132. // reality, it may not be the case when a partial block is inside the range,
  133. // or there are data in the middle that is not read. We ignore those cases
  134. // for simplicity.
  135. assert(!sorted.empty());
  136. size_t prev_size = sorted[0];
  137. size_t max_qualified_size = sorted[0];
  138. size_t wasted = 0;
  139. for (size_t i = 1; i < sorted.size(); i++) {
  140. size_t read = sorted[i] * sorted.size();
  141. wasted += (sorted[i] - prev_size) * i;
  142. if (wasted <= read / 8) {
  143. max_qualified_size = sorted[i];
  144. }
  145. prev_size = sorted[i];
  146. }
  147. const size_t kMaxPrefetchSize = 512 * 1024; // Never exceed 512KB
  148. return std::min(kMaxPrefetchSize, max_qualified_size);
  149. }
  150. // TODO(myabandeh): We should return an error instead of silently changing the
  151. // options
  152. BlockBasedTableFactory::BlockBasedTableFactory(
  153. const BlockBasedTableOptions& _table_options)
  154. : table_options_(_table_options) {
  155. if (table_options_.flush_block_policy_factory == nullptr) {
  156. table_options_.flush_block_policy_factory.reset(
  157. new FlushBlockBySizePolicyFactory());
  158. }
  159. if (table_options_.no_block_cache) {
  160. table_options_.block_cache.reset();
  161. } else if (table_options_.block_cache == nullptr) {
  162. LRUCacheOptions co;
  163. co.capacity = 8 << 20;
  164. // It makes little sense to pay overhead for mid-point insertion while the
  165. // block size is only 8MB.
  166. co.high_pri_pool_ratio = 0.0;
  167. table_options_.block_cache = NewLRUCache(co);
  168. }
  169. if (table_options_.block_size_deviation < 0 ||
  170. table_options_.block_size_deviation > 100) {
  171. table_options_.block_size_deviation = 0;
  172. }
  173. if (table_options_.block_restart_interval < 1) {
  174. table_options_.block_restart_interval = 1;
  175. }
  176. if (table_options_.index_block_restart_interval < 1) {
  177. table_options_.index_block_restart_interval = 1;
  178. }
  179. if (table_options_.index_type == BlockBasedTableOptions::kHashSearch &&
  180. table_options_.index_block_restart_interval != 1) {
  181. // Currently kHashSearch is incompatible with index_block_restart_interval > 1
  182. table_options_.index_block_restart_interval = 1;
  183. }
  184. if (table_options_.partition_filters &&
  185. table_options_.index_type !=
  186. BlockBasedTableOptions::kTwoLevelIndexSearch) {
  187. // We do not support partitioned filters without partitioning indexes
  188. table_options_.partition_filters = false;
  189. }
  190. }
  191. Status BlockBasedTableFactory::NewTableReader(
  192. const TableReaderOptions& table_reader_options,
  193. std::unique_ptr<RandomAccessFileReader>&& file, uint64_t file_size,
  194. std::unique_ptr<TableReader>* table_reader,
  195. bool prefetch_index_and_filter_in_cache) const {
  196. return BlockBasedTable::Open(
  197. table_reader_options.ioptions, table_reader_options.env_options,
  198. table_options_, table_reader_options.internal_comparator, std::move(file),
  199. file_size, table_reader, table_reader_options.prefix_extractor,
  200. prefetch_index_and_filter_in_cache, table_reader_options.skip_filters,
  201. table_reader_options.level, table_reader_options.immortal,
  202. table_reader_options.largest_seqno, &tail_prefetch_stats_,
  203. table_reader_options.block_cache_tracer);
  204. }
  205. TableBuilder* BlockBasedTableFactory::NewTableBuilder(
  206. const TableBuilderOptions& table_builder_options, uint32_t column_family_id,
  207. WritableFileWriter* file) const {
  208. auto table_builder = new BlockBasedTableBuilder(
  209. table_builder_options.ioptions, table_builder_options.moptions,
  210. table_options_, table_builder_options.internal_comparator,
  211. table_builder_options.int_tbl_prop_collector_factories, column_family_id,
  212. file, table_builder_options.compression_type,
  213. table_builder_options.sample_for_compression,
  214. table_builder_options.compression_opts,
  215. table_builder_options.skip_filters,
  216. table_builder_options.column_family_name, table_builder_options.level,
  217. table_builder_options.creation_time,
  218. table_builder_options.oldest_key_time,
  219. table_builder_options.target_file_size,
  220. table_builder_options.file_creation_time);
  221. return table_builder;
  222. }
  223. Status BlockBasedTableFactory::SanitizeOptions(
  224. const DBOptions& db_opts, const ColumnFamilyOptions& cf_opts) const {
  225. if (table_options_.index_type == BlockBasedTableOptions::kHashSearch &&
  226. cf_opts.prefix_extractor == nullptr) {
  227. return Status::InvalidArgument(
  228. "Hash index is specified for block-based "
  229. "table, but prefix_extractor is not given");
  230. }
  231. if (table_options_.cache_index_and_filter_blocks &&
  232. table_options_.no_block_cache) {
  233. return Status::InvalidArgument(
  234. "Enable cache_index_and_filter_blocks, "
  235. ", but block cache is disabled");
  236. }
  237. if (table_options_.pin_l0_filter_and_index_blocks_in_cache &&
  238. table_options_.no_block_cache) {
  239. return Status::InvalidArgument(
  240. "Enable pin_l0_filter_and_index_blocks_in_cache, "
  241. ", but block cache is disabled");
  242. }
  243. if (!BlockBasedTableSupportedVersion(table_options_.format_version)) {
  244. return Status::InvalidArgument(
  245. "Unsupported BlockBasedTable format_version. Please check "
  246. "include/rocksdb/table.h for more info");
  247. }
  248. if (table_options_.block_align && (cf_opts.compression != kNoCompression)) {
  249. return Status::InvalidArgument(
  250. "Enable block_align, but compression "
  251. "enabled");
  252. }
  253. if (table_options_.block_align &&
  254. (table_options_.block_size & (table_options_.block_size - 1))) {
  255. return Status::InvalidArgument(
  256. "Block alignment requested but block size is not a power of 2");
  257. }
  258. if (table_options_.block_size > port::kMaxUint32) {
  259. return Status::InvalidArgument(
  260. "block size exceeds maximum number (4GiB) allowed");
  261. }
  262. if (table_options_.data_block_index_type ==
  263. BlockBasedTableOptions::kDataBlockBinaryAndHash &&
  264. table_options_.data_block_hash_table_util_ratio <= 0) {
  265. return Status::InvalidArgument(
  266. "data_block_hash_table_util_ratio should be greater than 0 when "
  267. "data_block_index_type is set to kDataBlockBinaryAndHash");
  268. }
  269. if (db_opts.unordered_write && cf_opts.max_successive_merges > 0) {
  270. // TODO(myabandeh): support it
  271. return Status::InvalidArgument(
  272. "max_successive_merges larger than 0 is currently inconsistent with "
  273. "unordered_write");
  274. }
  275. return Status::OK();
  276. }
  277. std::string BlockBasedTableFactory::GetPrintableTableOptions() const {
  278. std::string ret;
  279. ret.reserve(20000);
  280. const int kBufferSize = 200;
  281. char buffer[kBufferSize];
  282. snprintf(buffer, kBufferSize, " flush_block_policy_factory: %s (%p)\n",
  283. table_options_.flush_block_policy_factory->Name(),
  284. static_cast<void*>(table_options_.flush_block_policy_factory.get()));
  285. ret.append(buffer);
  286. snprintf(buffer, kBufferSize, " cache_index_and_filter_blocks: %d\n",
  287. table_options_.cache_index_and_filter_blocks);
  288. ret.append(buffer);
  289. snprintf(buffer, kBufferSize,
  290. " cache_index_and_filter_blocks_with_high_priority: %d\n",
  291. table_options_.cache_index_and_filter_blocks_with_high_priority);
  292. ret.append(buffer);
  293. snprintf(buffer, kBufferSize,
  294. " pin_l0_filter_and_index_blocks_in_cache: %d\n",
  295. table_options_.pin_l0_filter_and_index_blocks_in_cache);
  296. ret.append(buffer);
  297. snprintf(buffer, kBufferSize, " pin_top_level_index_and_filter: %d\n",
  298. table_options_.pin_top_level_index_and_filter);
  299. ret.append(buffer);
  300. snprintf(buffer, kBufferSize, " index_type: %d\n",
  301. table_options_.index_type);
  302. ret.append(buffer);
  303. snprintf(buffer, kBufferSize, " data_block_index_type: %d\n",
  304. table_options_.data_block_index_type);
  305. ret.append(buffer);
  306. snprintf(buffer, kBufferSize, " index_shortening: %d\n",
  307. static_cast<int>(table_options_.index_shortening));
  308. ret.append(buffer);
  309. snprintf(buffer, kBufferSize, " data_block_hash_table_util_ratio: %lf\n",
  310. table_options_.data_block_hash_table_util_ratio);
  311. ret.append(buffer);
  312. snprintf(buffer, kBufferSize, " hash_index_allow_collision: %d\n",
  313. table_options_.hash_index_allow_collision);
  314. ret.append(buffer);
  315. snprintf(buffer, kBufferSize, " checksum: %d\n", table_options_.checksum);
  316. ret.append(buffer);
  317. snprintf(buffer, kBufferSize, " no_block_cache: %d\n",
  318. table_options_.no_block_cache);
  319. ret.append(buffer);
  320. snprintf(buffer, kBufferSize, " block_cache: %p\n",
  321. static_cast<void*>(table_options_.block_cache.get()));
  322. ret.append(buffer);
  323. if (table_options_.block_cache) {
  324. const char* block_cache_name = table_options_.block_cache->Name();
  325. if (block_cache_name != nullptr) {
  326. snprintf(buffer, kBufferSize, " block_cache_name: %s\n",
  327. block_cache_name);
  328. ret.append(buffer);
  329. }
  330. ret.append(" block_cache_options:\n");
  331. ret.append(table_options_.block_cache->GetPrintableOptions());
  332. }
  333. snprintf(buffer, kBufferSize, " block_cache_compressed: %p\n",
  334. static_cast<void*>(table_options_.block_cache_compressed.get()));
  335. ret.append(buffer);
  336. if (table_options_.block_cache_compressed) {
  337. const char* block_cache_compressed_name =
  338. table_options_.block_cache_compressed->Name();
  339. if (block_cache_compressed_name != nullptr) {
  340. snprintf(buffer, kBufferSize, " block_cache_name: %s\n",
  341. block_cache_compressed_name);
  342. ret.append(buffer);
  343. }
  344. ret.append(" block_cache_compressed_options:\n");
  345. ret.append(table_options_.block_cache_compressed->GetPrintableOptions());
  346. }
  347. snprintf(buffer, kBufferSize, " persistent_cache: %p\n",
  348. static_cast<void*>(table_options_.persistent_cache.get()));
  349. ret.append(buffer);
  350. if (table_options_.persistent_cache) {
  351. snprintf(buffer, kBufferSize, " persistent_cache_options:\n");
  352. ret.append(buffer);
  353. ret.append(table_options_.persistent_cache->GetPrintableOptions());
  354. }
  355. snprintf(buffer, kBufferSize, " block_size: %" ROCKSDB_PRIszt "\n",
  356. table_options_.block_size);
  357. ret.append(buffer);
  358. snprintf(buffer, kBufferSize, " block_size_deviation: %d\n",
  359. table_options_.block_size_deviation);
  360. ret.append(buffer);
  361. snprintf(buffer, kBufferSize, " block_restart_interval: %d\n",
  362. table_options_.block_restart_interval);
  363. ret.append(buffer);
  364. snprintf(buffer, kBufferSize, " index_block_restart_interval: %d\n",
  365. table_options_.index_block_restart_interval);
  366. ret.append(buffer);
  367. snprintf(buffer, kBufferSize, " metadata_block_size: %" PRIu64 "\n",
  368. table_options_.metadata_block_size);
  369. ret.append(buffer);
  370. snprintf(buffer, kBufferSize, " partition_filters: %d\n",
  371. table_options_.partition_filters);
  372. ret.append(buffer);
  373. snprintf(buffer, kBufferSize, " use_delta_encoding: %d\n",
  374. table_options_.use_delta_encoding);
  375. ret.append(buffer);
  376. snprintf(buffer, kBufferSize, " filter_policy: %s\n",
  377. table_options_.filter_policy == nullptr
  378. ? "nullptr"
  379. : table_options_.filter_policy->Name());
  380. ret.append(buffer);
  381. snprintf(buffer, kBufferSize, " whole_key_filtering: %d\n",
  382. table_options_.whole_key_filtering);
  383. ret.append(buffer);
  384. snprintf(buffer, kBufferSize, " verify_compression: %d\n",
  385. table_options_.verify_compression);
  386. ret.append(buffer);
  387. snprintf(buffer, kBufferSize, " read_amp_bytes_per_bit: %d\n",
  388. table_options_.read_amp_bytes_per_bit);
  389. ret.append(buffer);
  390. snprintf(buffer, kBufferSize, " format_version: %d\n",
  391. table_options_.format_version);
  392. ret.append(buffer);
  393. snprintf(buffer, kBufferSize, " enable_index_compression: %d\n",
  394. table_options_.enable_index_compression);
  395. ret.append(buffer);
  396. snprintf(buffer, kBufferSize, " block_align: %d\n",
  397. table_options_.block_align);
  398. ret.append(buffer);
  399. return ret;
  400. }
  401. #ifndef ROCKSDB_LITE
  402. namespace {
  403. bool SerializeSingleBlockBasedTableOption(
  404. std::string* opt_string, const BlockBasedTableOptions& bbt_options,
  405. const std::string& name, const std::string& delimiter) {
  406. auto iter = block_based_table_type_info.find(name);
  407. if (iter == block_based_table_type_info.end()) {
  408. return false;
  409. }
  410. auto& opt_info = iter->second;
  411. const char* opt_address =
  412. reinterpret_cast<const char*>(&bbt_options) + opt_info.offset;
  413. std::string value;
  414. bool result = SerializeSingleOptionHelper(opt_address, opt_info.type, &value);
  415. if (result) {
  416. *opt_string = name + "=" + value + delimiter;
  417. }
  418. return result;
  419. }
  420. } // namespace
  421. Status BlockBasedTableFactory::GetOptionString(
  422. std::string* opt_string, const std::string& delimiter) const {
  423. assert(opt_string);
  424. opt_string->clear();
  425. for (auto iter = block_based_table_type_info.begin();
  426. iter != block_based_table_type_info.end(); ++iter) {
  427. if (iter->second.verification == OptionVerificationType::kDeprecated) {
  428. // If the option is no longer used in rocksdb and marked as deprecated,
  429. // we skip it in the serialization.
  430. continue;
  431. }
  432. std::string single_output;
  433. bool result = SerializeSingleBlockBasedTableOption(
  434. &single_output, table_options_, iter->first, delimiter);
  435. assert(result);
  436. if (result) {
  437. opt_string->append(single_output);
  438. }
  439. }
  440. return Status::OK();
  441. }
  442. #else
  443. Status BlockBasedTableFactory::GetOptionString(
  444. std::string* /*opt_string*/, const std::string& /*delimiter*/) const {
  445. return Status::OK();
  446. }
  447. #endif // !ROCKSDB_LITE
  448. const BlockBasedTableOptions& BlockBasedTableFactory::table_options() const {
  449. return table_options_;
  450. }
  451. #ifndef ROCKSDB_LITE
  452. namespace {
  453. std::string ParseBlockBasedTableOption(const std::string& name,
  454. const std::string& org_value,
  455. BlockBasedTableOptions* new_options,
  456. bool input_strings_escaped = false,
  457. bool ignore_unknown_options = false) {
  458. const std::string& value =
  459. input_strings_escaped ? UnescapeOptionString(org_value) : org_value;
  460. if (!input_strings_escaped) {
  461. // if the input string is not escaped, it means this function is
  462. // invoked from SetOptions, which takes the old format.
  463. if (name == "block_cache" || name == "block_cache_compressed") {
  464. // cache options can be specified in the following format
  465. // "block_cache={capacity=1M;num_shard_bits=4;
  466. // strict_capacity_limit=true;high_pri_pool_ratio=0.5;}"
  467. // To support backward compatibility, the following format
  468. // is also supported.
  469. // "block_cache=1M"
  470. std::shared_ptr<Cache> cache;
  471. // block_cache is specified in format block_cache=<cache_size>.
  472. if (value.find('=') == std::string::npos) {
  473. cache = NewLRUCache(ParseSizeT(value));
  474. } else {
  475. LRUCacheOptions cache_opts;
  476. if (!ParseOptionHelper(reinterpret_cast<char*>(&cache_opts),
  477. OptionType::kLRUCacheOptions, value)) {
  478. return "Invalid cache options";
  479. }
  480. cache = NewLRUCache(cache_opts);
  481. }
  482. if (name == "block_cache") {
  483. new_options->block_cache = cache;
  484. } else {
  485. new_options->block_cache_compressed = cache;
  486. }
  487. return "";
  488. } else if (name == "filter_policy") {
  489. // Expect the following format
  490. // bloomfilter:int:bool
  491. const std::string kName = "bloomfilter:";
  492. if (value.compare(0, kName.size(), kName) != 0) {
  493. return "Invalid filter policy name";
  494. }
  495. size_t pos = value.find(':', kName.size());
  496. if (pos == std::string::npos) {
  497. return "Invalid filter policy config, missing bits_per_key";
  498. }
  499. double bits_per_key =
  500. ParseDouble(trim(value.substr(kName.size(), pos - kName.size())));
  501. bool use_block_based_builder =
  502. ParseBoolean("use_block_based_builder", trim(value.substr(pos + 1)));
  503. new_options->filter_policy.reset(
  504. NewBloomFilterPolicy(bits_per_key, use_block_based_builder));
  505. return "";
  506. }
  507. }
  508. const auto iter = block_based_table_type_info.find(name);
  509. if (iter == block_based_table_type_info.end()) {
  510. if (ignore_unknown_options) {
  511. return "";
  512. } else {
  513. return "Unrecognized option";
  514. }
  515. }
  516. const auto& opt_info = iter->second;
  517. if (opt_info.verification != OptionVerificationType::kDeprecated &&
  518. !ParseOptionHelper(reinterpret_cast<char*>(new_options) + opt_info.offset,
  519. opt_info.type, value)) {
  520. return "Invalid value";
  521. }
  522. return "";
  523. }
  524. } // namespace
  525. Status GetBlockBasedTableOptionsFromString(
  526. const BlockBasedTableOptions& table_options, const std::string& opts_str,
  527. BlockBasedTableOptions* new_table_options) {
  528. std::unordered_map<std::string, std::string> opts_map;
  529. Status s = StringToMap(opts_str, &opts_map);
  530. if (!s.ok()) {
  531. return s;
  532. }
  533. return GetBlockBasedTableOptionsFromMap(table_options, opts_map,
  534. new_table_options);
  535. }
  536. Status GetBlockBasedTableOptionsFromMap(
  537. const BlockBasedTableOptions& table_options,
  538. const std::unordered_map<std::string, std::string>& opts_map,
  539. BlockBasedTableOptions* new_table_options, bool input_strings_escaped,
  540. bool ignore_unknown_options) {
  541. assert(new_table_options);
  542. *new_table_options = table_options;
  543. for (const auto& o : opts_map) {
  544. auto error_message = ParseBlockBasedTableOption(
  545. o.first, o.second, new_table_options, input_strings_escaped,
  546. ignore_unknown_options);
  547. if (error_message != "") {
  548. const auto iter = block_based_table_type_info.find(o.first);
  549. if (iter == block_based_table_type_info.end() ||
  550. !input_strings_escaped || // !input_strings_escaped indicates
  551. // the old API, where everything is
  552. // parsable.
  553. (iter->second.verification != OptionVerificationType::kByName &&
  554. iter->second.verification !=
  555. OptionVerificationType::kByNameAllowNull &&
  556. iter->second.verification !=
  557. OptionVerificationType::kByNameAllowFromNull &&
  558. iter->second.verification != OptionVerificationType::kDeprecated)) {
  559. // Restore "new_options" to the default "base_options".
  560. *new_table_options = table_options;
  561. return Status::InvalidArgument("Can't parse BlockBasedTableOptions:",
  562. o.first + " " + error_message);
  563. }
  564. }
  565. }
  566. return Status::OK();
  567. }
  568. Status VerifyBlockBasedTableFactory(
  569. const BlockBasedTableFactory* base_tf,
  570. const BlockBasedTableFactory* file_tf,
  571. OptionsSanityCheckLevel sanity_check_level) {
  572. if ((base_tf != nullptr) != (file_tf != nullptr) &&
  573. sanity_check_level > kSanityLevelNone) {
  574. return Status::Corruption(
  575. "[RocksDBOptionsParser]: Inconsistent TableFactory class type");
  576. }
  577. if (base_tf == nullptr) {
  578. return Status::OK();
  579. }
  580. assert(file_tf != nullptr);
  581. const auto& base_opt = base_tf->table_options();
  582. const auto& file_opt = file_tf->table_options();
  583. for (auto& pair : block_based_table_type_info) {
  584. if (pair.second.verification == OptionVerificationType::kDeprecated) {
  585. // We skip checking deprecated variables as they might
  586. // contain random values since they might not be initialized
  587. continue;
  588. }
  589. if (BBTOptionSanityCheckLevel(pair.first) <= sanity_check_level) {
  590. if (!AreEqualOptions(reinterpret_cast<const char*>(&base_opt),
  591. reinterpret_cast<const char*>(&file_opt),
  592. pair.second, pair.first, nullptr)) {
  593. return Status::Corruption(
  594. "[RocksDBOptionsParser]: "
  595. "failed the verification on BlockBasedTableOptions::",
  596. pair.first);
  597. }
  598. }
  599. }
  600. return Status::OK();
  601. }
  602. #endif // !ROCKSDB_LITE
  603. TableFactory* NewBlockBasedTableFactory(
  604. const BlockBasedTableOptions& _table_options) {
  605. return new BlockBasedTableFactory(_table_options);
  606. }
  607. const std::string BlockBasedTableFactory::kName = "BlockBasedTable";
  608. const std::string BlockBasedTablePropertyNames::kIndexType =
  609. "rocksdb.block.based.table.index.type";
  610. const std::string BlockBasedTablePropertyNames::kWholeKeyFiltering =
  611. "rocksdb.block.based.table.whole.key.filtering";
  612. const std::string BlockBasedTablePropertyNames::kPrefixFiltering =
  613. "rocksdb.block.based.table.prefix.filtering";
  614. const std::string kHashIndexPrefixesBlock = "rocksdb.hashindex.prefixes";
  615. const std::string kHashIndexPrefixesMetadataBlock =
  616. "rocksdb.hashindex.metadata";
  617. const std::string kPropTrue = "1";
  618. const std::string kPropFalse = "0";
  619. } // namespace ROCKSDB_NAMESPACE