sst_dump_test.cc 22 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658
  1. // Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
  2. // This source code is licensed under both the GPLv2 (found in the
  3. // COPYING file in the root directory) and Apache 2.0 License
  4. // (found in the LICENSE.Apache file in the root directory).
  5. //
  6. // Copyright (c) 2012 The LevelDB Authors. All rights reserved.
  7. // Use of this source code is governed by a BSD-style license that can be
  8. // found in the LICENSE file. See the AUTHORS file for names of contributors.
  9. #include <cstdint>
  10. #include "db/wide/wide_column_serialization.h"
  11. #include "file/random_access_file_reader.h"
  12. #include "port/stack_trace.h"
  13. #include "rocksdb/convenience.h"
  14. #include "rocksdb/filter_policy.h"
  15. #include "rocksdb/sst_dump_tool.h"
  16. #include "rocksdb/utilities/object_registry.h"
  17. #include "table/block_based/block_based_table_factory.h"
  18. #include "table/sst_file_dumper.h"
  19. #include "table/table_builder.h"
  20. #include "test_util/testharness.h"
  21. #include "test_util/testutil.h"
  22. #include "util/defer.h"
  23. namespace ROCKSDB_NAMESPACE {
  24. const uint32_t kOptLength = 1024;
  25. namespace {
  26. static std::string MakeKey(int i,
  27. ValueType value_type = ValueType::kTypeValue) {
  28. char buf[100];
  29. snprintf(buf, sizeof(buf), "k_%04d", i);
  30. InternalKey key(std::string(buf), 0, value_type);
  31. return key.Encode().ToString();
  32. }
  33. static std::string MakeKeyWithTimeStamp(int i, uint64_t ts) {
  34. char buf[100];
  35. snprintf(buf, sizeof(buf), "k_%04d", i);
  36. return test::KeyStr(ts, std::string(buf), /*seq=*/0, kTypeValue);
  37. }
  38. static std::string MakeValue(int i) {
  39. char buf[100];
  40. snprintf(buf, sizeof(buf), "v_%04d", i);
  41. InternalKey key(std::string(buf), 0, ValueType::kTypeValue);
  42. return key.Encode().ToString();
  43. }
  44. static std::string MakeWideColumn(int i) {
  45. std::string val = MakeValue(i);
  46. std::string val1 = "attr_1_val_" + val;
  47. std::string val2 = "attr_2_val_" + val;
  48. WideColumns columns{{"attr_1", val1}, {"attr_2", val2}};
  49. std::string entity;
  50. EXPECT_OK(WideColumnSerialization::Serialize(columns, entity));
  51. return entity;
  52. }
  53. void cleanup(const Options& opts, const std::string& file_name) {
  54. Env* env = opts.env;
  55. ASSERT_OK(env->DeleteFile(file_name));
  56. std::string outfile_name = file_name.substr(0, file_name.length() - 4);
  57. outfile_name.append("_dump.txt");
  58. env->DeleteFile(outfile_name).PermitUncheckedError();
  59. }
  60. } // namespace
  61. // Test for sst dump tool "raw" mode
  62. class SSTDumpToolTest : public testing::Test {
  63. std::string test_dir_;
  64. Env* env_;
  65. std::shared_ptr<Env> env_guard_;
  66. public:
  67. SSTDumpToolTest() : env_(Env::Default()) {
  68. EXPECT_OK(test::CreateEnvFromSystem(ConfigOptions(), &env_, &env_guard_));
  69. test_dir_ = test::PerThreadDBPath(env_, "sst_dump_test_db");
  70. Status s = env_->CreateDirIfMissing(test_dir_);
  71. EXPECT_OK(s);
  72. }
  73. ~SSTDumpToolTest() override {
  74. if (getenv("KEEP_DB")) {
  75. fprintf(stdout, "Data is still at %s\n", test_dir_.c_str());
  76. } else {
  77. EXPECT_OK(env_->DeleteDir(test_dir_));
  78. }
  79. }
  80. Env* env() { return env_; }
  81. std::string MakeFilePath(const std::string& file_name) const {
  82. std::string path(test_dir_);
  83. path.append("/").append(file_name);
  84. return path;
  85. }
  86. // RAII class to ensure cleanup of usage array
  87. template <std::size_t N>
  88. struct CleanupUsage {
  89. char* (&usage)[N];
  90. explicit CleanupUsage(char* (&_usage)[N]) : usage(_usage) {}
  91. // No copies/moves
  92. CleanupUsage(const CleanupUsage&) = delete;
  93. CleanupUsage& operator=(const CleanupUsage&) = delete;
  94. CleanupUsage(CleanupUsage&&) = delete;
  95. CleanupUsage& operator=(CleanupUsage&&) = delete;
  96. ~CleanupUsage() {
  97. for (std::size_t i = 0; i < N; ++i) {
  98. delete[] usage[i];
  99. }
  100. }
  101. };
  102. #define ASSERT_TOOL_PASS(tool_expr) ASSERT_EQ(0, (tool_expr));
  103. #define ASSERT_TOOL_FAIL(tool_expr) ASSERT_NE(0, (tool_expr));
  104. template <std::size_t N>
  105. void PopulateCommandArgs(const std::string& file_path, const char* command,
  106. char* (&usage)[N]) const {
  107. for (int i = 0; i < static_cast<int>(N); ++i) {
  108. usage[i] = new char[kOptLength];
  109. }
  110. snprintf(usage[0], kOptLength, "./sst_dump");
  111. snprintf(usage[1], kOptLength, "%s", command);
  112. snprintf(usage[2], kOptLength, "--file=%s", file_path.c_str());
  113. }
  114. void createSST(const Options& opts, const std::string& file_name,
  115. uint32_t wide_column_one_in = 0, bool range_del = false) {
  116. Env* test_env = opts.env;
  117. FileOptions file_options(opts);
  118. ReadOptions read_options;
  119. const ImmutableOptions imoptions(opts);
  120. const MutableCFOptions moptions(opts);
  121. ROCKSDB_NAMESPACE::InternalKeyComparator ikc(opts.comparator);
  122. std::unique_ptr<TableBuilder> tb;
  123. InternalTblPropCollFactories internal_tbl_prop_coll_factories;
  124. std::unique_ptr<WritableFileWriter> file_writer;
  125. ASSERT_OK(WritableFileWriter::Create(test_env->GetFileSystem(), file_name,
  126. file_options, &file_writer, nullptr));
  127. std::string column_family_name;
  128. int unknown_level = -1;
  129. const WriteOptions write_options;
  130. tb.reset(opts.table_factory->NewTableBuilder(
  131. TableBuilderOptions(
  132. imoptions, moptions, read_options, write_options, ikc,
  133. &internal_tbl_prop_coll_factories, CompressionType::kNoCompression,
  134. CompressionOptions(),
  135. TablePropertiesCollectorFactory::Context::kUnknownColumnFamily,
  136. column_family_name, unknown_level, kUnknownNewestKeyTime),
  137. file_writer.get()));
  138. // Populate slightly more than 1K keys
  139. uint32_t num_keys = kNumKey;
  140. const char* comparator_name = ikc.user_comparator()->Name();
  141. if (strcmp(comparator_name, ReverseBytewiseComparator()->Name()) == 0) {
  142. for (int32_t i = num_keys; i > 0; i--) {
  143. if (wide_column_one_in == 0 || i % wide_column_one_in != 0) {
  144. tb->Add(MakeKey(i), MakeValue(i));
  145. } else {
  146. tb->Add(MakeKey(i, ValueType::kTypeWideColumnEntity),
  147. MakeWideColumn(i));
  148. }
  149. }
  150. } else if (strcmp(comparator_name,
  151. test::BytewiseComparatorWithU64TsWrapper()->Name()) ==
  152. 0) {
  153. for (uint32_t i = 0; i < num_keys; i++) {
  154. tb->Add(MakeKeyWithTimeStamp(i, 100 + i), MakeValue(i));
  155. }
  156. } else {
  157. uint32_t i = 0;
  158. if (range_del) {
  159. tb->Add(MakeKey(i, kTypeRangeDeletion), MakeValue(i + 1));
  160. i = 1;
  161. }
  162. for (; i < num_keys; i++) {
  163. if (wide_column_one_in == 0 || i % wide_column_one_in != 0) {
  164. tb->Add(MakeKey(i), MakeValue(i));
  165. } else {
  166. tb->Add(MakeKey(i, ValueType::kTypeWideColumnEntity),
  167. MakeWideColumn(i));
  168. }
  169. }
  170. }
  171. ASSERT_OK(tb->Finish());
  172. ASSERT_OK(file_writer->Close(IOOptions()));
  173. }
  174. protected:
  175. constexpr static int kNumKey = 1024;
  176. void SSTDumpToolTestCase(Options& opts, bool filter, int wide_column_one_in,
  177. const char* cmd_arg) {
  178. opts.env = env();
  179. BlockBasedTableOptions table_opts;
  180. if (filter) {
  181. table_opts.filter_policy.reset(
  182. ROCKSDB_NAMESPACE::NewBloomFilterPolicy(10));
  183. }
  184. opts.table_factory.reset(new BlockBasedTableFactory(table_opts));
  185. std::string file_path = MakeFilePath("rocksdb_sst_test.sst");
  186. createSST(opts, file_path, wide_column_one_in);
  187. char* usage[3];
  188. auto cleanup_usage = CleanupUsage{usage};
  189. PopulateCommandArgs(file_path, cmd_arg, usage);
  190. ROCKSDB_NAMESPACE::SSTDumpTool tool;
  191. ASSERT_TOOL_PASS(tool.Run(3, usage, opts));
  192. cleanup(opts, file_path);
  193. }
  194. };
  195. TEST_F(SSTDumpToolTest, HelpAndVersion) {
  196. Options opts;
  197. opts.env = env();
  198. ROCKSDB_NAMESPACE::SSTDumpTool tool;
  199. static const char* help[] = {"./sst_dump", "--help"};
  200. ASSERT_TOOL_PASS(tool.Run(2, help, opts));
  201. static const char* bad_help[] = {"./sst_dump", "--", "--help"};
  202. ASSERT_TOOL_FAIL(tool.Run(3, bad_help, opts));
  203. static const char* version[] = {"./sst_dump", "--version"};
  204. ASSERT_TOOL_PASS(tool.Run(2, version, opts));
  205. static const char* bad[] = {"./sst_dump", "--not_an_option"};
  206. ASSERT_TOOL_FAIL(tool.Run(2, bad, opts));
  207. }
  208. TEST_F(SSTDumpToolTest, EmptyFilter) {
  209. Options opts;
  210. SSTDumpToolTestCase(opts, /*filter=*/false, /*wide_column_one_in=*/10,
  211. "--command=raw");
  212. }
  213. TEST_F(SSTDumpToolTest, SstDumpReverseBytewiseComparator) {
  214. Options opts;
  215. opts.comparator = ReverseBytewiseComparator();
  216. SSTDumpToolTestCase(opts, /*filter=*/true, /*wide_column_one_in=*/10,
  217. "--command=raw");
  218. }
  219. TEST_F(SSTDumpToolTest, SstDumpComparatorWithU64Ts) {
  220. Options opts;
  221. opts.comparator = test::BytewiseComparatorWithU64TsWrapper();
  222. SSTDumpToolTestCase(opts, /*filter=*/true, /*wide_column_one_in=*/10,
  223. "--command=raw");
  224. }
  225. TEST_F(SSTDumpToolTest, FilterBlockWideColumn) {
  226. Options opts;
  227. SSTDumpToolTestCase(opts, /*filter=*/true, /*wide_column_one_in=*/10,
  228. "--command=raw");
  229. }
  230. TEST_F(SSTDumpToolTest, FilterBlock) {
  231. Options opts;
  232. SSTDumpToolTestCase(opts, /*filter=*/true, /*wide_column_one_in=*/0,
  233. "--command=raw");
  234. }
  235. TEST_F(SSTDumpToolTest, GetProperties) {
  236. Options opts;
  237. SSTDumpToolTestCase(opts, /*filter=*/true, /*wide_column_one_in=*/0,
  238. "--show_properties");
  239. }
  240. TEST_F(SSTDumpToolTest, CompressedSizes) {
  241. Options opts;
  242. SSTDumpToolTestCase(opts, /*filter=*/true, /*wide_column_one_in=*/10,
  243. "--command=recompress");
  244. }
  245. TEST_F(SSTDumpToolTest, ListMetaBlocks) {
  246. Options opts;
  247. SSTDumpToolTestCase(opts, /*filter=*/true, /*wide_column_one_in=*/0,
  248. "--list_meta_blocks");
  249. }
  250. namespace {
  251. using Compressor8A = test::CompressorCustomAlg<kCustomCompression8A>;
  252. class MyManager : public CompressionManager {
  253. public:
  254. static constexpr const char* kCompatibilityName = "SSTDumpToolTest:MyManager";
  255. const char* Name() const override { return kCompatibilityName; }
  256. const char* CompatibilityName() const override { return kCompatibilityName; }
  257. bool SupportsCompressionType(CompressionType type) const override {
  258. return type == kCustomCompression8A;
  259. }
  260. std::unique_ptr<Compressor> GetCompressor(const CompressionOptions& /*opts*/,
  261. CompressionType type) override {
  262. switch (static_cast<unsigned char>(type)) {
  263. case kCustomCompression8A:
  264. return std::make_unique<Compressor8A>();
  265. default:
  266. return nullptr;
  267. }
  268. }
  269. std::shared_ptr<Decompressor> GetDecompressor() override {
  270. return std::make_shared<test::DecompressorCustomAlg>();
  271. }
  272. };
  273. } // namespace
  274. TEST_F(SSTDumpToolTest, CompressionManager) {
  275. if (!Compressor8A::Supported()) {
  276. fprintf(stderr,
  277. "Prerequisite compression library not supported. Skipping\n");
  278. return;
  279. }
  280. // Registery in ObjectLibrary to check that sst_dump can use named
  281. // CompressionManagers with dependency injection
  282. auto& library = *ObjectLibrary::Default();
  283. library.AddFactory<CompressionManager>(
  284. MyManager::kCompatibilityName,
  285. [](const std::string& /*uri*/, std::unique_ptr<CompressionManager>* guard,
  286. std::string* /*errmsg*/) {
  287. *guard = std::make_unique<MyManager>();
  288. return guard->get();
  289. });
  290. Options opts;
  291. opts.env = env();
  292. BlockBasedTableOptions table_opts;
  293. table_opts.filter_policy.reset(
  294. ROCKSDB_NAMESPACE::NewBloomFilterPolicy(10, false));
  295. opts.table_factory.reset(new BlockBasedTableFactory(table_opts));
  296. std::string file_path = MakeFilePath("rocksdb_sst_test.sst");
  297. createSST(opts, file_path, 10);
  298. char* usage[5];
  299. auto cleanup_usage = CleanupUsage{usage};
  300. PopulateCommandArgs(file_path, "--command=recompress", usage);
  301. snprintf(usage[3], kOptLength, "--compression_manager=%s",
  302. MyManager::kCompatibilityName);
  303. snprintf(usage[4], kOptLength, "--compression_types=kCustomCompression8A");
  304. ROCKSDB_NAMESPACE::SSTDumpTool tool;
  305. ASSERT_TOOL_PASS(tool.Run(5, usage, opts));
  306. cleanup(opts, file_path);
  307. }
  308. TEST_F(SSTDumpToolTest, MemEnv) {
  309. std::unique_ptr<Env> mem_env(NewMemEnv(env()));
  310. Options opts;
  311. opts.env = mem_env.get();
  312. std::string file_path = MakeFilePath("rocksdb_sst_test.sst");
  313. createSST(opts, file_path);
  314. char* usage[3];
  315. auto cleanup_usage = CleanupUsage{usage};
  316. PopulateCommandArgs(file_path, "--command=verify_checksum", usage);
  317. ROCKSDB_NAMESPACE::SSTDumpTool tool;
  318. ASSERT_TOOL_PASS(tool.Run(3, usage, opts));
  319. cleanup(opts, file_path);
  320. }
  321. TEST_F(SSTDumpToolTest, ReadaheadSize) {
  322. Options opts;
  323. opts.env = env();
  324. std::string file_path = MakeFilePath("rocksdb_sst_test.sst");
  325. createSST(opts, file_path);
  326. char* usage[4];
  327. auto cleanup_usage = CleanupUsage{usage};
  328. PopulateCommandArgs(file_path, "--command=verify", usage);
  329. snprintf(usage[3], kOptLength, "--readahead_size=4000000");
  330. int num_reads = 0;
  331. SyncPoint::GetInstance()->SetCallBack("RandomAccessFileReader::Read",
  332. [&](void*) { num_reads++; });
  333. SyncPoint::GetInstance()->EnableProcessing();
  334. SSTDumpTool tool;
  335. ASSERT_TOOL_PASS(tool.Run(4, usage, opts));
  336. // The file is approximately 10MB. Readahead is 4MB.
  337. // We usually need 3 reads + one metadata read.
  338. // Three extra read is needed before opening the file for metadata.
  339. ASSERT_EQ(7, num_reads);
  340. SyncPoint::GetInstance()->ClearAllCallBacks();
  341. SyncPoint::GetInstance()->DisableProcessing();
  342. cleanup(opts, file_path);
  343. }
  344. #ifndef __clang_analyzer__ // False positive memory leaks reported
  345. TEST_F(SSTDumpToolTest, NoSstFile) {
  346. Options opts;
  347. opts.env = env();
  348. std::string file_path = MakeFilePath("no_such_file.sst");
  349. char* usage[3];
  350. auto cleanup_usage = CleanupUsage{usage};
  351. PopulateCommandArgs(file_path, "", usage);
  352. ROCKSDB_NAMESPACE::SSTDumpTool tool;
  353. for (const auto& command :
  354. {"--command=check", "--command=dump", "--command=raw",
  355. "--command=verify", "--command=recompress", "--command=verify_checksum",
  356. "--show_properties"}) {
  357. snprintf(usage[1], kOptLength, "%s", command);
  358. ASSERT_TOOL_FAIL(tool.Run(3, usage, opts));
  359. }
  360. }
  361. TEST_F(SSTDumpToolTest, ValidSSTPath) {
  362. Options opts;
  363. opts.env = env();
  364. char* usage[5];
  365. auto cleanup_usage = CleanupUsage{usage};
  366. PopulateCommandArgs("", "", usage);
  367. SSTDumpTool tool;
  368. std::string file_not_exists = MakeFilePath("file_not_exists.sst");
  369. std::string sst_file = MakeFilePath("rocksdb_sst_test.sst");
  370. createSST(opts, sst_file);
  371. std::string text_file = MakeFilePath("text_file");
  372. ASSERT_OK(WriteStringToFile(opts.env, "Hello World!", text_file, false));
  373. std::string fake_sst = MakeFilePath("fake_sst.sst");
  374. ASSERT_OK(WriteStringToFile(opts.env, "Not an SST file!", fake_sst, false));
  375. std::string good_dir = MakeFilePath("");
  376. for (const auto& command_arg : {"--command=verify", "--command=identify"}) {
  377. snprintf(usage[1], kOptLength, "%s", command_arg);
  378. // Test both classic --file and standalone argument
  379. for (const auto& file_fmt : {"--file=%s", "%s"}) {
  380. snprintf(usage[2], kOptLength, file_fmt, file_not_exists.c_str());
  381. ASSERT_TOOL_FAIL(tool.Run(3, usage, opts));
  382. snprintf(usage[2], kOptLength, file_fmt, sst_file.c_str());
  383. ASSERT_TOOL_PASS(tool.Run(3, usage, opts));
  384. snprintf(usage[2], kOptLength, file_fmt, good_dir.c_str());
  385. ASSERT_TOOL_PASS(tool.Run(3, usage, opts));
  386. snprintf(usage[2], kOptLength, file_fmt, text_file.c_str());
  387. ASSERT_TOOL_FAIL(tool.Run(3, usage, opts));
  388. snprintf(usage[2], kOptLength, file_fmt, fake_sst.c_str());
  389. ASSERT_TOOL_FAIL(tool.Run(3, usage, opts));
  390. }
  391. // If one file is valid, that's enough to succeed as long as the others
  392. // exist
  393. for (const auto& good : {sst_file, good_dir}) {
  394. // Additional file-or-dir argument
  395. snprintf(usage[3], kOptLength, "%s", good.c_str());
  396. snprintf(usage[2], kOptLength, "%s", file_not_exists.c_str());
  397. ASSERT_TOOL_FAIL(tool.Run(4, usage, opts));
  398. snprintf(usage[2], kOptLength, "%s", sst_file.c_str());
  399. ASSERT_TOOL_PASS(tool.Run(4, usage, opts));
  400. snprintf(usage[2], kOptLength, "%s", good_dir.c_str());
  401. ASSERT_TOOL_PASS(tool.Run(4, usage, opts));
  402. snprintf(usage[2], kOptLength, "%s", text_file.c_str());
  403. // DIFFERENT
  404. ASSERT_TOOL_PASS(tool.Run(4, usage, opts));
  405. snprintf(usage[2], kOptLength, "%s", fake_sst.c_str());
  406. // DIFFERENT
  407. ASSERT_TOOL_PASS(tool.Run(4, usage, opts));
  408. // Some extra cases to test "--" handling
  409. snprintf(usage[2], kOptLength, "%s", "--");
  410. ASSERT_TOOL_PASS(tool.Run(4, usage, opts));
  411. snprintf(usage[4], kOptLength, "%s", file_not_exists.c_str());
  412. ASSERT_TOOL_FAIL(tool.Run(5, usage, opts));
  413. snprintf(usage[4], kOptLength, "%s", fake_sst.c_str());
  414. ASSERT_TOOL_PASS(tool.Run(5, usage, opts));
  415. }
  416. }
  417. ASSERT_OK(opts.env->DeleteFile(sst_file));
  418. ASSERT_OK(opts.env->DeleteFile(text_file));
  419. ASSERT_OK(opts.env->DeleteFile(fake_sst));
  420. }
  421. #endif // __clang_analyzer__
  422. TEST_F(SSTDumpToolTest, RawOutput) {
  423. Options opts;
  424. opts.env = env();
  425. std::string file_path = MakeFilePath("rocksdb_sst_test.sst");
  426. createSST(opts, file_path, 10);
  427. char* usage[3];
  428. auto cleanup_usage = CleanupUsage{usage};
  429. PopulateCommandArgs(file_path, "--command=raw", usage);
  430. ROCKSDB_NAMESPACE::SSTDumpTool tool;
  431. ASSERT_TOOL_PASS(tool.Run(3, usage, opts));
  432. const std::string raw_path = MakeFilePath("rocksdb_sst_test_dump.txt");
  433. std::ifstream raw_file(raw_path);
  434. std::string tp;
  435. bool is_data_block = false;
  436. int key_count = 0;
  437. while (getline(raw_file, tp)) {
  438. if (tp.find("Data Block #") != std::string::npos) {
  439. is_data_block = true;
  440. }
  441. if (is_data_block && tp.find("HEX") != std::string::npos) {
  442. key_count++;
  443. }
  444. }
  445. ASSERT_EQ(kNumKey, key_count);
  446. raw_file.close();
  447. cleanup(opts, file_path);
  448. }
  449. TEST_F(SSTDumpToolTest, SstFileDumperMmapReads) {
  450. Options opts;
  451. opts.env = env();
  452. std::string file_path = MakeFilePath("rocksdb_sst_test.sst");
  453. createSST(opts, file_path, 10);
  454. EnvOptions env_opts;
  455. uint64_t data_size = 0;
  456. // Test all combinations of mmap read options
  457. for (int i = 0; i < 4; ++i) {
  458. SaveAndRestore<bool> sar_opts(&opts.allow_mmap_reads, (i & 1) != 0);
  459. SaveAndRestore<bool> sar_env_opts(&env_opts.use_mmap_reads, (i & 2) != 0);
  460. SstFileDumper dumper(opts, file_path, Temperature::kUnknown,
  461. 1024 /*readahead_size*/, true /*verify_checksum*/,
  462. false /*output_hex*/, false /*decode_blob_index*/,
  463. env_opts);
  464. ASSERT_OK(dumper.getStatus());
  465. std::shared_ptr<const TableProperties> tp;
  466. ASSERT_OK(dumper.ReadTableProperties(&tp));
  467. ASSERT_NE(tp.get(), nullptr);
  468. if (i == 0) {
  469. // Verify consistency of a populated field with some entropy
  470. data_size = tp->data_size;
  471. ASSERT_GT(data_size, 0);
  472. } else {
  473. ASSERT_EQ(data_size, tp->data_size);
  474. }
  475. }
  476. cleanup(opts, file_path);
  477. }
  478. TEST_F(SSTDumpToolTest, SstFileDumperVerifyNumRecords) {
  479. Options opts;
  480. opts.env = env();
  481. EnvOptions env_opts;
  482. std::string file_path = MakeFilePath("rocksdb_sst_test.sst");
  483. {
  484. createSST(opts, file_path, 10);
  485. SstFileDumper dumper(opts, file_path, Temperature::kUnknown,
  486. 1024 /*readahead_size*/, true /*verify_checksum*/,
  487. false /*output_hex*/, false /*decode_blob_index*/,
  488. env_opts, /*silent=*/true);
  489. ASSERT_OK(dumper.getStatus());
  490. ASSERT_OK(dumper.ReadSequential(
  491. /*print_kv=*/false,
  492. /*read_num_limit=*/std::numeric_limits<uint64_t>::max(),
  493. /*has_from=*/false, /*from_key=*/"",
  494. /*has_to=*/false, /*to_key=*/""));
  495. cleanup(opts, file_path);
  496. }
  497. {
  498. // Test with range del
  499. createSST(opts, file_path, 10, /*range_del=*/true);
  500. SstFileDumper dumper(opts, file_path, Temperature::kUnknown,
  501. 1024 /*readahead_size*/, true /*verify_checksum*/,
  502. false /*output_hex*/, false /*decode_blob_index*/,
  503. env_opts, /*silent=*/true);
  504. ASSERT_OK(dumper.getStatus());
  505. ASSERT_OK(dumper.ReadSequential(
  506. /*print_kv=*/false,
  507. /*read_num_limit=*/std::numeric_limits<uint64_t>::max(),
  508. /*has_from=*/false, /*from_key=*/"",
  509. /*has_to=*/false, /*to_key=*/""));
  510. cleanup(opts, file_path);
  511. }
  512. {
  513. SyncPoint::GetInstance()->SetCallBack(
  514. "PropertyBlockBuilder::AddTableProperty:Start", [&](void* arg) {
  515. TableProperties* props = reinterpret_cast<TableProperties*>(arg);
  516. props->num_entries = kNumKey + 2;
  517. });
  518. SyncPoint::GetInstance()->EnableProcessing();
  519. createSST(opts, file_path, 10);
  520. SstFileDumper dumper(opts, file_path, Temperature::kUnknown,
  521. 1024 /*readahead_size*/, true /*verify_checksum*/,
  522. false /*output_hex*/, false /*decode_blob_index*/,
  523. env_opts, /*silent=*/true);
  524. ASSERT_OK(dumper.getStatus());
  525. Status s = dumper.ReadSequential(
  526. /*print_kv=*/false,
  527. /*read_num_limit==*/std::numeric_limits<uint64_t>::max(),
  528. /*has_from=*/false, /*from_key=*/"",
  529. /*has_to=*/false, /*to_key=*/"");
  530. ASSERT_TRUE(s.IsCorruption());
  531. ASSERT_TRUE(
  532. std::strstr("Table property expects 1026 entries when excluding range "
  533. "deletions, but scanning the table returned 1024 entries",
  534. s.getState()));
  535. // Validation is not performed when read_num, has_from, has_to are set
  536. ASSERT_OK(dumper.ReadSequential(
  537. /*print_kv=*/false, /*read_num_limit=*/10,
  538. /*has_from=*/false, /*from_key=*/"",
  539. /*has_to=*/false, /*to_key=*/""));
  540. ASSERT_OK(dumper.ReadSequential(
  541. /*print_kv=*/false,
  542. /*read_num_limit=*/std::numeric_limits<uint64_t>::max(),
  543. /*has_from=*/true, /*from_key=*/MakeKey(100),
  544. /*has_to=*/false, /*to_key=*/""));
  545. ASSERT_OK(dumper.ReadSequential(
  546. /*print_kv=*/false,
  547. /*read_num_limit=*/std::numeric_limits<uint64_t>::max(),
  548. /*has_from=*/false, /*from_key=*/"",
  549. /*has_to=*/true, /*to_key=*/MakeKey(100)));
  550. cleanup(opts, file_path);
  551. }
  552. }
  553. } // namespace ROCKSDB_NAMESPACE
  554. int main(int argc, char** argv) {
  555. ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
  556. ::testing::InitGoogleTest(&argc, argv);
  557. RegisterCustomObjects(argc, argv);
  558. return RUN_ALL_TESTS();
  559. }