ldb_cmd_test.cc 40 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264
  1. // Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
  2. // This source code is licensed under both the GPLv2 (found in the
  3. // COPYING file in the root directory) and Apache 2.0 License
  4. // (found in the LICENSE.Apache file in the root directory).
  5. //
  6. #include "rocksdb/utilities/ldb_cmd.h"
  7. #include <cinttypes>
  8. #include <iomanip>
  9. #include "db/db_test_util.h"
  10. #include "db/version_edit.h"
  11. #include "db/version_set.h"
  12. #include "env/composite_env_wrapper.h"
  13. #include "file/filename.h"
  14. #include "port/stack_trace.h"
  15. #include "rocksdb/advanced_options.h"
  16. #include "rocksdb/comparator.h"
  17. #include "rocksdb/convenience.h"
  18. #include "rocksdb/db.h"
  19. #include "rocksdb/file_checksum.h"
  20. #include "rocksdb/file_system.h"
  21. #include "rocksdb/utilities/options_util.h"
  22. #include "test_util/sync_point.h"
  23. #include "test_util/testharness.h"
  24. #include "test_util/testutil.h"
  25. #include "util/file_checksum_helper.h"
  26. #include "util/random.h"
  27. using std::map;
  28. using std::string;
  29. using std::vector;
  30. namespace ROCKSDB_NAMESPACE {
  31. class LdbCmdTest : public testing::Test {
  32. public:
  33. LdbCmdTest() : testing::Test() {}
  34. Env* TryLoadCustomOrDefaultEnv() {
  35. Env* env = Env::Default();
  36. EXPECT_OK(test::CreateEnvFromSystem(ConfigOptions(), &env, &env_guard_));
  37. return env;
  38. }
  39. private:
  40. std::shared_ptr<Env> env_guard_;
  41. };
  42. TEST_F(LdbCmdTest, HelpAndVersion) {
  43. Options o;
  44. o.env = TryLoadCustomOrDefaultEnv();
  45. LDBOptions lo;
  46. static const char* help[] = {"./ldb", "--help"};
  47. ASSERT_EQ(0, LDBCommandRunner::RunCommand(2, help, o, lo, nullptr));
  48. static const char* version[] = {"./ldb", "--version"};
  49. ASSERT_EQ(0, LDBCommandRunner::RunCommand(2, version, o, lo, nullptr));
  50. static const char* bad[] = {"./ldb", "--not_an_option"};
  51. ASSERT_NE(0, LDBCommandRunner::RunCommand(2, bad, o, lo, nullptr));
  52. }
  53. TEST_F(LdbCmdTest, HexToString) {
  54. // map input to expected outputs.
  55. // odd number of "hex" half bytes doesn't make sense
  56. map<string, vector<int>> inputMap = {
  57. {"0x07", {7}}, {"0x5050", {80, 80}}, {"0xFF", {-1}},
  58. {"0x1234", {18, 52}}, {"0xaaAbAC", {-86, -85, -84}}, {"0x1203", {18, 3}},
  59. };
  60. for (const auto& inPair : inputMap) {
  61. auto actual = ROCKSDB_NAMESPACE::LDBCommand::HexToString(inPair.first);
  62. auto expected = inPair.second;
  63. for (unsigned int i = 0; i < actual.length(); i++) {
  64. EXPECT_EQ(expected[i], static_cast<int>((signed char)actual[i]));
  65. }
  66. auto reverse = ROCKSDB_NAMESPACE::LDBCommand::StringToHex(actual);
  67. EXPECT_STRCASEEQ(inPair.first.c_str(), reverse.c_str());
  68. }
  69. }
  70. TEST_F(LdbCmdTest, HexToStringBadInputs) {
  71. const vector<string> badInputs = {
  72. "0xZZ", "123", "0xx5", "0x111G", "0x123", "Ox12", "0xT", "0x1Q1",
  73. };
  74. for (const auto& badInput : badInputs) {
  75. try {
  76. ROCKSDB_NAMESPACE::LDBCommand::HexToString(badInput);
  77. std::cerr << "Should fail on bad hex value: " << badInput << "\n";
  78. FAIL();
  79. } catch (...) {
  80. }
  81. }
  82. }
  83. TEST_F(LdbCmdTest, MemEnv) {
  84. Env* base_env = TryLoadCustomOrDefaultEnv();
  85. std::unique_ptr<Env> env(NewMemEnv(base_env));
  86. Options opts;
  87. opts.env = env.get();
  88. opts.create_if_missing = true;
  89. DB* db = nullptr;
  90. std::string dbname = test::PerThreadDBPath(env.get(), "ldb_cmd_test");
  91. ASSERT_OK(DB::Open(opts, dbname, &db));
  92. WriteOptions wopts;
  93. for (int i = 0; i < 100; i++) {
  94. char buf[16];
  95. snprintf(buf, sizeof(buf), "%08d", i);
  96. ASSERT_OK(db->Put(wopts, buf, buf));
  97. }
  98. FlushOptions fopts;
  99. fopts.wait = true;
  100. ASSERT_OK(db->Flush(fopts));
  101. delete db;
  102. char arg1[] = "./ldb";
  103. char arg2[1024];
  104. snprintf(arg2, sizeof(arg2), "--db=%s", dbname.c_str());
  105. char arg3[] = "dump_live_files";
  106. char* argv[] = {arg1, arg2, arg3};
  107. ASSERT_EQ(0,
  108. LDBCommandRunner::RunCommand(3, argv, opts, LDBOptions(), nullptr));
  109. }
  110. class FileChecksumTestHelper {
  111. private:
  112. Options options_;
  113. DB* db_;
  114. std::string dbname_;
  115. Status VerifyChecksum(LiveFileMetaData& file_meta) {
  116. std::string cur_checksum;
  117. std::string checksum_func_name;
  118. Status s;
  119. EnvOptions soptions;
  120. std::unique_ptr<SequentialFile> file_reader;
  121. std::string file_path = dbname_ + "/" + file_meta.name;
  122. s = options_.env->NewSequentialFile(file_path, &file_reader, soptions);
  123. if (!s.ok()) {
  124. return s;
  125. }
  126. std::unique_ptr<char[]> scratch(new char[2048]);
  127. Slice result;
  128. FileChecksumGenFactory* file_checksum_gen_factory =
  129. options_.file_checksum_gen_factory.get();
  130. if (file_checksum_gen_factory == nullptr) {
  131. cur_checksum = kUnknownFileChecksum;
  132. checksum_func_name = kUnknownFileChecksumFuncName;
  133. } else {
  134. FileChecksumGenContext gen_context;
  135. gen_context.file_name = file_meta.name;
  136. std::unique_ptr<FileChecksumGenerator> file_checksum_gen =
  137. file_checksum_gen_factory->CreateFileChecksumGenerator(gen_context);
  138. checksum_func_name = file_checksum_gen->Name();
  139. s = file_reader->Read(2048, &result, scratch.get());
  140. if (!s.ok()) {
  141. return s;
  142. }
  143. while (result.size() != 0) {
  144. file_checksum_gen->Update(scratch.get(), result.size());
  145. s = file_reader->Read(2048, &result, scratch.get());
  146. if (!s.ok()) {
  147. return s;
  148. }
  149. }
  150. file_checksum_gen->Finalize();
  151. cur_checksum = file_checksum_gen->GetChecksum();
  152. }
  153. std::string stored_checksum = file_meta.file_checksum;
  154. std::string stored_checksum_func_name = file_meta.file_checksum_func_name;
  155. if ((cur_checksum != stored_checksum) ||
  156. (checksum_func_name != stored_checksum_func_name)) {
  157. return Status::Corruption(
  158. "Checksum does not match! The file: " + file_meta.name +
  159. ", checksum name: " + stored_checksum_func_name + " and checksum " +
  160. stored_checksum + ". However, expected checksum name: " +
  161. checksum_func_name + " and checksum " + cur_checksum);
  162. }
  163. return Status::OK();
  164. }
  165. public:
  166. FileChecksumTestHelper(Options& options, DB* db, std::string db_name)
  167. : options_(options), db_(db), dbname_(db_name) {}
  168. ~FileChecksumTestHelper() = default;
  169. // Verify the checksum information in Manifest.
  170. Status VerifyChecksumInManifest(
  171. const std::vector<LiveFileMetaData>& live_files) {
  172. // Step 1: verify if the dbname_ is correct
  173. if (dbname_.back() != '/') {
  174. dbname_.append("/");
  175. }
  176. // Step 2, get the the checksum information by recovering the VersionSet
  177. // from Manifest.
  178. std::unique_ptr<FileChecksumList> checksum_list(NewFileChecksumList());
  179. EnvOptions sopt;
  180. std::shared_ptr<Cache> tc(NewLRUCache(options_.max_open_files - 10,
  181. options_.table_cache_numshardbits));
  182. options_.db_paths.emplace_back(dbname_, 0);
  183. options_.num_levels = 64;
  184. WriteController wc(options_.delayed_write_rate);
  185. WriteBufferManager wb(options_.db_write_buffer_size);
  186. ImmutableDBOptions immutable_db_options(options_);
  187. VersionSet versions(dbname_, &immutable_db_options, sopt, tc.get(), &wb,
  188. &wc, nullptr, nullptr, "", "",
  189. options_.daily_offpeak_time_utc, nullptr,
  190. /*read_only=*/false);
  191. std::vector<std::string> cf_name_list;
  192. Status s;
  193. s = versions.ListColumnFamilies(&cf_name_list, dbname_,
  194. immutable_db_options.fs.get());
  195. if (s.ok()) {
  196. std::vector<ColumnFamilyDescriptor> cf_list;
  197. for (const auto& name : cf_name_list) {
  198. fprintf(stdout, "cf_name: %s", name.c_str());
  199. cf_list.emplace_back(name, ColumnFamilyOptions(options_));
  200. }
  201. s = versions.Recover(cf_list, true);
  202. }
  203. if (s.ok()) {
  204. s = versions.GetLiveFilesChecksumInfo(checksum_list.get());
  205. }
  206. if (!s.ok()) {
  207. return s;
  208. }
  209. // Step 3 verify the checksum
  210. if (live_files.size() != checksum_list->size()) {
  211. return Status::Corruption("The number of files does not match!");
  212. }
  213. for (size_t i = 0; i < live_files.size(); i++) {
  214. std::string stored_checksum;
  215. std::string stored_func_name;
  216. s = checksum_list->SearchOneFileChecksum(
  217. live_files[i].file_number, &stored_checksum, &stored_func_name);
  218. if (s.IsNotFound()) {
  219. return s;
  220. }
  221. if (live_files[i].file_checksum != stored_checksum ||
  222. live_files[i].file_checksum_func_name != stored_func_name) {
  223. return Status::Corruption(
  224. "Checksum does not match! The file: " +
  225. std::to_string(live_files[i].file_number) +
  226. ". In Manifest, checksum name: " + stored_func_name +
  227. " and checksum " + stored_checksum +
  228. ". However, expected checksum name: " +
  229. live_files[i].file_checksum_func_name + " and checksum " +
  230. live_files[i].file_checksum);
  231. }
  232. }
  233. return Status::OK();
  234. }
  235. // Verify the checksum of each file by recalculting the checksum and
  236. // comparing it with the one being generated when a SST file is created.
  237. Status VerifyEachFileChecksum() {
  238. assert(db_ != nullptr);
  239. EXPECT_OK(db_->DisableFileDeletions());
  240. std::vector<LiveFileMetaData> live_files;
  241. db_->GetLiveFilesMetaData(&live_files);
  242. Status cs;
  243. for (auto a_file : live_files) {
  244. cs = VerifyChecksum(a_file);
  245. if (!cs.ok()) {
  246. break;
  247. }
  248. }
  249. EXPECT_OK(db_->EnableFileDeletions());
  250. return cs;
  251. }
  252. };
  253. TEST_F(LdbCmdTest, DumpFileChecksumNoChecksum) {
  254. Env* base_env = TryLoadCustomOrDefaultEnv();
  255. std::unique_ptr<Env> env(NewMemEnv(base_env));
  256. Options opts;
  257. opts.env = env.get();
  258. opts.create_if_missing = true;
  259. DB* db = nullptr;
  260. std::string dbname = test::PerThreadDBPath(env.get(), "ldb_cmd_test");
  261. ASSERT_OK(DB::Open(opts, dbname, &db));
  262. WriteOptions wopts;
  263. FlushOptions fopts;
  264. fopts.wait = true;
  265. Random rnd(test::RandomSeed());
  266. for (int i = 0; i < 200; i++) {
  267. char buf[16];
  268. snprintf(buf, sizeof(buf), "%08d", i);
  269. std::string v = rnd.RandomString(100);
  270. ASSERT_OK(db->Put(wopts, buf, v));
  271. }
  272. ASSERT_OK(db->Flush(fopts));
  273. for (int i = 100; i < 300; i++) {
  274. char buf[16];
  275. snprintf(buf, sizeof(buf), "%08d", i);
  276. std::string v = rnd.RandomString(100);
  277. ASSERT_OK(db->Put(wopts, buf, v));
  278. }
  279. ASSERT_OK(db->Flush(fopts));
  280. for (int i = 200; i < 400; i++) {
  281. char buf[16];
  282. snprintf(buf, sizeof(buf), "%08d", i);
  283. std::string v = rnd.RandomString(100);
  284. ASSERT_OK(db->Put(wopts, buf, v));
  285. }
  286. ASSERT_OK(db->Flush(fopts));
  287. for (int i = 300; i < 400; i++) {
  288. char buf[16];
  289. snprintf(buf, sizeof(buf), "%08d", i);
  290. std::string v = rnd.RandomString(100);
  291. ASSERT_OK(db->Put(wopts, buf, v));
  292. }
  293. ASSERT_OK(db->Flush(fopts));
  294. ASSERT_OK(db->Close());
  295. delete db;
  296. char arg1[] = "./ldb";
  297. char arg2[1024];
  298. snprintf(arg2, sizeof(arg2), "--db=%s", dbname.c_str());
  299. char arg3[] = "file_checksum_dump";
  300. char arg4[] = "--hex";
  301. char* argv[] = {arg1, arg2, arg3, arg4};
  302. ASSERT_EQ(0,
  303. LDBCommandRunner::RunCommand(4, argv, opts, LDBOptions(), nullptr));
  304. ASSERT_OK(DB::Open(opts, dbname, &db));
  305. // Verify each sst file checksum value and checksum name
  306. FileChecksumTestHelper fct_helper(opts, db, dbname);
  307. ASSERT_OK(fct_helper.VerifyEachFileChecksum());
  308. // Manually trigger compaction
  309. char b_buf[16];
  310. snprintf(b_buf, sizeof(b_buf), "%08d", 0);
  311. char e_buf[16];
  312. snprintf(e_buf, sizeof(e_buf), "%08d", 399);
  313. Slice begin(b_buf);
  314. Slice end(e_buf);
  315. CompactRangeOptions options;
  316. ASSERT_OK(db->CompactRange(options, &begin, &end));
  317. // Verify each sst file checksum after compaction
  318. FileChecksumTestHelper fct_helper_ac(opts, db, dbname);
  319. ASSERT_OK(fct_helper_ac.VerifyEachFileChecksum());
  320. ASSERT_OK(db->Close());
  321. delete db;
  322. ASSERT_EQ(0,
  323. LDBCommandRunner::RunCommand(4, argv, opts, LDBOptions(), nullptr));
  324. ASSERT_OK(DB::Open(opts, dbname, &db));
  325. // Verify the checksum information in memory is the same as that in Manifest;
  326. std::vector<LiveFileMetaData> live_files;
  327. db->GetLiveFilesMetaData(&live_files);
  328. delete db;
  329. ASSERT_OK(fct_helper_ac.VerifyChecksumInManifest(live_files));
  330. }
  331. TEST_F(LdbCmdTest, BlobDBDumpFileChecksumNoChecksum) {
  332. Env* base_env = TryLoadCustomOrDefaultEnv();
  333. std::unique_ptr<Env> env(NewMemEnv(base_env));
  334. Options opts;
  335. opts.env = env.get();
  336. opts.create_if_missing = true;
  337. opts.enable_blob_files = true;
  338. DB* db = nullptr;
  339. std::string dbname = test::PerThreadDBPath(env.get(), "ldb_cmd_test");
  340. ASSERT_OK(DB::Open(opts, dbname, &db));
  341. WriteOptions wopts;
  342. FlushOptions fopts;
  343. fopts.wait = true;
  344. Random rnd(test::RandomSeed());
  345. for (int i = 0; i < 200; i++) {
  346. std::ostringstream oss;
  347. oss << std::setfill('0') << std::setw(8) << std::fixed << i;
  348. std::string v = rnd.RandomString(100);
  349. ASSERT_OK(db->Put(wopts, oss.str(), v));
  350. }
  351. ASSERT_OK(db->Flush(fopts));
  352. for (int i = 100; i < 300; i++) {
  353. std::ostringstream oss;
  354. oss << std::setfill('0') << std::setw(8) << std::fixed << i;
  355. std::string v = rnd.RandomString(100);
  356. ASSERT_OK(db->Put(wopts, oss.str(), v));
  357. }
  358. ASSERT_OK(db->Flush(fopts));
  359. for (int i = 200; i < 400; i++) {
  360. std::ostringstream oss;
  361. oss << std::setfill('0') << std::setw(8) << std::fixed << i;
  362. std::string v = rnd.RandomString(100);
  363. ASSERT_OK(db->Put(wopts, oss.str(), v));
  364. }
  365. ASSERT_OK(db->Flush(fopts));
  366. for (int i = 300; i < 400; i++) {
  367. std::ostringstream oss;
  368. oss << std::setfill('0') << std::setw(8) << std::fixed << i;
  369. std::string v = rnd.RandomString(100);
  370. ASSERT_OK(db->Put(wopts, oss.str(), v));
  371. }
  372. ASSERT_OK(db->Flush(fopts));
  373. ASSERT_OK(db->Close());
  374. delete db;
  375. char arg1[] = "./ldb";
  376. std::string arg2_str = "--db=" + dbname;
  377. char arg3[] = "file_checksum_dump";
  378. char arg4[] = "--hex";
  379. char* argv[] = {arg1, const_cast<char*>(arg2_str.c_str()), arg3, arg4};
  380. ASSERT_EQ(0,
  381. LDBCommandRunner::RunCommand(4, argv, opts, LDBOptions(), nullptr));
  382. ASSERT_OK(DB::Open(opts, dbname, &db));
  383. // Verify each sst and blob file checksum value and checksum name
  384. FileChecksumTestHelper fct_helper(opts, db, dbname);
  385. ASSERT_OK(fct_helper.VerifyEachFileChecksum());
  386. // Manually trigger compaction
  387. std::ostringstream oss_b_buf;
  388. oss_b_buf << std::setfill('0') << std::setw(8) << std::fixed << 0;
  389. std::ostringstream oss_e_buf;
  390. oss_e_buf << std::setfill('0') << std::setw(8) << std::fixed << 399;
  391. std::string b_buf = oss_b_buf.str();
  392. std::string e_buf = oss_e_buf.str();
  393. Slice begin(b_buf);
  394. Slice end(e_buf);
  395. CompactRangeOptions options;
  396. ASSERT_OK(db->CompactRange(options, &begin, &end));
  397. // Verify each sst file checksum after compaction
  398. FileChecksumTestHelper fct_helper_ac(opts, db, dbname);
  399. ASSERT_OK(fct_helper_ac.VerifyEachFileChecksum());
  400. ASSERT_OK(db->Close());
  401. delete db;
  402. ASSERT_EQ(0,
  403. LDBCommandRunner::RunCommand(4, argv, opts, LDBOptions(), nullptr));
  404. }
  405. TEST_F(LdbCmdTest, DumpFileChecksumCRC32) {
  406. Env* base_env = TryLoadCustomOrDefaultEnv();
  407. std::unique_ptr<Env> env(NewMemEnv(base_env));
  408. Options opts;
  409. opts.env = env.get();
  410. opts.create_if_missing = true;
  411. opts.file_checksum_gen_factory = GetFileChecksumGenCrc32cFactory();
  412. DB* db = nullptr;
  413. std::string dbname = test::PerThreadDBPath(env.get(), "ldb_cmd_test");
  414. ASSERT_OK(DB::Open(opts, dbname, &db));
  415. WriteOptions wopts;
  416. FlushOptions fopts;
  417. fopts.wait = true;
  418. Random rnd(test::RandomSeed());
  419. for (int i = 0; i < 100; i++) {
  420. char buf[16];
  421. snprintf(buf, sizeof(buf), "%08d", i);
  422. std::string v = rnd.RandomString(100);
  423. ASSERT_OK(db->Put(wopts, buf, v));
  424. }
  425. ASSERT_OK(db->Flush(fopts));
  426. for (int i = 50; i < 150; i++) {
  427. char buf[16];
  428. snprintf(buf, sizeof(buf), "%08d", i);
  429. std::string v = rnd.RandomString(100);
  430. ASSERT_OK(db->Put(wopts, buf, v));
  431. }
  432. ASSERT_OK(db->Flush(fopts));
  433. for (int i = 100; i < 200; i++) {
  434. char buf[16];
  435. snprintf(buf, sizeof(buf), "%08d", i);
  436. std::string v = rnd.RandomString(100);
  437. ASSERT_OK(db->Put(wopts, buf, v));
  438. }
  439. ASSERT_OK(db->Flush(fopts));
  440. for (int i = 150; i < 250; i++) {
  441. char buf[16];
  442. snprintf(buf, sizeof(buf), "%08d", i);
  443. std::string v = rnd.RandomString(100);
  444. ASSERT_OK(db->Put(wopts, buf, v));
  445. }
  446. ASSERT_OK(db->Flush(fopts));
  447. ASSERT_OK(db->Close());
  448. delete db;
  449. char arg1[] = "./ldb";
  450. char arg2[1024];
  451. snprintf(arg2, sizeof(arg2), "--db=%s", dbname.c_str());
  452. char arg3[] = "file_checksum_dump";
  453. char arg4[] = "--hex";
  454. char* argv[] = {arg1, arg2, arg3, arg4};
  455. ASSERT_EQ(0,
  456. LDBCommandRunner::RunCommand(4, argv, opts, LDBOptions(), nullptr));
  457. ASSERT_OK(DB::Open(opts, dbname, &db));
  458. // Verify each sst file checksum value and checksum name
  459. FileChecksumTestHelper fct_helper(opts, db, dbname);
  460. ASSERT_OK(fct_helper.VerifyEachFileChecksum());
  461. // Manually trigger compaction
  462. char b_buf[16];
  463. snprintf(b_buf, sizeof(b_buf), "%08d", 0);
  464. char e_buf[16];
  465. snprintf(e_buf, sizeof(e_buf), "%08d", 249);
  466. Slice begin(b_buf);
  467. Slice end(e_buf);
  468. CompactRangeOptions options;
  469. ASSERT_OK(db->CompactRange(options, &begin, &end));
  470. // Verify each sst file checksum after compaction
  471. FileChecksumTestHelper fct_helper_ac(opts, db, dbname);
  472. ASSERT_OK(fct_helper_ac.VerifyEachFileChecksum());
  473. ASSERT_OK(db->Close());
  474. delete db;
  475. ASSERT_EQ(0,
  476. LDBCommandRunner::RunCommand(4, argv, opts, LDBOptions(), nullptr));
  477. ASSERT_OK(DB::Open(opts, dbname, &db));
  478. // Verify the checksum information in memory is the same as that in Manifest;
  479. std::vector<LiveFileMetaData> live_files;
  480. db->GetLiveFilesMetaData(&live_files);
  481. ASSERT_OK(fct_helper_ac.VerifyChecksumInManifest(live_files));
  482. ASSERT_OK(db->Close());
  483. delete db;
  484. }
  485. TEST_F(LdbCmdTest, BlobDBDumpFileChecksumCRC32) {
  486. Env* base_env = TryLoadCustomOrDefaultEnv();
  487. std::unique_ptr<Env> env(NewMemEnv(base_env));
  488. Options opts;
  489. opts.env = env.get();
  490. opts.create_if_missing = true;
  491. opts.file_checksum_gen_factory = GetFileChecksumGenCrc32cFactory();
  492. opts.enable_blob_files = true;
  493. DB* db = nullptr;
  494. std::string dbname = test::PerThreadDBPath(env.get(), "ldb_cmd_test");
  495. ASSERT_OK(DB::Open(opts, dbname, &db));
  496. WriteOptions wopts;
  497. FlushOptions fopts;
  498. fopts.wait = true;
  499. Random rnd(test::RandomSeed());
  500. for (int i = 0; i < 100; i++) {
  501. std::ostringstream oss;
  502. oss << std::setfill('0') << std::setw(8) << std::fixed << i;
  503. std::string v = rnd.RandomString(100);
  504. ASSERT_OK(db->Put(wopts, oss.str(), v));
  505. }
  506. ASSERT_OK(db->Flush(fopts));
  507. for (int i = 50; i < 150; i++) {
  508. std::ostringstream oss;
  509. oss << std::setfill('0') << std::setw(8) << std::fixed << i;
  510. std::string v = rnd.RandomString(100);
  511. ASSERT_OK(db->Put(wopts, oss.str(), v));
  512. }
  513. ASSERT_OK(db->Flush(fopts));
  514. for (int i = 100; i < 200; i++) {
  515. std::ostringstream oss;
  516. oss << std::setfill('0') << std::setw(8) << std::fixed << i;
  517. std::string v = rnd.RandomString(100);
  518. ASSERT_OK(db->Put(wopts, oss.str(), v));
  519. }
  520. ASSERT_OK(db->Flush(fopts));
  521. for (int i = 150; i < 250; i++) {
  522. std::ostringstream oss;
  523. oss << std::setfill('0') << std::setw(8) << std::fixed << i;
  524. std::string v = rnd.RandomString(100);
  525. ASSERT_OK(db->Put(wopts, oss.str(), v));
  526. }
  527. ASSERT_OK(db->Flush(fopts));
  528. ASSERT_OK(db->Close());
  529. delete db;
  530. char arg1[] = "./ldb";
  531. std::string arg2_str = "--db=" + dbname;
  532. char arg3[] = "file_checksum_dump";
  533. char arg4[] = "--hex";
  534. char* argv[] = {arg1, const_cast<char*>(arg2_str.c_str()), arg3, arg4};
  535. ASSERT_EQ(0,
  536. LDBCommandRunner::RunCommand(4, argv, opts, LDBOptions(), nullptr));
  537. ASSERT_OK(DB::Open(opts, dbname, &db));
  538. // Verify each sst and blob file checksum value and checksum name
  539. FileChecksumTestHelper fct_helper(opts, db, dbname);
  540. ASSERT_OK(fct_helper.VerifyEachFileChecksum());
  541. // Manually trigger compaction
  542. std::ostringstream oss_b_buf;
  543. oss_b_buf << std::setfill('0') << std::setw(8) << std::fixed << 0;
  544. std::ostringstream oss_e_buf;
  545. oss_e_buf << std::setfill('0') << std::setw(8) << std::fixed << 249;
  546. std::string b_buf = oss_b_buf.str();
  547. std::string e_buf = oss_e_buf.str();
  548. Slice begin(b_buf);
  549. Slice end(e_buf);
  550. CompactRangeOptions options;
  551. ASSERT_OK(db->CompactRange(options, &begin, &end));
  552. // Verify each sst file checksum after compaction
  553. FileChecksumTestHelper fct_helper_ac(opts, db, dbname);
  554. ASSERT_OK(fct_helper_ac.VerifyEachFileChecksum());
  555. ASSERT_OK(db->Close());
  556. delete db;
  557. ASSERT_EQ(0,
  558. LDBCommandRunner::RunCommand(4, argv, opts, LDBOptions(), nullptr));
  559. }
  560. TEST_F(LdbCmdTest, OptionParsing) {
  561. // test parsing flags
  562. Options opts;
  563. opts.env = TryLoadCustomOrDefaultEnv();
  564. {
  565. std::vector<std::string> args;
  566. args.emplace_back("scan");
  567. args.emplace_back("--ttl");
  568. args.emplace_back("--timestamp");
  569. LDBCommand* command = ROCKSDB_NAMESPACE::LDBCommand::InitFromCmdLineArgs(
  570. args, opts, LDBOptions(), nullptr);
  571. const std::vector<std::string> flags = command->TEST_GetFlags();
  572. EXPECT_EQ(flags.size(), 2);
  573. EXPECT_EQ(flags[0], "ttl");
  574. EXPECT_EQ(flags[1], "timestamp");
  575. delete command;
  576. }
  577. // test parsing options which contains equal sign in the option value
  578. {
  579. std::vector<std::string> args;
  580. args.emplace_back("scan");
  581. args.emplace_back("--db=/dev/shm/ldbtest/");
  582. args.emplace_back(
  583. "--from='abcd/efg/hijk/lmn/"
  584. "opq:__rst.uvw.xyz?a=3+4+bcd+efghi&jk=lm_no&pq=rst-0&uv=wx-8&yz=a&bcd_"
  585. "ef=gh.ijk'");
  586. LDBCommand* command = ROCKSDB_NAMESPACE::LDBCommand::InitFromCmdLineArgs(
  587. args, opts, LDBOptions(), nullptr);
  588. const std::map<std::string, std::string> option_map =
  589. command->TEST_GetOptionMap();
  590. EXPECT_EQ(option_map.at("db"), "/dev/shm/ldbtest/");
  591. EXPECT_EQ(option_map.at("from"),
  592. "'abcd/efg/hijk/lmn/"
  593. "opq:__rst.uvw.xyz?a=3+4+bcd+efghi&jk=lm_no&pq=rst-0&uv=wx-8&yz="
  594. "a&bcd_ef=gh.ijk'");
  595. delete command;
  596. }
  597. }
  598. TEST_F(LdbCmdTest, ListFileTombstone) {
  599. Env* base_env = TryLoadCustomOrDefaultEnv();
  600. std::unique_ptr<Env> env(NewMemEnv(base_env));
  601. Options opts;
  602. opts.env = env.get();
  603. opts.create_if_missing = true;
  604. DB* db = nullptr;
  605. std::string dbname = test::PerThreadDBPath(env.get(), "ldb_cmd_test");
  606. ASSERT_OK(DB::Open(opts, dbname, &db));
  607. WriteOptions wopts;
  608. ASSERT_OK(db->Put(wopts, "foo", "1"));
  609. ASSERT_OK(db->Put(wopts, "bar", "2"));
  610. FlushOptions fopts;
  611. fopts.wait = true;
  612. ASSERT_OK(db->Flush(fopts));
  613. ASSERT_OK(db->DeleteRange(wopts, db->DefaultColumnFamily(), "foo", "foo2"));
  614. ASSERT_OK(db->DeleteRange(wopts, db->DefaultColumnFamily(), "bar", "foo2"));
  615. ASSERT_OK(db->Flush(fopts));
  616. delete db;
  617. {
  618. char arg1[] = "./ldb";
  619. char arg2[1024];
  620. snprintf(arg2, sizeof(arg2), "--db=%s", dbname.c_str());
  621. char arg3[] = "list_file_range_deletes";
  622. char* argv[] = {arg1, arg2, arg3};
  623. ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
  624. "ListFileRangeDeletesCommand::DoCommand:BeforePrint", [&](void* arg) {
  625. std::string* out_str = static_cast<std::string*>(arg);
  626. // Count number of tombstones printed
  627. int num_tb = 0;
  628. const std::string kFingerprintStr = "start: ";
  629. auto offset = out_str->find(kFingerprintStr);
  630. while (offset != std::string::npos) {
  631. num_tb++;
  632. offset =
  633. out_str->find(kFingerprintStr, offset + kFingerprintStr.size());
  634. }
  635. EXPECT_EQ(2, num_tb);
  636. });
  637. ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
  638. ASSERT_EQ(
  639. 0, LDBCommandRunner::RunCommand(3, argv, opts, LDBOptions(), nullptr));
  640. ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks();
  641. ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
  642. }
  643. // Test the case of limiting tombstones
  644. {
  645. char arg1[] = "./ldb";
  646. char arg2[1024];
  647. snprintf(arg2, sizeof(arg2), "--db=%s", dbname.c_str());
  648. char arg3[] = "list_file_range_deletes";
  649. char arg4[] = "--max_keys=1";
  650. char* argv[] = {arg1, arg2, arg3, arg4};
  651. ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
  652. "ListFileRangeDeletesCommand::DoCommand:BeforePrint", [&](void* arg) {
  653. std::string* out_str = static_cast<std::string*>(arg);
  654. // Count number of tombstones printed
  655. int num_tb = 0;
  656. const std::string kFingerprintStr = "start: ";
  657. auto offset = out_str->find(kFingerprintStr);
  658. while (offset != std::string::npos) {
  659. num_tb++;
  660. offset =
  661. out_str->find(kFingerprintStr, offset + kFingerprintStr.size());
  662. }
  663. EXPECT_EQ(1, num_tb);
  664. });
  665. ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
  666. ASSERT_EQ(
  667. 0, LDBCommandRunner::RunCommand(4, argv, opts, LDBOptions(), nullptr));
  668. ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks();
  669. ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
  670. }
  671. }
  672. TEST_F(LdbCmdTest, DisableConsistencyChecks) {
  673. Env* base_env = TryLoadCustomOrDefaultEnv();
  674. std::unique_ptr<Env> env(NewMemEnv(base_env));
  675. Options opts;
  676. opts.env = env.get();
  677. opts.create_if_missing = true;
  678. std::string dbname = test::PerThreadDBPath(env.get(), "ldb_cmd_test");
  679. {
  680. DB* db = nullptr;
  681. ASSERT_OK(DB::Open(opts, dbname, &db));
  682. WriteOptions wopts;
  683. FlushOptions fopts;
  684. fopts.wait = true;
  685. ASSERT_OK(db->Put(wopts, "foo1", "1"));
  686. ASSERT_OK(db->Put(wopts, "bar1", "2"));
  687. ASSERT_OK(db->Flush(fopts));
  688. ASSERT_OK(db->Put(wopts, "foo2", "3"));
  689. ASSERT_OK(db->Put(wopts, "bar2", "4"));
  690. ASSERT_OK(db->Flush(fopts));
  691. delete db;
  692. }
  693. {
  694. char arg1[] = "./ldb";
  695. char arg2[1024];
  696. snprintf(arg2, sizeof(arg2), "--db=%s", dbname.c_str());
  697. char arg3[] = "checkconsistency";
  698. char* argv[] = {arg1, arg2, arg3};
  699. SyncPoint::GetInstance()->SetCallBack(
  700. "Version::PrepareAppend:forced_check", [&](void* arg) {
  701. bool* forced = static_cast<bool*>(arg);
  702. ASSERT_TRUE(*forced);
  703. });
  704. SyncPoint::GetInstance()->EnableProcessing();
  705. ASSERT_EQ(
  706. 0, LDBCommandRunner::RunCommand(3, argv, opts, LDBOptions(), nullptr));
  707. SyncPoint::GetInstance()->ClearAllCallBacks();
  708. SyncPoint::GetInstance()->DisableProcessing();
  709. }
  710. {
  711. char arg1[] = "./ldb";
  712. char arg2[1024];
  713. snprintf(arg2, sizeof(arg2), "--db=%s", dbname.c_str());
  714. char arg3[] = "scan";
  715. char* argv[] = {arg1, arg2, arg3};
  716. SyncPoint::GetInstance()->SetCallBack(
  717. "Version::PrepareAppend:forced_check", [&](void* arg) {
  718. bool* forced = static_cast<bool*>(arg);
  719. ASSERT_TRUE(*forced);
  720. });
  721. SyncPoint::GetInstance()->EnableProcessing();
  722. ASSERT_EQ(
  723. 0, LDBCommandRunner::RunCommand(3, argv, opts, LDBOptions(), nullptr));
  724. SyncPoint::GetInstance()->ClearAllCallBacks();
  725. SyncPoint::GetInstance()->DisableProcessing();
  726. }
  727. {
  728. char arg1[] = "./ldb";
  729. char arg2[1024];
  730. snprintf(arg2, sizeof(arg2), "--db=%s", dbname.c_str());
  731. char arg3[] = "scan";
  732. char arg4[] = "--disable_consistency_checks";
  733. char* argv[] = {arg1, arg2, arg3, arg4};
  734. SyncPoint::GetInstance()->SetCallBack(
  735. "ColumnFamilyData::ColumnFamilyData", [&](void* arg) {
  736. ColumnFamilyOptions* cfo = static_cast<ColumnFamilyOptions*>(arg);
  737. ASSERT_FALSE(cfo->force_consistency_checks);
  738. });
  739. SyncPoint::GetInstance()->EnableProcessing();
  740. ASSERT_EQ(
  741. 0, LDBCommandRunner::RunCommand(4, argv, opts, LDBOptions(), nullptr));
  742. SyncPoint::GetInstance()->ClearAllCallBacks();
  743. SyncPoint::GetInstance()->DisableProcessing();
  744. }
  745. }
  746. TEST_F(LdbCmdTest, TestBadDbPath) {
  747. Env* base_env = TryLoadCustomOrDefaultEnv();
  748. std::unique_ptr<Env> env(NewMemEnv(base_env));
  749. Options opts;
  750. opts.env = env.get();
  751. opts.create_if_missing = true;
  752. std::string dbname = test::PerThreadDBPath(env.get(), "ldb_cmd_test");
  753. char arg1[] = "./ldb";
  754. char arg2[1024];
  755. snprintf(arg2, sizeof(arg2), "--db=%s/.no_such_dir", dbname.c_str());
  756. char arg3[1024];
  757. snprintf(arg3, sizeof(arg3), "create_column_family");
  758. char arg4[] = "bad cf";
  759. char* argv[] = {arg1, arg2, arg3, arg4};
  760. ASSERT_EQ(1,
  761. LDBCommandRunner::RunCommand(4, argv, opts, LDBOptions(), nullptr));
  762. snprintf(arg3, sizeof(arg3), "drop_column_family");
  763. ASSERT_EQ(1,
  764. LDBCommandRunner::RunCommand(4, argv, opts, LDBOptions(), nullptr));
  765. }
  766. namespace {
  767. class WrappedEnv : public EnvWrapper {
  768. public:
  769. explicit WrappedEnv(Env* t) : EnvWrapper(t) {}
  770. static const char* kClassName() { return "WrappedEnv"; }
  771. const char* Name() const override { return kClassName(); }
  772. };
  773. } // namespace
  774. TEST_F(LdbCmdTest, LoadCFOptionsAndOverride) {
  775. // Env* base_env = TryLoadCustomOrDefaultEnv();
  776. // std::unique_ptr<Env> env(NewMemEnv(base_env));
  777. std::unique_ptr<Env> env(new WrappedEnv(Env::Default()));
  778. Options opts;
  779. opts.env = env.get();
  780. opts.create_if_missing = true;
  781. DB* db = nullptr;
  782. std::string dbname = test::PerThreadDBPath(env.get(), "ldb_cmd_test");
  783. ASSERT_OK(DestroyDB(dbname, opts));
  784. ASSERT_OK(DB::Open(opts, dbname, &db));
  785. ColumnFamilyHandle* cf_handle;
  786. ColumnFamilyOptions cf_opts;
  787. cf_opts.num_levels = 20;
  788. ASSERT_OK(db->CreateColumnFamily(cf_opts, "cf1", &cf_handle));
  789. delete cf_handle;
  790. delete db;
  791. char arg1[] = "./ldb";
  792. char arg2[1024];
  793. snprintf(arg2, sizeof(arg2), "--db=%s", dbname.c_str());
  794. char arg3[] = "put";
  795. char arg4[] = "key1";
  796. char arg5[] = "value1";
  797. char arg6[] = "--try_load_options";
  798. char arg7[] = "--column_family=cf1";
  799. char arg8[] = "--write_buffer_size=268435456";
  800. char* argv[] = {arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8};
  801. ASSERT_EQ(0,
  802. LDBCommandRunner::RunCommand(8, argv, opts, LDBOptions(), nullptr));
  803. ConfigOptions config_opts;
  804. Options options;
  805. std::vector<ColumnFamilyDescriptor> column_families;
  806. config_opts.env = env.get();
  807. ASSERT_OK(LoadLatestOptions(config_opts, dbname, &options, &column_families));
  808. ASSERT_EQ(column_families.size(), 2);
  809. ASSERT_EQ(options.num_levels, opts.num_levels);
  810. ASSERT_EQ(column_families[1].options.num_levels, cf_opts.num_levels);
  811. ASSERT_EQ(column_families[1].options.write_buffer_size, 268435456);
  812. }
  813. TEST_F(LdbCmdTest, UnsafeRemoveSstFile) {
  814. Options opts;
  815. opts.level0_file_num_compaction_trigger = 10;
  816. opts.create_if_missing = true;
  817. DB* db = nullptr;
  818. std::string dbname = test::PerThreadDBPath(Env::Default(), "ldb_cmd_test");
  819. ASSERT_OK(DestroyDB(dbname, opts));
  820. ASSERT_OK(DB::Open(opts, dbname, &db));
  821. // Create three SST files
  822. for (size_t i = 0; i < 3; ++i) {
  823. ASSERT_OK(db->Put(WriteOptions(), std::to_string(i), std::to_string(i)));
  824. ASSERT_OK(db->Flush(FlushOptions()));
  825. }
  826. // Determine which is the "middle" one
  827. std::vector<LiveFileMetaData> sst_files;
  828. db->GetLiveFilesMetaData(&sst_files);
  829. std::vector<uint64_t> numbers;
  830. for (auto& f : sst_files) {
  831. numbers.push_back(f.file_number);
  832. }
  833. ASSERT_EQ(numbers.size(), 3);
  834. std::sort(numbers.begin(), numbers.end());
  835. uint64_t to_remove = numbers[1];
  836. // Close for unsafe_remove_sst_file
  837. delete db;
  838. db = nullptr;
  839. char arg1[] = "./ldb";
  840. char arg2[1024];
  841. snprintf(arg2, sizeof(arg2), "--db=%s", dbname.c_str());
  842. char arg3[] = "unsafe_remove_sst_file";
  843. char arg4[20];
  844. snprintf(arg4, sizeof(arg4), "%" PRIu64, to_remove);
  845. char* argv[] = {arg1, arg2, arg3, arg4};
  846. ASSERT_EQ(0,
  847. LDBCommandRunner::RunCommand(4, argv, opts, LDBOptions(), nullptr));
  848. // Re-open, and verify with Get that middle file is gone
  849. ASSERT_OK(DB::Open(opts, dbname, &db));
  850. std::string val;
  851. ASSERT_OK(db->Get(ReadOptions(), "0", &val));
  852. ASSERT_EQ(val, "0");
  853. ASSERT_OK(db->Get(ReadOptions(), "2", &val));
  854. ASSERT_EQ(val, "2");
  855. ASSERT_TRUE(db->Get(ReadOptions(), "1", &val).IsNotFound());
  856. // Now with extra CF, two more files
  857. ColumnFamilyHandle* cf_handle;
  858. ColumnFamilyOptions cf_opts;
  859. ASSERT_OK(db->CreateColumnFamily(cf_opts, "cf1", &cf_handle));
  860. for (size_t i = 3; i < 5; ++i) {
  861. ASSERT_OK(db->Put(WriteOptions(), cf_handle, std::to_string(i),
  862. std::to_string(i)));
  863. ASSERT_OK(db->Flush(FlushOptions(), cf_handle));
  864. }
  865. // Determine which is the "last" one
  866. sst_files.clear();
  867. db->GetLiveFilesMetaData(&sst_files);
  868. numbers.clear();
  869. for (auto& f : sst_files) {
  870. numbers.push_back(f.file_number);
  871. }
  872. ASSERT_EQ(numbers.size(), 4);
  873. std::sort(numbers.begin(), numbers.end());
  874. to_remove = numbers.back();
  875. // Close for unsafe_remove_sst_file
  876. delete cf_handle;
  877. delete db;
  878. db = nullptr;
  879. snprintf(arg4, sizeof(arg4), "%" PRIu64, to_remove);
  880. ASSERT_EQ(0,
  881. LDBCommandRunner::RunCommand(4, argv, opts, LDBOptions(), nullptr));
  882. std::vector<ColumnFamilyDescriptor> cfds = {{kDefaultColumnFamilyName, opts},
  883. {"cf1", cf_opts}};
  884. std::vector<ColumnFamilyHandle*> handles;
  885. ASSERT_OK(DB::Open(opts, dbname, cfds, &handles, &db));
  886. ASSERT_OK(db->Get(ReadOptions(), handles[1], "3", &val));
  887. ASSERT_EQ(val, "3");
  888. ASSERT_TRUE(db->Get(ReadOptions(), handles[1], "4", &val).IsNotFound());
  889. ASSERT_OK(db->Get(ReadOptions(), handles[0], "0", &val));
  890. ASSERT_EQ(val, "0");
  891. // Determine which is the "first" one (most likely to be opened in recovery)
  892. sst_files.clear();
  893. db->GetLiveFilesMetaData(&sst_files);
  894. numbers.clear();
  895. for (auto& f : sst_files) {
  896. numbers.push_back(f.file_number);
  897. }
  898. ASSERT_EQ(numbers.size(), 3);
  899. std::sort(numbers.begin(), numbers.end());
  900. to_remove = numbers.front();
  901. // This time physically delete the file before unsafe_remove
  902. {
  903. std::string f = dbname + "/" + MakeTableFileName(to_remove);
  904. ASSERT_OK(Env::Default()->DeleteFile(f));
  905. }
  906. // Close for unsafe_remove_sst_file
  907. for (auto& h : handles) {
  908. delete h;
  909. }
  910. delete db;
  911. db = nullptr;
  912. snprintf(arg4, sizeof(arg4), "%" PRIu64, to_remove);
  913. ASSERT_EQ(0,
  914. LDBCommandRunner::RunCommand(4, argv, opts, LDBOptions(), nullptr));
  915. ASSERT_OK(DB::Open(opts, dbname, cfds, &handles, &db));
  916. ASSERT_OK(db->Get(ReadOptions(), handles[1], "3", &val));
  917. ASSERT_EQ(val, "3");
  918. ASSERT_TRUE(db->Get(ReadOptions(), handles[0], "0", &val).IsNotFound());
  919. for (auto& h : handles) {
  920. delete h;
  921. }
  922. delete db;
  923. }
  924. TEST_F(LdbCmdTest, FileTemperatureUpdateManifest) {
  925. auto test_fs = std::make_shared<FileTemperatureTestFS>(FileSystem::Default());
  926. std::unique_ptr<Env> env(new CompositeEnvWrapper(Env::Default(), test_fs));
  927. Options opts;
  928. opts.last_level_temperature = Temperature::kWarm;
  929. opts.level0_file_num_compaction_trigger = 10;
  930. opts.create_if_missing = true;
  931. opts.env = env.get();
  932. DB* db = nullptr;
  933. std::string dbname = test::PerThreadDBPath(env.get(), "ldb_cmd_test");
  934. ASSERT_OK(DestroyDB(dbname, opts));
  935. ASSERT_OK(DB::Open(opts, dbname, &db));
  936. std::array<Temperature, 5> kTestTemps = {
  937. Temperature::kCold, Temperature::kWarm, Temperature::kHot,
  938. Temperature::kWarm, Temperature::kCold};
  939. std::map<uint64_t, Temperature> number_to_temp;
  940. for (size_t i = 0; i < kTestTemps.size(); ++i) {
  941. ASSERT_OK(db->Put(WriteOptions(), std::to_string(i), std::to_string(i)));
  942. ASSERT_OK(db->Flush(FlushOptions()));
  943. std::map<uint64_t, Temperature> current_temps;
  944. test_fs->CopyCurrentSstFileTemperatures(&current_temps);
  945. for (auto e : current_temps) {
  946. if (e.second == Temperature::kUnknown) {
  947. test_fs->OverrideSstFileTemperature(e.first, kTestTemps[i]);
  948. number_to_temp[e.first] = kTestTemps[i];
  949. }
  950. }
  951. }
  952. // Close & reopen
  953. delete db;
  954. db = nullptr;
  955. test_fs->PopRequestedSstFileTemperatures();
  956. ASSERT_OK(DB::Open(opts, dbname, &db));
  957. for (size_t i = 0; i < kTestTemps.size(); ++i) {
  958. std::string val;
  959. ASSERT_OK(db->Get(ReadOptions(), std::to_string(i), &val));
  960. ASSERT_EQ(val, std::to_string(i));
  961. }
  962. // Still all unknown
  963. std::vector<std::pair<uint64_t, Temperature>> requests;
  964. test_fs->PopRequestedSstFileTemperatures(&requests);
  965. ASSERT_EQ(requests.size(), kTestTemps.size());
  966. for (auto& r : requests) {
  967. ASSERT_EQ(r.second, Temperature::kUnknown);
  968. }
  969. // Close for update_manifest
  970. delete db;
  971. db = nullptr;
  972. char arg1[] = "./ldb";
  973. char arg2[1024];
  974. snprintf(arg2, sizeof(arg2), "--db=%s", dbname.c_str());
  975. char arg3[] = "update_manifest";
  976. char arg4[] = "--update_temperatures";
  977. char* argv[] = {arg1, arg2, arg3, arg4};
  978. ASSERT_EQ(0,
  979. LDBCommandRunner::RunCommand(4, argv, opts, LDBOptions(), nullptr));
  980. // Re-open, get, and verify manifest temps (based on request)
  981. test_fs->PopRequestedSstFileTemperatures();
  982. ASSERT_OK(DB::Open(opts, dbname, &db));
  983. for (size_t i = 0; i < kTestTemps.size(); ++i) {
  984. std::string val;
  985. ASSERT_OK(db->Get(ReadOptions(), std::to_string(i), &val));
  986. ASSERT_EQ(val, std::to_string(i));
  987. }
  988. requests.clear();
  989. test_fs->PopRequestedSstFileTemperatures(&requests);
  990. ASSERT_EQ(requests.size(), kTestTemps.size());
  991. for (auto& r : requests) {
  992. ASSERT_EQ(r.second, number_to_temp[r.first]);
  993. }
  994. delete db;
  995. }
  996. TEST_F(LdbCmdTest, RenameDbAndLoadOptions) {
  997. Env* env = TryLoadCustomOrDefaultEnv();
  998. Options opts;
  999. opts.env = env;
  1000. opts.create_if_missing = false;
  1001. std::string old_dbname = test::PerThreadDBPath(env, "ldb_cmd_test");
  1002. std::string new_dbname = old_dbname + "_2";
  1003. ASSERT_OK(DestroyDB(old_dbname, opts));
  1004. ASSERT_OK(DestroyDB(new_dbname, opts));
  1005. char old_arg[1024];
  1006. snprintf(old_arg, sizeof(old_arg), "--db=%s", old_dbname.c_str());
  1007. char new_arg[1024];
  1008. snprintf(new_arg, sizeof(old_arg), "--db=%s", new_dbname.c_str());
  1009. const char* argv1[] = {"./ldb",
  1010. old_arg,
  1011. "put",
  1012. "key1",
  1013. "value1",
  1014. "--try_load_options",
  1015. "--create_if_missing"};
  1016. const char* argv2[] = {"./ldb", old_arg, "get", "key1", "--try_load_options"};
  1017. const char* argv3[] = {"./ldb", new_arg, "put",
  1018. "key2", "value2", "--try_load_options"};
  1019. const char* argv4[] = {"./ldb", new_arg, "get", "key1", "--try_load_options"};
  1020. const char* argv5[] = {"./ldb", new_arg, "get", "key2", "--try_load_options"};
  1021. ASSERT_EQ(
  1022. 0, LDBCommandRunner::RunCommand(7, argv1, opts, LDBOptions(), nullptr));
  1023. ASSERT_EQ(
  1024. 0, LDBCommandRunner::RunCommand(5, argv2, opts, LDBOptions(), nullptr));
  1025. ConfigOptions config_opts;
  1026. Options options;
  1027. std::vector<ColumnFamilyDescriptor> column_families;
  1028. config_opts.env = env;
  1029. ASSERT_OK(
  1030. LoadLatestOptions(config_opts, old_dbname, &options, &column_families));
  1031. ASSERT_EQ(options.wal_dir, "");
  1032. ASSERT_OK(env->RenameFile(old_dbname, new_dbname));
  1033. ASSERT_NE(
  1034. 0, LDBCommandRunner::RunCommand(6, argv1, opts, LDBOptions(), nullptr));
  1035. ASSERT_NE(
  1036. 0, LDBCommandRunner::RunCommand(5, argv2, opts, LDBOptions(), nullptr));
  1037. ASSERT_EQ(
  1038. 0, LDBCommandRunner::RunCommand(6, argv3, opts, LDBOptions(), nullptr));
  1039. ASSERT_EQ(
  1040. 0, LDBCommandRunner::RunCommand(5, argv4, opts, LDBOptions(), nullptr));
  1041. ASSERT_EQ(
  1042. 0, LDBCommandRunner::RunCommand(5, argv5, opts, LDBOptions(), nullptr));
  1043. ASSERT_OK(DestroyDB(new_dbname, opts));
  1044. }
  1045. class MyComparator : public Comparator {
  1046. public:
  1047. int Compare(const Slice& a, const Slice& b) const override {
  1048. return a.compare(b);
  1049. }
  1050. void FindShortSuccessor(std::string* /*key*/) const override {}
  1051. void FindShortestSeparator(std::string* /*start*/,
  1052. const Slice& /*limit*/) const override {}
  1053. const char* Name() const override { return "my_comparator"; }
  1054. };
  1055. TEST_F(LdbCmdTest, CustomComparator) {
  1056. Env* env = TryLoadCustomOrDefaultEnv();
  1057. MyComparator my_comparator;
  1058. Options opts;
  1059. opts.env = env;
  1060. opts.create_if_missing = true;
  1061. opts.create_missing_column_families = true;
  1062. opts.comparator = &my_comparator;
  1063. std::string dbname = test::PerThreadDBPath(env, "ldb_cmd_test");
  1064. DB* db = nullptr;
  1065. std::vector<ColumnFamilyDescriptor> cfds = {
  1066. {kDefaultColumnFamilyName, opts}, {"cf1", opts}, {"cf2", opts}};
  1067. std::vector<ColumnFamilyHandle*> handles;
  1068. ASSERT_OK(DestroyDB(dbname, opts));
  1069. ASSERT_OK(DB::Open(opts, dbname, cfds, &handles, &db));
  1070. ASSERT_OK(db->Put(WriteOptions(), "k1", "v1"));
  1071. for (auto& h : handles) {
  1072. ASSERT_OK(db->DestroyColumnFamilyHandle(h));
  1073. }
  1074. delete db;
  1075. char arg1[] = "./ldb";
  1076. std::string arg2 = "--db=" + dbname;
  1077. char arg3[] = "get";
  1078. char arg4[] = "k1";
  1079. char* argv[] = {arg1, const_cast<char*>(arg2.c_str()), arg3, arg4};
  1080. ASSERT_EQ(0,
  1081. LDBCommandRunner::RunCommand(4, argv, opts, LDBOptions(), &cfds));
  1082. }
  1083. } // namespace ROCKSDB_NAMESPACE
  1084. int main(int argc, char** argv) {
  1085. ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
  1086. ::testing::InitGoogleTest(&argc, argv);
  1087. RegisterCustomObjects(argc, argv);
  1088. return RUN_ALL_TESTS();
  1089. }