ttl_test.cc 22 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693
  1. // Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
  2. // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
  3. // Use of this source code is governed by a BSD-style license that can be
  4. // found in the LICENSE file. See the AUTHORS file for names of contributors.
  5. #ifndef ROCKSDB_LITE
  6. #include <map>
  7. #include <memory>
  8. #include "rocksdb/compaction_filter.h"
  9. #include "rocksdb/utilities/db_ttl.h"
  10. #include "test_util/testharness.h"
  11. #include "util/string_util.h"
  12. #ifndef OS_WIN
  13. #include <unistd.h>
  14. #endif
  15. namespace ROCKSDB_NAMESPACE {
  16. namespace {
  17. typedef std::map<std::string, std::string> KVMap;
  18. enum BatchOperation { OP_PUT = 0, OP_DELETE = 1 };
  19. }
  20. class SpecialTimeEnv : public EnvWrapper {
  21. public:
  22. explicit SpecialTimeEnv(Env* base) : EnvWrapper(base) {
  23. base->GetCurrentTime(&current_time_);
  24. }
  25. void Sleep(int64_t sleep_time) { current_time_ += sleep_time; }
  26. Status GetCurrentTime(int64_t* current_time) override {
  27. *current_time = current_time_;
  28. return Status::OK();
  29. }
  30. private:
  31. int64_t current_time_ = 0;
  32. };
  33. class TtlTest : public testing::Test {
  34. public:
  35. TtlTest() {
  36. env_.reset(new SpecialTimeEnv(Env::Default()));
  37. dbname_ = test::PerThreadDBPath("db_ttl");
  38. options_.create_if_missing = true;
  39. options_.env = env_.get();
  40. // ensure that compaction is kicked in to always strip timestamp from kvs
  41. options_.max_compaction_bytes = 1;
  42. // compaction should take place always from level0 for determinism
  43. db_ttl_ = nullptr;
  44. DestroyDB(dbname_, Options());
  45. }
  46. ~TtlTest() override {
  47. CloseTtl();
  48. DestroyDB(dbname_, Options());
  49. }
  50. // Open database with TTL support when TTL not provided with db_ttl_ pointer
  51. void OpenTtl() {
  52. ASSERT_TRUE(db_ttl_ ==
  53. nullptr); // db should be closed before opening again
  54. ASSERT_OK(DBWithTTL::Open(options_, dbname_, &db_ttl_));
  55. }
  56. // Open database with TTL support when TTL provided with db_ttl_ pointer
  57. void OpenTtl(int32_t ttl) {
  58. ASSERT_TRUE(db_ttl_ == nullptr);
  59. ASSERT_OK(DBWithTTL::Open(options_, dbname_, &db_ttl_, ttl));
  60. }
  61. // Open with TestFilter compaction filter
  62. void OpenTtlWithTestCompaction(int32_t ttl) {
  63. options_.compaction_filter_factory =
  64. std::shared_ptr<CompactionFilterFactory>(
  65. new TestFilterFactory(kSampleSize_, kNewValue_));
  66. OpenTtl(ttl);
  67. }
  68. // Open database with TTL support in read_only mode
  69. void OpenReadOnlyTtl(int32_t ttl) {
  70. ASSERT_TRUE(db_ttl_ == nullptr);
  71. ASSERT_OK(DBWithTTL::Open(options_, dbname_, &db_ttl_, ttl, true));
  72. }
  73. // Call db_ttl_->Close() before delete db_ttl_
  74. void CloseTtl() { CloseTtlHelper(true); }
  75. // No db_ttl_->Close() before delete db_ttl_
  76. void CloseTtlNoDBClose() { CloseTtlHelper(false); }
  77. void CloseTtlHelper(bool close_db) {
  78. if (db_ttl_ != nullptr) {
  79. if (close_db) {
  80. db_ttl_->Close();
  81. }
  82. delete db_ttl_;
  83. db_ttl_ = nullptr;
  84. }
  85. }
  86. // Populates and returns a kv-map
  87. void MakeKVMap(int64_t num_entries) {
  88. kvmap_.clear();
  89. int digits = 1;
  90. for (int64_t dummy = num_entries; dummy /= 10; ++digits) {
  91. }
  92. int digits_in_i = 1;
  93. for (int64_t i = 0; i < num_entries; i++) {
  94. std::string key = "key";
  95. std::string value = "value";
  96. if (i % 10 == 0) {
  97. digits_in_i++;
  98. }
  99. for(int j = digits_in_i; j < digits; j++) {
  100. key.append("0");
  101. value.append("0");
  102. }
  103. AppendNumberTo(&key, i);
  104. AppendNumberTo(&value, i);
  105. kvmap_[key] = value;
  106. }
  107. ASSERT_EQ(static_cast<int64_t>(kvmap_.size()),
  108. num_entries); // check all insertions done
  109. }
  110. // Makes a write-batch with key-vals from kvmap_ and 'Write''s it
  111. void MakePutWriteBatch(const BatchOperation* batch_ops, int64_t num_ops) {
  112. ASSERT_LE(num_ops, static_cast<int64_t>(kvmap_.size()));
  113. static WriteOptions wopts;
  114. static FlushOptions flush_opts;
  115. WriteBatch batch;
  116. kv_it_ = kvmap_.begin();
  117. for (int64_t i = 0; i < num_ops && kv_it_ != kvmap_.end(); i++, ++kv_it_) {
  118. switch (batch_ops[i]) {
  119. case OP_PUT:
  120. batch.Put(kv_it_->first, kv_it_->second);
  121. break;
  122. case OP_DELETE:
  123. batch.Delete(kv_it_->first);
  124. break;
  125. default:
  126. FAIL();
  127. }
  128. }
  129. db_ttl_->Write(wopts, &batch);
  130. db_ttl_->Flush(flush_opts);
  131. }
  132. // Puts num_entries starting from start_pos_map from kvmap_ into the database
  133. void PutValues(int64_t start_pos_map, int64_t num_entries, bool flush = true,
  134. ColumnFamilyHandle* cf = nullptr) {
  135. ASSERT_TRUE(db_ttl_);
  136. ASSERT_LE(start_pos_map + num_entries, static_cast<int64_t>(kvmap_.size()));
  137. static WriteOptions wopts;
  138. static FlushOptions flush_opts;
  139. kv_it_ = kvmap_.begin();
  140. advance(kv_it_, start_pos_map);
  141. for (int64_t i = 0; kv_it_ != kvmap_.end() && i < num_entries;
  142. i++, ++kv_it_) {
  143. ASSERT_OK(cf == nullptr
  144. ? db_ttl_->Put(wopts, kv_it_->first, kv_it_->second)
  145. : db_ttl_->Put(wopts, cf, kv_it_->first, kv_it_->second));
  146. }
  147. // Put a mock kv at the end because CompactionFilter doesn't delete last key
  148. ASSERT_OK(cf == nullptr ? db_ttl_->Put(wopts, "keymock", "valuemock")
  149. : db_ttl_->Put(wopts, cf, "keymock", "valuemock"));
  150. if (flush) {
  151. if (cf == nullptr) {
  152. db_ttl_->Flush(flush_opts);
  153. } else {
  154. db_ttl_->Flush(flush_opts, cf);
  155. }
  156. }
  157. }
  158. // Runs a manual compaction
  159. void ManualCompact(ColumnFamilyHandle* cf = nullptr) {
  160. if (cf == nullptr) {
  161. db_ttl_->CompactRange(CompactRangeOptions(), nullptr, nullptr);
  162. } else {
  163. db_ttl_->CompactRange(CompactRangeOptions(), cf, nullptr, nullptr);
  164. }
  165. }
  166. // checks the whole kvmap_ to return correct values using KeyMayExist
  167. void SimpleKeyMayExistCheck() {
  168. static ReadOptions ropts;
  169. bool value_found;
  170. std::string val;
  171. for(auto &kv : kvmap_) {
  172. bool ret = db_ttl_->KeyMayExist(ropts, kv.first, &val, &value_found);
  173. if (ret == false || value_found == false) {
  174. fprintf(stderr, "KeyMayExist could not find key=%s in the database but"
  175. " should have\n", kv.first.c_str());
  176. FAIL();
  177. } else if (val.compare(kv.second) != 0) {
  178. fprintf(stderr, " value for key=%s present in database is %s but"
  179. " should be %s\n", kv.first.c_str(), val.c_str(),
  180. kv.second.c_str());
  181. FAIL();
  182. }
  183. }
  184. }
  185. // checks the whole kvmap_ to return correct values using MultiGet
  186. void SimpleMultiGetTest() {
  187. static ReadOptions ropts;
  188. std::vector<Slice> keys;
  189. std::vector<std::string> values;
  190. for (auto& kv : kvmap_) {
  191. keys.emplace_back(kv.first);
  192. }
  193. auto statuses = db_ttl_->MultiGet(ropts, keys, &values);
  194. size_t i = 0;
  195. for (auto& kv : kvmap_) {
  196. ASSERT_OK(statuses[i]);
  197. ASSERT_EQ(values[i], kv.second);
  198. ++i;
  199. }
  200. }
  201. // Sleeps for slp_tim then runs a manual compaction
  202. // Checks span starting from st_pos from kvmap_ in the db and
  203. // Gets should return true if check is true and false otherwise
  204. // Also checks that value that we got is the same as inserted; and =kNewValue
  205. // if test_compaction_change is true
  206. void SleepCompactCheck(int slp_tim, int64_t st_pos, int64_t span,
  207. bool check = true, bool test_compaction_change = false,
  208. ColumnFamilyHandle* cf = nullptr) {
  209. ASSERT_TRUE(db_ttl_);
  210. env_->Sleep(slp_tim);
  211. ManualCompact(cf);
  212. static ReadOptions ropts;
  213. kv_it_ = kvmap_.begin();
  214. advance(kv_it_, st_pos);
  215. std::string v;
  216. for (int64_t i = 0; kv_it_ != kvmap_.end() && i < span; i++, ++kv_it_) {
  217. Status s = (cf == nullptr) ? db_ttl_->Get(ropts, kv_it_->first, &v)
  218. : db_ttl_->Get(ropts, cf, kv_it_->first, &v);
  219. if (s.ok() != check) {
  220. fprintf(stderr, "key=%s ", kv_it_->first.c_str());
  221. if (!s.ok()) {
  222. fprintf(stderr, "is absent from db but was expected to be present\n");
  223. } else {
  224. fprintf(stderr, "is present in db but was expected to be absent\n");
  225. }
  226. FAIL();
  227. } else if (s.ok()) {
  228. if (test_compaction_change && v.compare(kNewValue_) != 0) {
  229. fprintf(stderr, " value for key=%s present in database is %s but "
  230. " should be %s\n", kv_it_->first.c_str(), v.c_str(),
  231. kNewValue_.c_str());
  232. FAIL();
  233. } else if (!test_compaction_change && v.compare(kv_it_->second) !=0) {
  234. fprintf(stderr, " value for key=%s present in database is %s but "
  235. " should be %s\n", kv_it_->first.c_str(), v.c_str(),
  236. kv_it_->second.c_str());
  237. FAIL();
  238. }
  239. }
  240. }
  241. }
  242. // Similar as SleepCompactCheck but uses TtlIterator to read from db
  243. void SleepCompactCheckIter(int slp, int st_pos, int64_t span,
  244. bool check = true) {
  245. ASSERT_TRUE(db_ttl_);
  246. env_->Sleep(slp);
  247. ManualCompact();
  248. static ReadOptions ropts;
  249. Iterator *dbiter = db_ttl_->NewIterator(ropts);
  250. kv_it_ = kvmap_.begin();
  251. advance(kv_it_, st_pos);
  252. dbiter->Seek(kv_it_->first);
  253. if (!check) {
  254. if (dbiter->Valid()) {
  255. ASSERT_NE(dbiter->value().compare(kv_it_->second), 0);
  256. }
  257. } else { // dbiter should have found out kvmap_[st_pos]
  258. for (int64_t i = st_pos; kv_it_ != kvmap_.end() && i < st_pos + span;
  259. i++, ++kv_it_) {
  260. ASSERT_TRUE(dbiter->Valid());
  261. ASSERT_EQ(dbiter->value().compare(kv_it_->second), 0);
  262. dbiter->Next();
  263. }
  264. }
  265. delete dbiter;
  266. }
  267. // Set ttl on open db
  268. void SetTtl(int32_t ttl, ColumnFamilyHandle* cf = nullptr) {
  269. ASSERT_TRUE(db_ttl_);
  270. cf == nullptr ? db_ttl_->SetTtl(ttl) : db_ttl_->SetTtl(cf, ttl);
  271. }
  272. class TestFilter : public CompactionFilter {
  273. public:
  274. TestFilter(const int64_t kSampleSize, const std::string& kNewValue)
  275. : kSampleSize_(kSampleSize),
  276. kNewValue_(kNewValue) {
  277. }
  278. // Works on keys of the form "key<number>"
  279. // Drops key if number at the end of key is in [0, kSampleSize_/3),
  280. // Keeps key if it is in [kSampleSize_/3, 2*kSampleSize_/3),
  281. // Change value if it is in [2*kSampleSize_/3, kSampleSize_)
  282. // Eg. kSampleSize_=6. Drop:key0-1...Keep:key2-3...Change:key4-5...
  283. bool Filter(int /*level*/, const Slice& key, const Slice& /*value*/,
  284. std::string* new_value, bool* value_changed) const override {
  285. assert(new_value != nullptr);
  286. std::string search_str = "0123456789";
  287. std::string key_string = key.ToString();
  288. size_t pos = key_string.find_first_of(search_str);
  289. int num_key_end;
  290. if (pos != std::string::npos) {
  291. auto key_substr = key_string.substr(pos, key.size() - pos);
  292. #ifndef CYGWIN
  293. num_key_end = std::stoi(key_substr);
  294. #else
  295. num_key_end = std::strtol(key_substr.c_str(), 0, 10);
  296. #endif
  297. } else {
  298. return false; // Keep keys not matching the format "key<NUMBER>"
  299. }
  300. int64_t partition = kSampleSize_ / 3;
  301. if (num_key_end < partition) {
  302. return true;
  303. } else if (num_key_end < partition * 2) {
  304. return false;
  305. } else {
  306. *new_value = kNewValue_;
  307. *value_changed = true;
  308. return false;
  309. }
  310. }
  311. const char* Name() const override { return "TestFilter"; }
  312. private:
  313. const int64_t kSampleSize_;
  314. const std::string kNewValue_;
  315. };
  316. class TestFilterFactory : public CompactionFilterFactory {
  317. public:
  318. TestFilterFactory(const int64_t kSampleSize, const std::string& kNewValue)
  319. : kSampleSize_(kSampleSize),
  320. kNewValue_(kNewValue) {
  321. }
  322. std::unique_ptr<CompactionFilter> CreateCompactionFilter(
  323. const CompactionFilter::Context& /*context*/) override {
  324. return std::unique_ptr<CompactionFilter>(
  325. new TestFilter(kSampleSize_, kNewValue_));
  326. }
  327. const char* Name() const override { return "TestFilterFactory"; }
  328. private:
  329. const int64_t kSampleSize_;
  330. const std::string kNewValue_;
  331. };
  332. // Choose carefully so that Put, Gets & Compaction complete in 1 second buffer
  333. static const int64_t kSampleSize_ = 100;
  334. std::string dbname_;
  335. DBWithTTL* db_ttl_;
  336. std::unique_ptr<SpecialTimeEnv> env_;
  337. private:
  338. Options options_;
  339. KVMap kvmap_;
  340. KVMap::iterator kv_it_;
  341. const std::string kNewValue_ = "new_value";
  342. std::unique_ptr<CompactionFilter> test_comp_filter_;
  343. }; // class TtlTest
  344. // If TTL is non positive or not provided, the behaviour is TTL = infinity
  345. // This test opens the db 3 times with such default behavior and inserts a
  346. // bunch of kvs each time. All kvs should accumulate in the db till the end
  347. // Partitions the sample-size provided into 3 sets over boundary1 and boundary2
  348. TEST_F(TtlTest, NoEffect) {
  349. MakeKVMap(kSampleSize_);
  350. int64_t boundary1 = kSampleSize_ / 3;
  351. int64_t boundary2 = 2 * boundary1;
  352. OpenTtl();
  353. PutValues(0, boundary1); //T=0: Set1 never deleted
  354. SleepCompactCheck(1, 0, boundary1); //T=1: Set1 still there
  355. CloseTtl();
  356. OpenTtl(0);
  357. PutValues(boundary1, boundary2 - boundary1); //T=1: Set2 never deleted
  358. SleepCompactCheck(1, 0, boundary2); //T=2: Sets1 & 2 still there
  359. CloseTtl();
  360. OpenTtl(-1);
  361. PutValues(boundary2, kSampleSize_ - boundary2); //T=3: Set3 never deleted
  362. SleepCompactCheck(1, 0, kSampleSize_, true); //T=4: Sets 1,2,3 still there
  363. CloseTtl();
  364. }
  365. // Rerun the NoEffect test with a different version of CloseTtl
  366. // function, where db is directly deleted without close.
  367. TEST_F(TtlTest, DestructWithoutClose) {
  368. MakeKVMap(kSampleSize_);
  369. int64_t boundary1 = kSampleSize_ / 3;
  370. int64_t boundary2 = 2 * boundary1;
  371. OpenTtl();
  372. PutValues(0, boundary1); // T=0: Set1 never deleted
  373. SleepCompactCheck(1, 0, boundary1); // T=1: Set1 still there
  374. CloseTtlNoDBClose();
  375. OpenTtl(0);
  376. PutValues(boundary1, boundary2 - boundary1); // T=1: Set2 never deleted
  377. SleepCompactCheck(1, 0, boundary2); // T=2: Sets1 & 2 still there
  378. CloseTtlNoDBClose();
  379. OpenTtl(-1);
  380. PutValues(boundary2, kSampleSize_ - boundary2); // T=3: Set3 never deleted
  381. SleepCompactCheck(1, 0, kSampleSize_, true); // T=4: Sets 1,2,3 still there
  382. CloseTtlNoDBClose();
  383. }
  384. // Puts a set of values and checks its presence using Get during ttl
  385. TEST_F(TtlTest, PresentDuringTTL) {
  386. MakeKVMap(kSampleSize_);
  387. OpenTtl(2); // T=0:Open the db with ttl = 2
  388. PutValues(0, kSampleSize_); // T=0:Insert Set1. Delete at t=2
  389. SleepCompactCheck(1, 0, kSampleSize_, true); // T=1:Set1 should still be there
  390. CloseTtl();
  391. }
  392. // Puts a set of values and checks its absence using Get after ttl
  393. TEST_F(TtlTest, AbsentAfterTTL) {
  394. MakeKVMap(kSampleSize_);
  395. OpenTtl(1); // T=0:Open the db with ttl = 2
  396. PutValues(0, kSampleSize_); // T=0:Insert Set1. Delete at t=2
  397. SleepCompactCheck(2, 0, kSampleSize_, false); // T=2:Set1 should not be there
  398. CloseTtl();
  399. }
  400. // Resets the timestamp of a set of kvs by updating them and checks that they
  401. // are not deleted according to the old timestamp
  402. TEST_F(TtlTest, ResetTimestamp) {
  403. MakeKVMap(kSampleSize_);
  404. OpenTtl(3);
  405. PutValues(0, kSampleSize_); // T=0: Insert Set1. Delete at t=3
  406. env_->Sleep(2); // T=2
  407. PutValues(0, kSampleSize_); // T=2: Insert Set1. Delete at t=5
  408. SleepCompactCheck(2, 0, kSampleSize_); // T=4: Set1 should still be there
  409. CloseTtl();
  410. }
  411. // Similar to PresentDuringTTL but uses Iterator
  412. TEST_F(TtlTest, IterPresentDuringTTL) {
  413. MakeKVMap(kSampleSize_);
  414. OpenTtl(2);
  415. PutValues(0, kSampleSize_); // T=0: Insert. Delete at t=2
  416. SleepCompactCheckIter(1, 0, kSampleSize_); // T=1: Set should be there
  417. CloseTtl();
  418. }
  419. // Similar to AbsentAfterTTL but uses Iterator
  420. TEST_F(TtlTest, IterAbsentAfterTTL) {
  421. MakeKVMap(kSampleSize_);
  422. OpenTtl(1);
  423. PutValues(0, kSampleSize_); // T=0: Insert. Delete at t=1
  424. SleepCompactCheckIter(2, 0, kSampleSize_, false); // T=2: Should not be there
  425. CloseTtl();
  426. }
  427. // Checks presence while opening the same db more than once with the same ttl
  428. // Note: The second open will open the same db
  429. TEST_F(TtlTest, MultiOpenSamePresent) {
  430. MakeKVMap(kSampleSize_);
  431. OpenTtl(2);
  432. PutValues(0, kSampleSize_); // T=0: Insert. Delete at t=2
  433. CloseTtl();
  434. OpenTtl(2); // T=0. Delete at t=2
  435. SleepCompactCheck(1, 0, kSampleSize_); // T=1: Set should be there
  436. CloseTtl();
  437. }
  438. // Checks absence while opening the same db more than once with the same ttl
  439. // Note: The second open will open the same db
  440. TEST_F(TtlTest, MultiOpenSameAbsent) {
  441. MakeKVMap(kSampleSize_);
  442. OpenTtl(1);
  443. PutValues(0, kSampleSize_); // T=0: Insert. Delete at t=1
  444. CloseTtl();
  445. OpenTtl(1); // T=0.Delete at t=1
  446. SleepCompactCheck(2, 0, kSampleSize_, false); // T=2: Set should not be there
  447. CloseTtl();
  448. }
  449. // Checks presence while opening the same db more than once with bigger ttl
  450. TEST_F(TtlTest, MultiOpenDifferent) {
  451. MakeKVMap(kSampleSize_);
  452. OpenTtl(1);
  453. PutValues(0, kSampleSize_); // T=0: Insert. Delete at t=1
  454. CloseTtl();
  455. OpenTtl(3); // T=0: Set deleted at t=3
  456. SleepCompactCheck(2, 0, kSampleSize_); // T=2: Set should be there
  457. CloseTtl();
  458. }
  459. // Checks presence during ttl in read_only mode
  460. TEST_F(TtlTest, ReadOnlyPresentForever) {
  461. MakeKVMap(kSampleSize_);
  462. OpenTtl(1); // T=0:Open the db normally
  463. PutValues(0, kSampleSize_); // T=0:Insert Set1. Delete at t=1
  464. CloseTtl();
  465. OpenReadOnlyTtl(1);
  466. SleepCompactCheck(2, 0, kSampleSize_); // T=2:Set1 should still be there
  467. CloseTtl();
  468. }
  469. // Checks whether WriteBatch works well with TTL
  470. // Puts all kvs in kvmap_ in a batch and writes first, then deletes first half
  471. TEST_F(TtlTest, WriteBatchTest) {
  472. MakeKVMap(kSampleSize_);
  473. BatchOperation batch_ops[kSampleSize_];
  474. for (int i = 0; i < kSampleSize_; i++) {
  475. batch_ops[i] = OP_PUT;
  476. }
  477. OpenTtl(2);
  478. MakePutWriteBatch(batch_ops, kSampleSize_);
  479. for (int i = 0; i < kSampleSize_ / 2; i++) {
  480. batch_ops[i] = OP_DELETE;
  481. }
  482. MakePutWriteBatch(batch_ops, kSampleSize_ / 2);
  483. SleepCompactCheck(0, 0, kSampleSize_ / 2, false);
  484. SleepCompactCheck(0, kSampleSize_ / 2, kSampleSize_ - kSampleSize_ / 2);
  485. CloseTtl();
  486. }
  487. // Checks user's compaction filter for correctness with TTL logic
  488. TEST_F(TtlTest, CompactionFilter) {
  489. MakeKVMap(kSampleSize_);
  490. OpenTtlWithTestCompaction(1);
  491. PutValues(0, kSampleSize_); // T=0:Insert Set1. Delete at t=1
  492. // T=2: TTL logic takes precedence over TestFilter:-Set1 should not be there
  493. SleepCompactCheck(2, 0, kSampleSize_, false);
  494. CloseTtl();
  495. OpenTtlWithTestCompaction(3);
  496. PutValues(0, kSampleSize_); // T=0:Insert Set1.
  497. int64_t partition = kSampleSize_ / 3;
  498. SleepCompactCheck(1, 0, partition, false); // Part dropped
  499. SleepCompactCheck(0, partition, partition); // Part kept
  500. SleepCompactCheck(0, 2 * partition, partition, true, true); // Part changed
  501. CloseTtl();
  502. }
  503. // Insert some key-values which KeyMayExist should be able to get and check that
  504. // values returned are fine
  505. TEST_F(TtlTest, KeyMayExist) {
  506. MakeKVMap(kSampleSize_);
  507. OpenTtl();
  508. PutValues(0, kSampleSize_, false);
  509. SimpleKeyMayExistCheck();
  510. CloseTtl();
  511. }
  512. TEST_F(TtlTest, MultiGetTest) {
  513. MakeKVMap(kSampleSize_);
  514. OpenTtl();
  515. PutValues(0, kSampleSize_, false);
  516. SimpleMultiGetTest();
  517. CloseTtl();
  518. }
  519. TEST_F(TtlTest, ColumnFamiliesTest) {
  520. DB* db;
  521. Options options;
  522. options.create_if_missing = true;
  523. options.env = env_.get();
  524. DB::Open(options, dbname_, &db);
  525. ColumnFamilyHandle* handle;
  526. ASSERT_OK(db->CreateColumnFamily(ColumnFamilyOptions(options),
  527. "ttl_column_family", &handle));
  528. delete handle;
  529. delete db;
  530. std::vector<ColumnFamilyDescriptor> column_families;
  531. column_families.push_back(ColumnFamilyDescriptor(
  532. kDefaultColumnFamilyName, ColumnFamilyOptions(options)));
  533. column_families.push_back(ColumnFamilyDescriptor(
  534. "ttl_column_family", ColumnFamilyOptions(options)));
  535. std::vector<ColumnFamilyHandle*> handles;
  536. ASSERT_OK(DBWithTTL::Open(DBOptions(options), dbname_, column_families,
  537. &handles, &db_ttl_, {3, 5}, false));
  538. ASSERT_EQ(handles.size(), 2U);
  539. ColumnFamilyHandle* new_handle;
  540. ASSERT_OK(db_ttl_->CreateColumnFamilyWithTtl(options, "ttl_column_family_2",
  541. &new_handle, 2));
  542. handles.push_back(new_handle);
  543. MakeKVMap(kSampleSize_);
  544. PutValues(0, kSampleSize_, false, handles[0]);
  545. PutValues(0, kSampleSize_, false, handles[1]);
  546. PutValues(0, kSampleSize_, false, handles[2]);
  547. // everything should be there after 1 second
  548. SleepCompactCheck(1, 0, kSampleSize_, true, false, handles[0]);
  549. SleepCompactCheck(0, 0, kSampleSize_, true, false, handles[1]);
  550. SleepCompactCheck(0, 0, kSampleSize_, true, false, handles[2]);
  551. // only column family 1 should be alive after 4 seconds
  552. SleepCompactCheck(3, 0, kSampleSize_, false, false, handles[0]);
  553. SleepCompactCheck(0, 0, kSampleSize_, true, false, handles[1]);
  554. SleepCompactCheck(0, 0, kSampleSize_, false, false, handles[2]);
  555. // nothing should be there after 6 seconds
  556. SleepCompactCheck(2, 0, kSampleSize_, false, false, handles[0]);
  557. SleepCompactCheck(0, 0, kSampleSize_, false, false, handles[1]);
  558. SleepCompactCheck(0, 0, kSampleSize_, false, false, handles[2]);
  559. for (auto h : handles) {
  560. delete h;
  561. }
  562. delete db_ttl_;
  563. db_ttl_ = nullptr;
  564. }
  565. // Puts a set of values and checks its absence using Get after ttl
  566. TEST_F(TtlTest, ChangeTtlOnOpenDb) {
  567. MakeKVMap(kSampleSize_);
  568. OpenTtl(1); // T=0:Open the db with ttl = 2
  569. SetTtl(3);
  570. // @lint-ignore TXT2 T25377293 Grandfathered in
  571. PutValues(0, kSampleSize_); // T=0:Insert Set1. Delete at t=2
  572. SleepCompactCheck(2, 0, kSampleSize_, true); // T=2:Set1 should be there
  573. CloseTtl();
  574. }
  575. } // namespace ROCKSDB_NAMESPACE
  576. // A black-box test for the ttl wrapper around rocksdb
  577. int main(int argc, char** argv) {
  578. ::testing::InitGoogleTest(&argc, argv);
  579. return RUN_ALL_TESTS();
  580. }
  581. #else
  582. #include <stdio.h>
  583. int main(int /*argc*/, char** /*argv*/) {
  584. fprintf(stderr, "SKIPPED as DBWithTTL is not supported in ROCKSDB_LITE\n");
  585. return 0;
  586. }
  587. #endif // !ROCKSDB_LITE