blob_file_builder_test.cc 23 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676
  1. // Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
  2. // This source code is licensed under both the GPLv2 (found in the
  3. // COPYING file in the root directory) and Apache 2.0 License
  4. // (found in the LICENSE.Apache file in the root directory).
  5. #include "db/blob/blob_file_builder.h"
  6. #include <cassert>
  7. #include <cinttypes>
  8. #include <string>
  9. #include <utility>
  10. #include <vector>
  11. #include "db/blob/blob_file_addition.h"
  12. #include "db/blob/blob_index.h"
  13. #include "db/blob/blob_log_format.h"
  14. #include "db/blob/blob_log_sequential_reader.h"
  15. #include "env/mock_env.h"
  16. #include "file/filename.h"
  17. #include "file/random_access_file_reader.h"
  18. #include "options/cf_options.h"
  19. #include "rocksdb/env.h"
  20. #include "rocksdb/file_checksum.h"
  21. #include "rocksdb/options.h"
  22. #include "test_util/sync_point.h"
  23. #include "test_util/testharness.h"
  24. #include "util/compression.h"
  25. #include "utilities/fault_injection_env.h"
  26. namespace ROCKSDB_NAMESPACE {
  27. class TestFileNumberGenerator {
  28. public:
  29. uint64_t operator()() { return ++next_file_number_; }
  30. private:
  31. uint64_t next_file_number_ = 1;
  32. };
  33. class BlobFileBuilderTest : public testing::Test {
  34. protected:
  35. BlobFileBuilderTest() {
  36. mock_env_.reset(MockEnv::Create(Env::Default()));
  37. fs_ = mock_env_->GetFileSystem().get();
  38. clock_ = mock_env_->GetSystemClock().get();
  39. write_options_.rate_limiter_priority = Env::IO_HIGH;
  40. }
  41. void VerifyBlobFile(uint64_t blob_file_number,
  42. const std::string& blob_file_path,
  43. uint32_t column_family_id,
  44. CompressionType blob_compression_type,
  45. const std::vector<std::pair<std::string, std::string>>&
  46. expected_key_value_pairs,
  47. const std::vector<std::string>& blob_indexes) {
  48. assert(expected_key_value_pairs.size() == blob_indexes.size());
  49. std::unique_ptr<FSRandomAccessFile> file;
  50. constexpr IODebugContext* dbg = nullptr;
  51. ASSERT_OK(
  52. fs_->NewRandomAccessFile(blob_file_path, file_options_, &file, dbg));
  53. std::unique_ptr<RandomAccessFileReader> file_reader(
  54. new RandomAccessFileReader(std::move(file), blob_file_path, clock_));
  55. constexpr Statistics* statistics = nullptr;
  56. BlobLogSequentialReader blob_log_reader(std::move(file_reader), clock_,
  57. statistics);
  58. BlobLogHeader header;
  59. ASSERT_OK(blob_log_reader.ReadHeader(&header));
  60. ASSERT_EQ(header.version, kVersion1);
  61. ASSERT_EQ(header.column_family_id, column_family_id);
  62. ASSERT_EQ(header.compression, blob_compression_type);
  63. ASSERT_FALSE(header.has_ttl);
  64. ASSERT_EQ(header.expiration_range, ExpirationRange());
  65. for (size_t i = 0; i < expected_key_value_pairs.size(); ++i) {
  66. BlobLogRecord record;
  67. uint64_t blob_offset = 0;
  68. ASSERT_OK(blob_log_reader.ReadRecord(
  69. &record, BlobLogSequentialReader::kReadHeaderKeyBlob, &blob_offset));
  70. // Check the contents of the blob file
  71. const auto& expected_key_value = expected_key_value_pairs[i];
  72. const auto& key = expected_key_value.first;
  73. const auto& value = expected_key_value.second;
  74. ASSERT_EQ(record.key_size, key.size());
  75. ASSERT_EQ(record.value_size, value.size());
  76. ASSERT_EQ(record.expiration, 0);
  77. ASSERT_EQ(record.key, key);
  78. ASSERT_EQ(record.value, value);
  79. // Make sure the blob reference returned by the builder points to the
  80. // right place
  81. BlobIndex blob_index;
  82. ASSERT_OK(blob_index.DecodeFrom(blob_indexes[i]));
  83. ASSERT_FALSE(blob_index.IsInlined());
  84. ASSERT_FALSE(blob_index.HasTTL());
  85. ASSERT_EQ(blob_index.file_number(), blob_file_number);
  86. ASSERT_EQ(blob_index.offset(), blob_offset);
  87. ASSERT_EQ(blob_index.size(), value.size());
  88. }
  89. BlobLogFooter footer;
  90. ASSERT_OK(blob_log_reader.ReadFooter(&footer));
  91. ASSERT_EQ(footer.blob_count, expected_key_value_pairs.size());
  92. ASSERT_EQ(footer.expiration_range, ExpirationRange());
  93. }
  94. std::unique_ptr<Env> mock_env_;
  95. FileSystem* fs_;
  96. SystemClock* clock_;
  97. FileOptions file_options_;
  98. WriteOptions write_options_;
  99. };
  100. TEST_F(BlobFileBuilderTest, BuildAndCheckOneFile) {
  101. // Build a single blob file
  102. constexpr size_t number_of_blobs = 10;
  103. constexpr size_t key_size = 1;
  104. constexpr size_t value_size = 4;
  105. constexpr size_t value_offset = 1234;
  106. Options options;
  107. options.cf_paths.emplace_back(
  108. test::PerThreadDBPath(mock_env_.get(),
  109. "BlobFileBuilderTest_BuildAndCheckOneFile"),
  110. 0);
  111. options.enable_blob_files = true;
  112. options.env = mock_env_.get();
  113. ImmutableOptions immutable_options(options);
  114. MutableCFOptions mutable_cf_options(options);
  115. constexpr int job_id = 1;
  116. constexpr uint32_t column_family_id = 123;
  117. constexpr char column_family_name[] = "foobar";
  118. constexpr Env::WriteLifeTimeHint write_hint = Env::WLTH_MEDIUM;
  119. std::vector<std::string> blob_file_paths;
  120. std::vector<BlobFileAddition> blob_file_additions;
  121. BlobFileBuilder builder(
  122. TestFileNumberGenerator(), fs_, &immutable_options, &mutable_cf_options,
  123. &file_options_, &write_options_, "" /*db_id*/, "" /*db_session_id*/,
  124. job_id, column_family_id, column_family_name, write_hint,
  125. nullptr /*IOTracer*/, nullptr /*BlobFileCompletionCallback*/,
  126. BlobFileCreationReason::kFlush, &blob_file_paths, &blob_file_additions);
  127. std::vector<std::pair<std::string, std::string>> expected_key_value_pairs(
  128. number_of_blobs);
  129. std::vector<std::string> blob_indexes(number_of_blobs);
  130. for (size_t i = 0; i < number_of_blobs; ++i) {
  131. auto& expected_key_value = expected_key_value_pairs[i];
  132. auto& key = expected_key_value.first;
  133. key = std::to_string(i);
  134. assert(key.size() == key_size);
  135. auto& value = expected_key_value.second;
  136. value = std::to_string(i + value_offset);
  137. assert(value.size() == value_size);
  138. auto& blob_index = blob_indexes[i];
  139. ASSERT_OK(builder.Add(key, value, &blob_index));
  140. ASSERT_FALSE(blob_index.empty());
  141. }
  142. ASSERT_OK(builder.Finish());
  143. // Check the metadata generated
  144. constexpr uint64_t blob_file_number = 2;
  145. ASSERT_EQ(blob_file_paths.size(), 1);
  146. const std::string& blob_file_path = blob_file_paths[0];
  147. ASSERT_EQ(
  148. blob_file_path,
  149. BlobFileName(immutable_options.cf_paths.front().path, blob_file_number));
  150. ASSERT_EQ(blob_file_additions.size(), 1);
  151. const auto& blob_file_addition = blob_file_additions[0];
  152. ASSERT_EQ(blob_file_addition.GetBlobFileNumber(), blob_file_number);
  153. ASSERT_EQ(blob_file_addition.GetTotalBlobCount(), number_of_blobs);
  154. ASSERT_EQ(
  155. blob_file_addition.GetTotalBlobBytes(),
  156. number_of_blobs * (BlobLogRecord::kHeaderSize + key_size + value_size));
  157. // Verify the contents of the new blob file as well as the blob references
  158. VerifyBlobFile(blob_file_number, blob_file_path, column_family_id,
  159. kNoCompression, expected_key_value_pairs, blob_indexes);
  160. }
  161. TEST_F(BlobFileBuilderTest, BuildAndCheckMultipleFiles) {
  162. // Build multiple blob files: file size limit is set to the size of a single
  163. // value, so each blob ends up in a file of its own
  164. constexpr size_t number_of_blobs = 10;
  165. constexpr size_t key_size = 1;
  166. constexpr size_t value_size = 10;
  167. constexpr size_t value_offset = 1234567890;
  168. Options options;
  169. options.cf_paths.emplace_back(
  170. test::PerThreadDBPath(mock_env_.get(),
  171. "BlobFileBuilderTest_BuildAndCheckMultipleFiles"),
  172. 0);
  173. options.enable_blob_files = true;
  174. options.blob_file_size = value_size;
  175. options.env = mock_env_.get();
  176. ImmutableOptions immutable_options(options);
  177. MutableCFOptions mutable_cf_options(options);
  178. constexpr int job_id = 1;
  179. constexpr uint32_t column_family_id = 123;
  180. constexpr char column_family_name[] = "foobar";
  181. constexpr Env::WriteLifeTimeHint write_hint = Env::WLTH_MEDIUM;
  182. std::vector<std::string> blob_file_paths;
  183. std::vector<BlobFileAddition> blob_file_additions;
  184. BlobFileBuilder builder(
  185. TestFileNumberGenerator(), fs_, &immutable_options, &mutable_cf_options,
  186. &file_options_, &write_options_, "" /*db_id*/, "" /*db_session_id*/,
  187. job_id, column_family_id, column_family_name, write_hint,
  188. nullptr /*IOTracer*/, nullptr /*BlobFileCompletionCallback*/,
  189. BlobFileCreationReason::kFlush, &blob_file_paths, &blob_file_additions);
  190. std::vector<std::pair<std::string, std::string>> expected_key_value_pairs(
  191. number_of_blobs);
  192. std::vector<std::string> blob_indexes(number_of_blobs);
  193. for (size_t i = 0; i < number_of_blobs; ++i) {
  194. auto& expected_key_value = expected_key_value_pairs[i];
  195. auto& key = expected_key_value.first;
  196. key = std::to_string(i);
  197. assert(key.size() == key_size);
  198. auto& value = expected_key_value.second;
  199. value = std::to_string(i + value_offset);
  200. assert(value.size() == value_size);
  201. auto& blob_index = blob_indexes[i];
  202. ASSERT_OK(builder.Add(key, value, &blob_index));
  203. ASSERT_FALSE(blob_index.empty());
  204. }
  205. ASSERT_OK(builder.Finish());
  206. // Check the metadata generated
  207. ASSERT_EQ(blob_file_paths.size(), number_of_blobs);
  208. ASSERT_EQ(blob_file_additions.size(), number_of_blobs);
  209. for (size_t i = 0; i < number_of_blobs; ++i) {
  210. const uint64_t blob_file_number = i + 2;
  211. ASSERT_EQ(blob_file_paths[i],
  212. BlobFileName(immutable_options.cf_paths.front().path,
  213. blob_file_number));
  214. const auto& blob_file_addition = blob_file_additions[i];
  215. ASSERT_EQ(blob_file_addition.GetBlobFileNumber(), blob_file_number);
  216. ASSERT_EQ(blob_file_addition.GetTotalBlobCount(), 1);
  217. ASSERT_EQ(blob_file_addition.GetTotalBlobBytes(),
  218. BlobLogRecord::kHeaderSize + key_size + value_size);
  219. }
  220. // Verify the contents of the new blob files as well as the blob references
  221. for (size_t i = 0; i < number_of_blobs; ++i) {
  222. std::vector<std::pair<std::string, std::string>> expected_key_value_pair{
  223. expected_key_value_pairs[i]};
  224. std::vector<std::string> blob_index{blob_indexes[i]};
  225. VerifyBlobFile(i + 2, blob_file_paths[i], column_family_id, kNoCompression,
  226. expected_key_value_pair, blob_index);
  227. }
  228. }
  229. TEST_F(BlobFileBuilderTest, InlinedValues) {
  230. // All values are below the min_blob_size threshold; no blob files get written
  231. constexpr size_t number_of_blobs = 10;
  232. constexpr size_t key_size = 1;
  233. constexpr size_t value_size = 10;
  234. constexpr size_t value_offset = 1234567890;
  235. Options options;
  236. options.cf_paths.emplace_back(
  237. test::PerThreadDBPath(mock_env_.get(),
  238. "BlobFileBuilderTest_InlinedValues"),
  239. 0);
  240. options.enable_blob_files = true;
  241. options.min_blob_size = 1024;
  242. options.env = mock_env_.get();
  243. ImmutableOptions immutable_options(options);
  244. MutableCFOptions mutable_cf_options(options);
  245. constexpr int job_id = 1;
  246. constexpr uint32_t column_family_id = 123;
  247. constexpr char column_family_name[] = "foobar";
  248. constexpr Env::WriteLifeTimeHint write_hint = Env::WLTH_MEDIUM;
  249. std::vector<std::string> blob_file_paths;
  250. std::vector<BlobFileAddition> blob_file_additions;
  251. BlobFileBuilder builder(
  252. TestFileNumberGenerator(), fs_, &immutable_options, &mutable_cf_options,
  253. &file_options_, &write_options_, "" /*db_id*/, "" /*db_session_id*/,
  254. job_id, column_family_id, column_family_name, write_hint,
  255. nullptr /*IOTracer*/, nullptr /*BlobFileCompletionCallback*/,
  256. BlobFileCreationReason::kFlush, &blob_file_paths, &blob_file_additions);
  257. for (size_t i = 0; i < number_of_blobs; ++i) {
  258. const std::string key = std::to_string(i);
  259. assert(key.size() == key_size);
  260. const std::string value = std::to_string(i + value_offset);
  261. assert(value.size() == value_size);
  262. std::string blob_index;
  263. ASSERT_OK(builder.Add(key, value, &blob_index));
  264. ASSERT_TRUE(blob_index.empty());
  265. }
  266. ASSERT_OK(builder.Finish());
  267. // Check the metadata generated
  268. ASSERT_TRUE(blob_file_paths.empty());
  269. ASSERT_TRUE(blob_file_additions.empty());
  270. }
  271. TEST_F(BlobFileBuilderTest, Compression) {
  272. // Build a blob file with a compressed blob
  273. if (!Snappy_Supported()) {
  274. return;
  275. }
  276. constexpr size_t key_size = 1;
  277. constexpr size_t value_size = 100;
  278. Options options;
  279. options.cf_paths.emplace_back(
  280. test::PerThreadDBPath(mock_env_.get(), "BlobFileBuilderTest_Compression"),
  281. 0);
  282. options.enable_blob_files = true;
  283. options.blob_compression_type = kSnappyCompression;
  284. options.env = mock_env_.get();
  285. ImmutableOptions immutable_options(options);
  286. MutableCFOptions mutable_cf_options(options);
  287. constexpr int job_id = 1;
  288. constexpr uint32_t column_family_id = 123;
  289. constexpr char column_family_name[] = "foobar";
  290. constexpr Env::WriteLifeTimeHint write_hint = Env::WLTH_MEDIUM;
  291. std::vector<std::string> blob_file_paths;
  292. std::vector<BlobFileAddition> blob_file_additions;
  293. BlobFileBuilder builder(
  294. TestFileNumberGenerator(), fs_, &immutable_options, &mutable_cf_options,
  295. &file_options_, &write_options_, "" /*db_id*/, "" /*db_session_id*/,
  296. job_id, column_family_id, column_family_name, write_hint,
  297. nullptr /*IOTracer*/, nullptr /*BlobFileCompletionCallback*/,
  298. BlobFileCreationReason::kFlush, &blob_file_paths, &blob_file_additions);
  299. const std::string key("1");
  300. const std::string uncompressed_value(value_size, 'x');
  301. std::string blob_index;
  302. ASSERT_OK(builder.Add(key, uncompressed_value, &blob_index));
  303. ASSERT_FALSE(blob_index.empty());
  304. ASSERT_OK(builder.Finish());
  305. // Check the metadata generated
  306. constexpr uint64_t blob_file_number = 2;
  307. ASSERT_EQ(blob_file_paths.size(), 1);
  308. const std::string& blob_file_path = blob_file_paths[0];
  309. ASSERT_EQ(
  310. blob_file_path,
  311. BlobFileName(immutable_options.cf_paths.front().path, blob_file_number));
  312. ASSERT_EQ(blob_file_additions.size(), 1);
  313. const auto& blob_file_addition = blob_file_additions[0];
  314. ASSERT_EQ(blob_file_addition.GetBlobFileNumber(), blob_file_number);
  315. ASSERT_EQ(blob_file_addition.GetTotalBlobCount(), 1);
  316. CompressionOptions opts;
  317. CompressionContext context(kSnappyCompression, opts);
  318. CompressionInfo info(opts, context, CompressionDict::GetEmptyDict(),
  319. kSnappyCompression);
  320. std::string compressed_value;
  321. ASSERT_TRUE(Snappy_Compress(info, uncompressed_value.data(),
  322. uncompressed_value.size(), &compressed_value));
  323. ASSERT_EQ(blob_file_addition.GetTotalBlobBytes(),
  324. BlobLogRecord::kHeaderSize + key_size + compressed_value.size());
  325. // Verify the contents of the new blob file as well as the blob reference
  326. std::vector<std::pair<std::string, std::string>> expected_key_value_pairs{
  327. {key, compressed_value}};
  328. std::vector<std::string> blob_indexes{blob_index};
  329. VerifyBlobFile(blob_file_number, blob_file_path, column_family_id,
  330. kSnappyCompression, expected_key_value_pairs, blob_indexes);
  331. }
  332. TEST_F(BlobFileBuilderTest, CompressionError) {
  333. // Simulate an error during compression
  334. if (!Snappy_Supported()) {
  335. return;
  336. }
  337. Options options;
  338. options.cf_paths.emplace_back(
  339. test::PerThreadDBPath(mock_env_.get(),
  340. "BlobFileBuilderTest_CompressionError"),
  341. 0);
  342. options.enable_blob_files = true;
  343. options.blob_compression_type = kSnappyCompression;
  344. options.env = mock_env_.get();
  345. ImmutableOptions immutable_options(options);
  346. MutableCFOptions mutable_cf_options(options);
  347. constexpr int job_id = 1;
  348. constexpr uint32_t column_family_id = 123;
  349. constexpr char column_family_name[] = "foobar";
  350. constexpr Env::WriteLifeTimeHint write_hint = Env::WLTH_MEDIUM;
  351. std::vector<std::string> blob_file_paths;
  352. std::vector<BlobFileAddition> blob_file_additions;
  353. BlobFileBuilder builder(
  354. TestFileNumberGenerator(), fs_, &immutable_options, &mutable_cf_options,
  355. &file_options_, &write_options_, "" /*db_id*/, "" /*db_session_id*/,
  356. job_id, column_family_id, column_family_name, write_hint,
  357. nullptr /*IOTracer*/, nullptr /*BlobFileCompletionCallback*/,
  358. BlobFileCreationReason::kFlush, &blob_file_paths, &blob_file_additions);
  359. SyncPoint::GetInstance()->SetCallBack("CompressData:TamperWithReturnValue",
  360. [](void* arg) {
  361. bool* ret = static_cast<bool*>(arg);
  362. *ret = false;
  363. });
  364. SyncPoint::GetInstance()->EnableProcessing();
  365. constexpr char key[] = "1";
  366. constexpr char value[] = "deadbeef";
  367. std::string blob_index;
  368. ASSERT_TRUE(builder.Add(key, value, &blob_index).IsCorruption());
  369. SyncPoint::GetInstance()->DisableProcessing();
  370. SyncPoint::GetInstance()->ClearAllCallBacks();
  371. constexpr uint64_t blob_file_number = 2;
  372. ASSERT_EQ(blob_file_paths.size(), 1);
  373. ASSERT_EQ(
  374. blob_file_paths[0],
  375. BlobFileName(immutable_options.cf_paths.front().path, blob_file_number));
  376. ASSERT_TRUE(blob_file_additions.empty());
  377. }
  378. TEST_F(BlobFileBuilderTest, Checksum) {
  379. // Build a blob file with checksum
  380. class DummyFileChecksumGenerator : public FileChecksumGenerator {
  381. public:
  382. void Update(const char* /* data */, size_t /* n */) override {}
  383. void Finalize() override {}
  384. std::string GetChecksum() const override { return std::string("dummy"); }
  385. const char* Name() const override { return "DummyFileChecksum"; }
  386. };
  387. class DummyFileChecksumGenFactory : public FileChecksumGenFactory {
  388. public:
  389. std::unique_ptr<FileChecksumGenerator> CreateFileChecksumGenerator(
  390. const FileChecksumGenContext& /* context */) override {
  391. return std::unique_ptr<FileChecksumGenerator>(
  392. new DummyFileChecksumGenerator);
  393. }
  394. const char* Name() const override { return "DummyFileChecksumGenFactory"; }
  395. };
  396. Options options;
  397. options.cf_paths.emplace_back(
  398. test::PerThreadDBPath(mock_env_.get(), "BlobFileBuilderTest_Checksum"),
  399. 0);
  400. options.enable_blob_files = true;
  401. options.file_checksum_gen_factory =
  402. std::make_shared<DummyFileChecksumGenFactory>();
  403. options.env = mock_env_.get();
  404. ImmutableOptions immutable_options(options);
  405. MutableCFOptions mutable_cf_options(options);
  406. constexpr int job_id = 1;
  407. constexpr uint32_t column_family_id = 123;
  408. constexpr char column_family_name[] = "foobar";
  409. constexpr Env::WriteLifeTimeHint write_hint = Env::WLTH_MEDIUM;
  410. std::vector<std::string> blob_file_paths;
  411. std::vector<BlobFileAddition> blob_file_additions;
  412. BlobFileBuilder builder(
  413. TestFileNumberGenerator(), fs_, &immutable_options, &mutable_cf_options,
  414. &file_options_, &write_options_, "" /*db_id*/, "" /*db_session_id*/,
  415. job_id, column_family_id, column_family_name, write_hint,
  416. nullptr /*IOTracer*/, nullptr /*BlobFileCompletionCallback*/,
  417. BlobFileCreationReason::kFlush, &blob_file_paths, &blob_file_additions);
  418. const std::string key("1");
  419. const std::string value("deadbeef");
  420. std::string blob_index;
  421. ASSERT_OK(builder.Add(key, value, &blob_index));
  422. ASSERT_FALSE(blob_index.empty());
  423. ASSERT_OK(builder.Finish());
  424. // Check the metadata generated
  425. constexpr uint64_t blob_file_number = 2;
  426. ASSERT_EQ(blob_file_paths.size(), 1);
  427. const std::string& blob_file_path = blob_file_paths[0];
  428. ASSERT_EQ(
  429. blob_file_path,
  430. BlobFileName(immutable_options.cf_paths.front().path, blob_file_number));
  431. ASSERT_EQ(blob_file_additions.size(), 1);
  432. const auto& blob_file_addition = blob_file_additions[0];
  433. ASSERT_EQ(blob_file_addition.GetBlobFileNumber(), blob_file_number);
  434. ASSERT_EQ(blob_file_addition.GetTotalBlobCount(), 1);
  435. ASSERT_EQ(blob_file_addition.GetTotalBlobBytes(),
  436. BlobLogRecord::kHeaderSize + key.size() + value.size());
  437. ASSERT_EQ(blob_file_addition.GetChecksumMethod(), "DummyFileChecksum");
  438. ASSERT_EQ(blob_file_addition.GetChecksumValue(), "dummy");
  439. // Verify the contents of the new blob file as well as the blob reference
  440. std::vector<std::pair<std::string, std::string>> expected_key_value_pairs{
  441. {key, value}};
  442. std::vector<std::string> blob_indexes{blob_index};
  443. VerifyBlobFile(blob_file_number, blob_file_path, column_family_id,
  444. kNoCompression, expected_key_value_pairs, blob_indexes);
  445. }
  446. class BlobFileBuilderIOErrorTest
  447. : public testing::Test,
  448. public testing::WithParamInterface<std::string> {
  449. protected:
  450. BlobFileBuilderIOErrorTest() : sync_point_(GetParam()) {
  451. mock_env_.reset(MockEnv::Create(Env::Default()));
  452. fs_ = mock_env_->GetFileSystem().get();
  453. write_options_.rate_limiter_priority = Env::IO_HIGH;
  454. }
  455. std::unique_ptr<Env> mock_env_;
  456. FileSystem* fs_;
  457. FileOptions file_options_;
  458. WriteOptions write_options_;
  459. std::string sync_point_;
  460. };
  461. INSTANTIATE_TEST_CASE_P(
  462. BlobFileBuilderTest, BlobFileBuilderIOErrorTest,
  463. ::testing::ValuesIn(std::vector<std::string>{
  464. "BlobFileBuilder::OpenBlobFileIfNeeded:NewWritableFile",
  465. "BlobFileBuilder::OpenBlobFileIfNeeded:WriteHeader",
  466. "BlobFileBuilder::WriteBlobToFile:AddRecord",
  467. "BlobFileBuilder::WriteBlobToFile:AppendFooter"}));
  468. TEST_P(BlobFileBuilderIOErrorTest, IOError) {
  469. // Simulate an I/O error during the specified step of Add()
  470. // Note: blob_file_size will be set to value_size in order for the first blob
  471. // to trigger close
  472. constexpr size_t value_size = 8;
  473. Options options;
  474. options.cf_paths.emplace_back(
  475. test::PerThreadDBPath(mock_env_.get(),
  476. "BlobFileBuilderIOErrorTest_IOError"),
  477. 0);
  478. options.enable_blob_files = true;
  479. options.blob_file_size = value_size;
  480. options.env = mock_env_.get();
  481. ImmutableOptions immutable_options(options);
  482. MutableCFOptions mutable_cf_options(options);
  483. constexpr int job_id = 1;
  484. constexpr uint32_t column_family_id = 123;
  485. constexpr char column_family_name[] = "foobar";
  486. constexpr Env::WriteLifeTimeHint write_hint = Env::WLTH_MEDIUM;
  487. std::vector<std::string> blob_file_paths;
  488. std::vector<BlobFileAddition> blob_file_additions;
  489. BlobFileBuilder builder(
  490. TestFileNumberGenerator(), fs_, &immutable_options, &mutable_cf_options,
  491. &file_options_, &write_options_, "" /*db_id*/, "" /*db_session_id*/,
  492. job_id, column_family_id, column_family_name, write_hint,
  493. nullptr /*IOTracer*/, nullptr /*BlobFileCompletionCallback*/,
  494. BlobFileCreationReason::kFlush, &blob_file_paths, &blob_file_additions);
  495. SyncPoint::GetInstance()->SetCallBack(sync_point_, [this](void* arg) {
  496. Status* const s = static_cast<Status*>(arg);
  497. assert(s);
  498. (*s) = Status::IOError(sync_point_);
  499. });
  500. SyncPoint::GetInstance()->EnableProcessing();
  501. constexpr char key[] = "1";
  502. constexpr char value[] = "deadbeef";
  503. std::string blob_index;
  504. ASSERT_TRUE(builder.Add(key, value, &blob_index).IsIOError());
  505. SyncPoint::GetInstance()->DisableProcessing();
  506. SyncPoint::GetInstance()->ClearAllCallBacks();
  507. if (sync_point_ == "BlobFileBuilder::OpenBlobFileIfNeeded:NewWritableFile") {
  508. ASSERT_TRUE(blob_file_paths.empty());
  509. } else {
  510. constexpr uint64_t blob_file_number = 2;
  511. ASSERT_EQ(blob_file_paths.size(), 1);
  512. ASSERT_EQ(blob_file_paths[0],
  513. BlobFileName(immutable_options.cf_paths.front().path,
  514. blob_file_number));
  515. }
  516. ASSERT_TRUE(blob_file_additions.empty());
  517. }
  518. } // namespace ROCKSDB_NAMESPACE
  519. int main(int argc, char** argv) {
  520. ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
  521. ::testing::InitGoogleTest(&argc, argv);
  522. return RUN_ALL_TESTS();
  523. }