testutil.cc 26 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776
  1. // Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
  2. // This source code is licensed under both the GPLv2 (found in the
  3. // COPYING file in the root directory) and Apache 2.0 License
  4. // (found in the LICENSE.Apache file in the root directory).
  5. //
  6. // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
  7. // Use of this source code is governed by a BSD-style license that can be
  8. // found in the LICENSE file. See the AUTHORS file for names of contributors.
  9. #include "test_util/testutil.h"
  10. #include <fcntl.h>
  11. #include <sys/stat.h>
  12. #include <array>
  13. #include <cctype>
  14. #include <fstream>
  15. #include <sstream>
  16. #include "db/memtable_list.h"
  17. #include "env/composite_env_wrapper.h"
  18. #include "file/random_access_file_reader.h"
  19. #include "file/sequence_file_reader.h"
  20. #include "file/writable_file_writer.h"
  21. #include "port/port.h"
  22. #include "rocksdb/convenience.h"
  23. #include "rocksdb/system_clock.h"
  24. #include "rocksdb/utilities/object_registry.h"
  25. #include "test_util/mock_time_env.h"
  26. #include "test_util/sync_point.h"
  27. #include "util/random.h"
  28. #ifndef ROCKSDB_UNITTESTS_WITH_CUSTOM_OBJECTS_FROM_STATIC_LIBS
  29. void RegisterCustomObjects(int /*argc*/, char** /*argv*/) {}
  30. #endif
  31. namespace ROCKSDB_NAMESPACE::test {
  32. const uint32_t kDefaultFormatVersion = BlockBasedTableOptions().format_version;
  33. const std::set<uint32_t> kFooterFormatVersionsToTest{
  34. // Non-legacy, before big footer changes
  35. 5U,
  36. // After big footer changes
  37. 6U,
  38. // In case any interesting future changes
  39. kDefaultFormatVersion,
  40. kLatestFormatVersion,
  41. };
  42. const ReadOptionsNoIo kReadOptionsNoIo;
  43. std::string RandomKey(Random* rnd, int len, RandomKeyType type) {
  44. // Make sure to generate a wide variety of characters so we
  45. // test the boundary conditions for short-key optimizations.
  46. static const char kTestChars[] = {'\0', '\1', 'a', 'b', 'c',
  47. 'd', 'e', '\xfd', '\xfe', '\xff'};
  48. std::string result;
  49. for (int i = 0; i < len; i++) {
  50. std::size_t indx = 0;
  51. switch (type) {
  52. case RandomKeyType::RANDOM:
  53. indx = rnd->Uniform(sizeof(kTestChars));
  54. break;
  55. case RandomKeyType::LARGEST:
  56. indx = sizeof(kTestChars) - 1;
  57. break;
  58. case RandomKeyType::MIDDLE:
  59. indx = sizeof(kTestChars) / 2;
  60. break;
  61. case RandomKeyType::SMALLEST:
  62. indx = 0;
  63. break;
  64. }
  65. result += kTestChars[indx];
  66. }
  67. return result;
  68. }
  69. const std::vector<UserDefinedTimestampTestMode>& GetUDTTestModes() {
  70. static std::vector<UserDefinedTimestampTestMode> udt_test_modes = {
  71. UserDefinedTimestampTestMode::kStripUserDefinedTimestamp,
  72. UserDefinedTimestampTestMode::kNormal,
  73. UserDefinedTimestampTestMode::kNone};
  74. return udt_test_modes;
  75. }
  76. bool IsUDTEnabled(const UserDefinedTimestampTestMode& test_mode) {
  77. return test_mode != UserDefinedTimestampTestMode::kNone;
  78. }
  79. bool ShouldPersistUDT(const UserDefinedTimestampTestMode& test_mode) {
  80. return test_mode != UserDefinedTimestampTestMode::kStripUserDefinedTimestamp;
  81. }
  82. Slice CompressibleString(Random* rnd, double compressed_to_fraction, int len,
  83. std::string* dst) {
  84. int raw = static_cast<int>(len * compressed_to_fraction);
  85. if (raw < 1) {
  86. raw = 1;
  87. }
  88. std::string raw_data = rnd->RandomBinaryString(raw);
  89. // Duplicate the random data until we have filled "len" bytes
  90. dst->clear();
  91. while (dst->size() < (unsigned int)len) {
  92. dst->append(raw_data);
  93. }
  94. dst->resize(len);
  95. return Slice(*dst);
  96. }
  97. namespace {
  98. class Uint64ComparatorImpl : public Comparator {
  99. public:
  100. Uint64ComparatorImpl() = default;
  101. const char* Name() const override { return "rocksdb.Uint64Comparator"; }
  102. int Compare(const Slice& a, const Slice& b) const override {
  103. assert(a.size() == sizeof(uint64_t) && b.size() == sizeof(uint64_t));
  104. const uint64_t* left = reinterpret_cast<const uint64_t*>(a.data());
  105. const uint64_t* right = reinterpret_cast<const uint64_t*>(b.data());
  106. uint64_t leftValue;
  107. uint64_t rightValue;
  108. GetUnaligned(left, &leftValue);
  109. GetUnaligned(right, &rightValue);
  110. if (leftValue == rightValue) {
  111. return 0;
  112. } else if (leftValue < rightValue) {
  113. return -1;
  114. } else {
  115. return 1;
  116. }
  117. }
  118. void FindShortestSeparator(std::string* /*start*/,
  119. const Slice& /*limit*/) const override {}
  120. void FindShortSuccessor(std::string* /*key*/) const override {}
  121. };
  122. } // namespace
  123. const Comparator* Uint64Comparator() {
  124. static Uint64ComparatorImpl uint64comp;
  125. return &uint64comp;
  126. }
  127. const Comparator* BytewiseComparatorWithU64TsWrapper() {
  128. ConfigOptions config_options;
  129. const Comparator* user_comparator = nullptr;
  130. Status s = Comparator::CreateFromString(
  131. config_options, "leveldb.BytewiseComparator.u64ts", &user_comparator);
  132. s.PermitUncheckedError();
  133. return user_comparator;
  134. }
  135. const Comparator* ReverseBytewiseComparatorWithU64TsWrapper() {
  136. ConfigOptions config_options;
  137. const Comparator* user_comparator = nullptr;
  138. Status s = Comparator::CreateFromString(
  139. config_options, "rocksdb.ReverseBytewiseComparator.u64ts",
  140. &user_comparator);
  141. s.PermitUncheckedError();
  142. return user_comparator;
  143. }
  144. void CorruptKeyType(InternalKey* ikey) {
  145. std::string keystr = ikey->Encode().ToString();
  146. keystr[keystr.size() - 8] = kTypeLogData;
  147. ikey->DecodeFrom(Slice(keystr.data(), keystr.size()));
  148. }
  149. std::string KeyStr(const std::string& user_key, const SequenceNumber& seq,
  150. const ValueType& t, bool corrupt) {
  151. InternalKey k(user_key, seq, t);
  152. if (corrupt) {
  153. CorruptKeyType(&k);
  154. }
  155. return k.Encode().ToString();
  156. }
  157. std::string KeyStr(uint64_t ts, const std::string& user_key,
  158. const SequenceNumber& seq, const ValueType& t,
  159. bool corrupt) {
  160. std::string user_key_with_ts(user_key);
  161. std::string ts_str;
  162. PutFixed64(&ts_str, ts);
  163. user_key_with_ts.append(ts_str);
  164. return KeyStr(user_key_with_ts, seq, t, corrupt);
  165. }
  166. bool SleepingBackgroundTask::TimedWaitUntilSleeping(uint64_t wait_time) {
  167. auto abs_time = SystemClock::Default()->NowMicros() + wait_time;
  168. MutexLock l(&mutex_);
  169. while (!sleeping_ || !should_sleep_) {
  170. if (bg_cv_.TimedWait(abs_time)) {
  171. return true;
  172. }
  173. }
  174. return false;
  175. }
  176. bool SleepingBackgroundTask::TimedWaitUntilDone(uint64_t wait_time) {
  177. auto abs_time = SystemClock::Default()->NowMicros() + wait_time;
  178. MutexLock l(&mutex_);
  179. while (!done_with_sleep_) {
  180. if (bg_cv_.TimedWait(abs_time)) {
  181. return true;
  182. }
  183. }
  184. return false;
  185. }
  186. std::string RandomName(Random* rnd, const size_t len) {
  187. std::stringstream ss;
  188. for (size_t i = 0; i < len; ++i) {
  189. ss << static_cast<char>(rnd->Uniform(26) + 'a');
  190. }
  191. return ss.str();
  192. }
  193. CompressionType RandomCompressionType(Random* rnd) {
  194. auto ret = static_cast<CompressionType>(rnd->Uniform(6));
  195. while (!CompressionTypeSupported(ret)) {
  196. ret = static_cast<CompressionType>((static_cast<int>(ret) + 1) % 6);
  197. }
  198. return ret;
  199. }
  200. void RandomCompressionTypeVector(const size_t count,
  201. std::vector<CompressionType>* types,
  202. Random* rnd) {
  203. types->clear();
  204. for (size_t i = 0; i < count; ++i) {
  205. types->emplace_back(RandomCompressionType(rnd));
  206. }
  207. }
  208. const SliceTransform* RandomSliceTransform(Random* rnd, int pre_defined) {
  209. int random_num = pre_defined >= 0 ? pre_defined : rnd->Uniform(4);
  210. switch (random_num) {
  211. case 0:
  212. return NewFixedPrefixTransform(rnd->Uniform(20) + 1);
  213. case 1:
  214. return NewCappedPrefixTransform(rnd->Uniform(20) + 1);
  215. case 2:
  216. return NewNoopTransform();
  217. default:
  218. return nullptr;
  219. }
  220. }
  221. BlockBasedTableOptions RandomBlockBasedTableOptions(Random* rnd) {
  222. BlockBasedTableOptions opt;
  223. opt.cache_index_and_filter_blocks = rnd->Uniform(2);
  224. opt.pin_l0_filter_and_index_blocks_in_cache = rnd->Uniform(2);
  225. opt.pin_top_level_index_and_filter = rnd->Uniform(2);
  226. using IndexType = BlockBasedTableOptions::IndexType;
  227. const std::array<IndexType, 4> index_types = {
  228. {IndexType::kBinarySearch, IndexType::kHashSearch,
  229. IndexType::kTwoLevelIndexSearch, IndexType::kBinarySearchWithFirstKey}};
  230. opt.index_type =
  231. index_types[rnd->Uniform(static_cast<int>(index_types.size()))];
  232. opt.checksum = static_cast<ChecksumType>(rnd->Uniform(3));
  233. opt.block_size = rnd->Uniform(10000000);
  234. opt.block_size_deviation = rnd->Uniform(100);
  235. opt.block_restart_interval = rnd->Uniform(100);
  236. opt.index_block_restart_interval = rnd->Uniform(100);
  237. opt.whole_key_filtering = rnd->Uniform(2);
  238. return opt;
  239. }
  240. TableFactory* RandomTableFactory(Random* rnd, int pre_defined) {
  241. int random_num = pre_defined >= 0 ? pre_defined : rnd->Uniform(4);
  242. switch (random_num) {
  243. case 0:
  244. return NewPlainTableFactory();
  245. case 1:
  246. return NewCuckooTableFactory();
  247. default:
  248. return NewBlockBasedTableFactory();
  249. }
  250. }
  251. MergeOperator* RandomMergeOperator(Random* rnd) {
  252. return new ChanglingMergeOperator(RandomName(rnd, 10));
  253. }
  254. CompactionFilter* RandomCompactionFilter(Random* rnd) {
  255. return new ChanglingCompactionFilter(RandomName(rnd, 10));
  256. }
  257. CompactionFilterFactory* RandomCompactionFilterFactory(Random* rnd) {
  258. return new ChanglingCompactionFilterFactory(RandomName(rnd, 10));
  259. }
  260. void RandomInitDBOptions(DBOptions* db_opt, Random* rnd) {
  261. // boolean options
  262. db_opt->advise_random_on_open = rnd->Uniform(2);
  263. db_opt->allow_mmap_reads = rnd->Uniform(2);
  264. db_opt->allow_mmap_writes = rnd->Uniform(2);
  265. db_opt->use_direct_reads = rnd->Uniform(2);
  266. db_opt->use_direct_io_for_flush_and_compaction = rnd->Uniform(2);
  267. db_opt->create_if_missing = rnd->Uniform(2);
  268. db_opt->create_missing_column_families = rnd->Uniform(2);
  269. db_opt->enable_thread_tracking = rnd->Uniform(2);
  270. db_opt->error_if_exists = rnd->Uniform(2);
  271. db_opt->is_fd_close_on_exec = rnd->Uniform(2);
  272. db_opt->paranoid_checks = rnd->Uniform(2);
  273. db_opt->track_and_verify_wals_in_manifest = rnd->Uniform(2);
  274. db_opt->track_and_verify_wals = rnd->Uniform(2);
  275. db_opt->verify_sst_unique_id_in_manifest = rnd->Uniform(2);
  276. db_opt->skip_stats_update_on_db_open = rnd->Uniform(2);
  277. db_opt->use_adaptive_mutex = rnd->Uniform(2);
  278. db_opt->use_fsync = rnd->Uniform(2);
  279. db_opt->recycle_log_file_num = rnd->Uniform(2);
  280. db_opt->avoid_flush_during_recovery = rnd->Uniform(2);
  281. db_opt->avoid_flush_during_shutdown = rnd->Uniform(2);
  282. db_opt->enforce_single_del_contracts = rnd->Uniform(2);
  283. // int options
  284. db_opt->max_background_compactions = rnd->Uniform(100);
  285. db_opt->max_background_flushes = rnd->Uniform(100);
  286. db_opt->max_file_opening_threads = rnd->Uniform(100);
  287. db_opt->max_open_files = rnd->Uniform(100);
  288. db_opt->table_cache_numshardbits = rnd->Uniform(100);
  289. // size_t options
  290. db_opt->db_write_buffer_size = rnd->Uniform(10000);
  291. db_opt->keep_log_file_num = rnd->Uniform(10000);
  292. db_opt->log_file_time_to_roll = rnd->Uniform(10000);
  293. db_opt->manifest_preallocation_size = rnd->Uniform(10000);
  294. db_opt->max_log_file_size = rnd->Uniform(10000);
  295. // std::string options
  296. db_opt->db_log_dir = "path/to/db_log_dir";
  297. db_opt->wal_dir = "path/to/wal_dir";
  298. // uint32_t options
  299. db_opt->max_subcompactions = rnd->Uniform(100000);
  300. // uint64_t options
  301. static const uint64_t uint_max = static_cast<uint64_t>(UINT_MAX);
  302. db_opt->WAL_size_limit_MB = uint_max + rnd->Uniform(100000);
  303. db_opt->WAL_ttl_seconds = uint_max + rnd->Uniform(100000);
  304. db_opt->bytes_per_sync = uint_max + rnd->Uniform(100000);
  305. db_opt->delayed_write_rate = uint_max + rnd->Uniform(100000);
  306. db_opt->delete_obsolete_files_period_micros = uint_max + rnd->Uniform(100000);
  307. db_opt->max_manifest_file_size = uint_max + rnd->Uniform(100000);
  308. db_opt->max_total_wal_size = uint_max + rnd->Uniform(100000);
  309. db_opt->wal_bytes_per_sync = uint_max + rnd->Uniform(100000);
  310. // unsigned int options
  311. db_opt->stats_dump_period_sec = rnd->Uniform(100000);
  312. }
  313. void RandomInitCFOptions(ColumnFamilyOptions* cf_opt, DBOptions& db_options,
  314. Random* rnd) {
  315. cf_opt->compaction_style = (CompactionStyle)(rnd->Uniform(4));
  316. // boolean options
  317. cf_opt->report_bg_io_stats = rnd->Uniform(2);
  318. cf_opt->disable_auto_compactions = rnd->Uniform(2);
  319. cf_opt->inplace_update_support = rnd->Uniform(2);
  320. cf_opt->level_compaction_dynamic_level_bytes = rnd->Uniform(2);
  321. cf_opt->optimize_filters_for_hits = rnd->Uniform(2);
  322. cf_opt->paranoid_file_checks = rnd->Uniform(2);
  323. cf_opt->force_consistency_checks = rnd->Uniform(2);
  324. cf_opt->compaction_options_fifo.allow_compaction = rnd->Uniform(2);
  325. cf_opt->memtable_whole_key_filtering = rnd->Uniform(2);
  326. cf_opt->enable_blob_files = rnd->Uniform(2);
  327. cf_opt->enable_blob_garbage_collection = rnd->Uniform(2);
  328. cf_opt->strict_max_successive_merges = rnd->Uniform(2);
  329. // double options
  330. cf_opt->memtable_prefix_bloom_size_ratio =
  331. static_cast<double>(rnd->Uniform(10000)) / 20000.0;
  332. cf_opt->blob_garbage_collection_age_cutoff = rnd->Uniform(10000) / 10000.0;
  333. cf_opt->blob_garbage_collection_force_threshold =
  334. rnd->Uniform(10000) / 10000.0;
  335. // int options
  336. cf_opt->level0_file_num_compaction_trigger = rnd->Uniform(100);
  337. cf_opt->level0_slowdown_writes_trigger = rnd->Uniform(100);
  338. cf_opt->level0_stop_writes_trigger = rnd->Uniform(100);
  339. cf_opt->max_bytes_for_level_multiplier = rnd->Uniform(100);
  340. cf_opt->max_write_buffer_number = rnd->Uniform(100);
  341. cf_opt->max_write_buffer_size_to_maintain = rnd->Uniform(10000);
  342. cf_opt->min_write_buffer_number_to_merge = rnd->Uniform(100);
  343. cf_opt->num_levels = rnd->Uniform(100);
  344. cf_opt->target_file_size_multiplier = rnd->Uniform(100);
  345. // vector int options
  346. cf_opt->max_bytes_for_level_multiplier_additional.resize(cf_opt->num_levels);
  347. for (int i = 0; i < cf_opt->num_levels; i++) {
  348. cf_opt->max_bytes_for_level_multiplier_additional[i] = rnd->Uniform(100);
  349. }
  350. // size_t options
  351. cf_opt->arena_block_size = rnd->Uniform(10000);
  352. cf_opt->inplace_update_num_locks = rnd->Uniform(10000);
  353. cf_opt->max_successive_merges = rnd->Uniform(10000);
  354. cf_opt->memtable_huge_page_size = rnd->Uniform(10000);
  355. cf_opt->write_buffer_size = rnd->Uniform(10000);
  356. // uint32_t options
  357. cf_opt->bloom_locality = rnd->Uniform(10000);
  358. cf_opt->max_bytes_for_level_base = rnd->Uniform(10000);
  359. // uint64_t options
  360. static const uint64_t uint_max = static_cast<uint64_t>(UINT_MAX);
  361. cf_opt->ttl =
  362. db_options.max_open_files == -1 ? uint_max + rnd->Uniform(10000) : 0;
  363. cf_opt->periodic_compaction_seconds =
  364. db_options.max_open_files == -1 ? uint_max + rnd->Uniform(10000) : 0;
  365. cf_opt->max_sequential_skip_in_iterations = uint_max + rnd->Uniform(10000);
  366. cf_opt->target_file_size_base = uint_max + rnd->Uniform(10000);
  367. cf_opt->max_compaction_bytes =
  368. cf_opt->target_file_size_base * rnd->Uniform(100);
  369. cf_opt->compaction_options_fifo.max_table_files_size =
  370. uint_max + rnd->Uniform(10000);
  371. cf_opt->min_blob_size = uint_max + rnd->Uniform(10000);
  372. cf_opt->blob_file_size = uint_max + rnd->Uniform(10000);
  373. cf_opt->blob_compaction_readahead_size = uint_max + rnd->Uniform(10000);
  374. // pointer typed options
  375. cf_opt->prefix_extractor.reset(RandomSliceTransform(rnd));
  376. cf_opt->table_factory.reset(RandomTableFactory(rnd));
  377. cf_opt->merge_operator.reset(RandomMergeOperator(rnd));
  378. if (cf_opt->compaction_filter) {
  379. delete cf_opt->compaction_filter;
  380. }
  381. cf_opt->compaction_filter = RandomCompactionFilter(rnd);
  382. cf_opt->compaction_filter_factory.reset(RandomCompactionFilterFactory(rnd));
  383. // custom typed options
  384. cf_opt->compression = RandomCompressionType(rnd);
  385. RandomCompressionTypeVector(cf_opt->num_levels,
  386. &cf_opt->compression_per_level, rnd);
  387. cf_opt->blob_compression_type = RandomCompressionType(rnd);
  388. }
  389. bool IsDirectIOSupported(Env* env, const std::string& dir) {
  390. EnvOptions env_options;
  391. env_options.use_mmap_writes = false;
  392. env_options.use_direct_writes = true;
  393. std::string tmp = TempFileName(dir, 999);
  394. Status s;
  395. {
  396. std::unique_ptr<WritableFile> file;
  397. s = env->NewWritableFile(tmp, &file, env_options);
  398. }
  399. if (s.ok()) {
  400. s = env->DeleteFile(tmp);
  401. }
  402. return s.ok();
  403. }
  404. bool IsPrefetchSupported(const std::shared_ptr<FileSystem>& fs,
  405. const std::string& dir) {
  406. bool supported = false;
  407. std::string tmp = TempFileName(dir, 999);
  408. Random rnd(301);
  409. std::string test_string = rnd.RandomString(4096);
  410. Slice data(test_string);
  411. IOOptions opts;
  412. Status s = WriteStringToFile(fs.get(), data, tmp, true, opts);
  413. if (s.ok()) {
  414. std::unique_ptr<FSRandomAccessFile> file;
  415. auto io_s = fs->NewRandomAccessFile(tmp, FileOptions(), &file, nullptr);
  416. if (io_s.ok()) {
  417. supported =
  418. !(file->Prefetch(0, data.size(), opts, nullptr).IsNotSupported());
  419. }
  420. s = fs->DeleteFile(tmp, opts, nullptr);
  421. }
  422. return s.ok() && supported;
  423. }
  424. size_t GetLinesCount(const std::string& fname, const std::string& pattern) {
  425. std::stringstream ssbuf;
  426. std::string line;
  427. size_t count = 0;
  428. std::ifstream inFile(fname.c_str());
  429. ssbuf << inFile.rdbuf();
  430. while (getline(ssbuf, line)) {
  431. if (line.find(pattern) != std::string::npos) {
  432. count++;
  433. }
  434. }
  435. return count;
  436. }
  437. Status CorruptFile(Env* env, const std::string& fname, int offset,
  438. int bytes_to_corrupt, bool verify_checksum /*=true*/) {
  439. uint64_t size;
  440. Status s = env->GetFileSize(fname, &size);
  441. if (!s.ok()) {
  442. return s;
  443. } else if (offset < 0) {
  444. // Relative to end of file; make it absolute
  445. if (-offset > static_cast<int>(size)) {
  446. offset = 0;
  447. } else {
  448. offset = static_cast<int>(size + offset);
  449. }
  450. }
  451. if (offset > static_cast<int>(size)) {
  452. offset = static_cast<int>(size);
  453. }
  454. if (offset + bytes_to_corrupt > static_cast<int>(size)) {
  455. bytes_to_corrupt = static_cast<int>(size - offset);
  456. }
  457. // Do it
  458. std::string contents;
  459. s = ReadFileToString(env, fname, &contents);
  460. if (s.ok()) {
  461. for (int i = 0; i < bytes_to_corrupt; i++) {
  462. contents[i + offset] ^= 0x80;
  463. }
  464. s = WriteStringToFile(env, contents, fname, false /* should_sync */);
  465. }
  466. if (s.ok() && verify_checksum) {
  467. Options options;
  468. options.env = env;
  469. EnvOptions env_options;
  470. Status v = VerifySstFileChecksum(options, env_options, fname);
  471. assert(!v.ok());
  472. }
  473. return s;
  474. }
  475. Status TruncateFile(Env* env, const std::string& fname, uint64_t new_length) {
  476. uint64_t old_length;
  477. Status s = env->GetFileSize(fname, &old_length);
  478. if (!s.ok() || new_length == old_length) {
  479. return s;
  480. }
  481. // Do it
  482. std::string contents;
  483. s = ReadFileToString(env, fname, &contents);
  484. if (s.ok()) {
  485. contents.resize(static_cast<size_t>(new_length), 'b');
  486. s = WriteStringToFile(env, contents, fname, false /* should_sync */);
  487. }
  488. return s;
  489. }
  490. // Try and delete a directory if it exists
  491. Status TryDeleteDir(Env* env, const std::string& dirname) {
  492. bool is_dir = false;
  493. Status s = env->IsDirectory(dirname, &is_dir);
  494. if (s.ok() && is_dir) {
  495. s = env->DeleteDir(dirname);
  496. }
  497. return s;
  498. }
  499. // Delete a directory if it exists
  500. void DeleteDir(Env* env, const std::string& dirname) {
  501. TryDeleteDir(env, dirname).PermitUncheckedError();
  502. }
  503. FileType GetFileType(const std::string& path) {
  504. FileType type = kTempFile;
  505. std::size_t found = path.find_last_of('/');
  506. if (found == std::string::npos) {
  507. found = 0;
  508. }
  509. std::string file_name = path.substr(found);
  510. uint64_t number = 0;
  511. ParseFileName(file_name, &number, &type);
  512. return type;
  513. }
  514. uint64_t GetFileNumber(const std::string& path) {
  515. FileType type = kTempFile;
  516. std::size_t found = path.find_last_of('/');
  517. if (found == std::string::npos) {
  518. found = 0;
  519. }
  520. std::string file_name = path.substr(found);
  521. uint64_t number = 0;
  522. ParseFileName(file_name, &number, &type);
  523. return number;
  524. }
  525. Status CreateEnvFromSystem(const ConfigOptions& config_options, Env** result,
  526. std::shared_ptr<Env>* guard) {
  527. const char* env_uri = getenv("TEST_ENV_URI");
  528. const char* fs_uri = getenv("TEST_FS_URI");
  529. if (env_uri || fs_uri) {
  530. return Env::CreateFromUri(config_options,
  531. (env_uri != nullptr) ? env_uri : "",
  532. (fs_uri != nullptr) ? fs_uri : "", result, guard);
  533. } else {
  534. // Neither specified. Use the default
  535. *result = config_options.env;
  536. guard->reset();
  537. return Status::OK();
  538. }
  539. }
  540. namespace {
  541. // A hacky skip list mem table that triggers flush after number of entries.
  542. class SpecialMemTableRep : public MemTableRep {
  543. public:
  544. explicit SpecialMemTableRep(Allocator* allocator, MemTableRep* memtable,
  545. int num_entries_flush)
  546. : MemTableRep(allocator),
  547. memtable_(memtable),
  548. num_entries_flush_(num_entries_flush),
  549. num_entries_(0) {}
  550. KeyHandle Allocate(const size_t len, char** buf) override {
  551. return memtable_->Allocate(len, buf);
  552. }
  553. // Insert key into the list.
  554. // REQUIRES: nothing that compares equal to key is currently in the list.
  555. void Insert(KeyHandle handle) override {
  556. num_entries_++;
  557. memtable_->Insert(handle);
  558. }
  559. void InsertConcurrently(KeyHandle handle) override {
  560. num_entries_++;
  561. memtable_->Insert(handle);
  562. }
  563. // Returns true iff an entry that compares equal to key is in the list.
  564. bool Contains(const char* key) const override {
  565. return memtable_->Contains(key);
  566. }
  567. size_t ApproximateMemoryUsage() override {
  568. // Return a high memory usage when number of entries exceeds the threshold
  569. // to trigger a flush.
  570. return (num_entries_ < num_entries_flush_) ? 0 : 1024 * 1024 * 1024;
  571. }
  572. void Get(const LookupKey& k, void* callback_args,
  573. bool (*callback_func)(void* arg, const char* entry)) override {
  574. memtable_->Get(k, callback_args, callback_func);
  575. }
  576. uint64_t ApproximateNumEntries(const Slice& start_ikey,
  577. const Slice& end_ikey) override {
  578. return memtable_->ApproximateNumEntries(start_ikey, end_ikey);
  579. }
  580. MemTableRep::Iterator* GetIterator(Arena* arena = nullptr) override {
  581. return memtable_->GetIterator(arena);
  582. }
  583. ~SpecialMemTableRep() override = default;
  584. private:
  585. std::unique_ptr<MemTableRep> memtable_;
  586. int num_entries_flush_;
  587. int num_entries_;
  588. };
  589. class SpecialSkipListFactory : public MemTableRepFactory {
  590. public:
  591. static bool Register(ObjectLibrary& library, const std::string& /*arg*/) {
  592. library.AddFactory<MemTableRepFactory>(
  593. ObjectLibrary::PatternEntry(SpecialSkipListFactory::kClassName(), true)
  594. .AddNumber(":"),
  595. [](const std::string& uri, std::unique_ptr<MemTableRepFactory>* guard,
  596. std::string* /* errmsg */) {
  597. auto colon = uri.find(':');
  598. if (colon != std::string::npos) {
  599. auto count = ParseInt(uri.substr(colon + 1));
  600. guard->reset(new SpecialSkipListFactory(count));
  601. } else {
  602. guard->reset(new SpecialSkipListFactory(2));
  603. }
  604. return guard->get();
  605. });
  606. return true;
  607. }
  608. // After number of inserts >= `num_entries_flush` in a mem table, trigger
  609. // flush.
  610. explicit SpecialSkipListFactory(int num_entries_flush)
  611. : num_entries_flush_(num_entries_flush) {}
  612. using MemTableRepFactory::CreateMemTableRep;
  613. MemTableRep* CreateMemTableRep(const MemTableRep::KeyComparator& compare,
  614. Allocator* allocator,
  615. const SliceTransform* transform,
  616. Logger* /*logger*/) override {
  617. return new SpecialMemTableRep(
  618. allocator,
  619. factory_.CreateMemTableRep(compare, allocator, transform, nullptr),
  620. num_entries_flush_);
  621. }
  622. static const char* kClassName() { return "SpecialSkipListFactory"; }
  623. const char* Name() const override { return kClassName(); }
  624. std::string GetId() const override {
  625. std::string id = Name();
  626. if (num_entries_flush_ > 0) {
  627. id.append(":").append(std::to_string(num_entries_flush_));
  628. }
  629. return id;
  630. }
  631. bool IsInsertConcurrentlySupported() const override {
  632. return factory_.IsInsertConcurrentlySupported();
  633. }
  634. private:
  635. SkipListFactory factory_;
  636. int num_entries_flush_;
  637. };
  638. } // namespace
  639. MemTableRepFactory* NewSpecialSkipListFactory(int num_entries_per_flush) {
  640. RegisterTestLibrary();
  641. return new SpecialSkipListFactory(num_entries_per_flush);
  642. }
  643. // This method loads existing test classes into the ObjectRegistry
  644. int RegisterTestObjects(ObjectLibrary& library, const std::string& arg) {
  645. size_t num_types;
  646. library.AddFactory<const Comparator>(
  647. test::SimpleSuffixReverseComparator::kClassName(),
  648. [](const std::string& /*uri*/,
  649. std::unique_ptr<const Comparator>* /*guard*/,
  650. std::string* /* errmsg */) {
  651. static test::SimpleSuffixReverseComparator ssrc;
  652. return &ssrc;
  653. });
  654. SpecialSkipListFactory::Register(library, arg);
  655. library.AddFactory<MergeOperator>(
  656. "Changling",
  657. [](const std::string& uri, std::unique_ptr<MergeOperator>* guard,
  658. std::string* /* errmsg */) {
  659. guard->reset(new test::ChanglingMergeOperator(uri));
  660. return guard->get();
  661. });
  662. library.AddFactory<CompactionFilter>(
  663. "Changling",
  664. [](const std::string& uri, std::unique_ptr<CompactionFilter>* /*guard*/,
  665. std::string* /* errmsg */) {
  666. return new test::ChanglingCompactionFilter(uri);
  667. });
  668. library.AddFactory<CompactionFilterFactory>(
  669. "Changling", [](const std::string& uri,
  670. std::unique_ptr<CompactionFilterFactory>* guard,
  671. std::string* /* errmsg */) {
  672. guard->reset(new test::ChanglingCompactionFilterFactory(uri));
  673. return guard->get();
  674. });
  675. library.AddFactory<SystemClock>(
  676. MockSystemClock::kClassName(),
  677. [](const std::string& /*uri*/, std::unique_ptr<SystemClock>* guard,
  678. std::string* /* errmsg */) {
  679. guard->reset(new MockSystemClock(SystemClock::Default()));
  680. return guard->get();
  681. });
  682. return static_cast<int>(library.GetFactoryCount(&num_types));
  683. }
  684. void RegisterTestLibrary(const std::string& arg) {
  685. static bool registered = false;
  686. if (!registered) {
  687. registered = true;
  688. ObjectRegistry::Default()->AddLibrary("test", RegisterTestObjects, arg);
  689. }
  690. }
  691. const std::string kUnitTestDbId = "UnitTest";
  692. } // namespace ROCKSDB_NAMESPACE::test