name: facebook/rocksdb/pr-jobs on: [push, pull_request] permissions: {} jobs: # NOTE: multiple workflows would be recommended, but the current GHA UI in # PRs doesn't make it clear when there's an overall error with a workflow, # making it easy to overlook something broken. Grouping everything into one # workflow minimizes the problem because it will be suspicious if there are # no GHA results. # # The if: ${{ github.repository_owner == 'facebook' }} lines prevent the # jobs from attempting to run on repo forks, because of a few problems: # * runs-on labels are repository (owner) specific, so the job might wait # for days waiting for a runner that simply isn't available. # * Pushes to branches on forks for pull requests (the normal process) would # run the workflow jobs twice: once in the pull-from fork and once for the PR # destination repo. This is wasteful and dumb. # * It is not known how to avoid copy-pasting the line to each job, # increasing the risk of misconfiguration, especially on forks that might # want to run with this GHA setup. # # DEBUGGING WITH SSH: Temporarily add this as a job step, either before the # step of interest without the "if:" line or after the failing step with the # "if:" line. Then use ssh command printed in CI output. # - name: Setup tmate session # TEMPORARY! # if: ${{ failure() }} # uses: mxschmitt/action-tmate@v3 # with: # limit-access-to-actor: true # ======================== Fast Initial Checks ====================== # check-format-and-targets: if: ${{ github.repository_owner == 'facebook' }} runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4.1.0 with: fetch-depth: 0 # Need full checkout to determine merge base fetch-tags: true - uses: "./.github/actions/setup-upstream" - name: Setup Python uses: actions/setup-python@v5 - name: Install Dependencies run: python -m pip install --upgrade pip - name: Install argparse run: pip install argparse - name: Download clang-format-diff.py run: wget https://rocksdb-deps.s3.us-west-2.amazonaws.com/llvm/llvm-project/release/12.x/clang/tools/clang-format/clang-format-diff.py - name: Check format run: VERBOSE_CHECK=1 make check-format - name: Compare buckify output run: make check-buck-targets - name: Simple source code checks run: make check-sources - name: Sanity check check_format_compatible.sh run: |- export TEST_TMPDIR=/dev/shm/rocksdb rm -rf /dev/shm/rocksdb mkdir /dev/shm/rocksdb git reset --hard git config --global --add safe.directory /__w/rocksdb/rocksdb SANITY_CHECK=1 LONG_TEST=1 tools/check_format_compatible.sh # ========================= Linux With Tests ======================== # build-linux: if: ${{ github.repository_owner == 'facebook' }} runs-on: labels: 16-core-ubuntu container: image: ghcr.io/facebook/rocksdb_ubuntu:22.1 options: --shm-size=16gb steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" - run: make V=1 J=32 -j32 check - uses: "./.github/actions/post-steps" build-linux-cmake-mingw: if: ${{ github.repository_owner == 'facebook' }} runs-on: labels: 4-core-ubuntu container: image: ghcr.io/facebook/rocksdb_ubuntu:24.0 options: --shm-size=16gb steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" - run: update-alternatives --set x86_64-w64-mingw32-g++ /usr/bin/x86_64-w64-mingw32-g++-posix - name: Build cmake-mingw run: |- export PATH=$JAVA_HOME/bin:$PATH echo "JAVA_HOME=${JAVA_HOME}" which java && java -version which javac && javac -version mkdir build && cd build && cmake -DJNI=1 -DWITH_GFLAGS=OFF .. -DCMAKE_C_COMPILER=x86_64-w64-mingw32-gcc -DCMAKE_CXX_COMPILER=x86_64-w64-mingw32-g++ -DCMAKE_SYSTEM_NAME=Windows && make -j4 rocksdb rocksdbjni - uses: "./.github/actions/post-steps" build-linux-cmake-with-folly: if: ${{ github.repository_owner == 'facebook' }} runs-on: labels: 16-core-ubuntu container: image: ghcr.io/facebook/rocksdb_ubuntu:22.1 options: --shm-size=16gb steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" - uses: "./.github/actions/setup-folly" - uses: "./.github/actions/cache-folly" id: cache-folly - uses: "./.github/actions/build-folly" with: cache-hit: ${{ steps.cache-folly.outputs.cache-hit }} - run: "(mkdir build && cd build && cmake -DUSE_FOLLY=1 -DWITH_GFLAGS=1 -DROCKSDB_BUILD_SHARED=0 .. && make V=1 -j20 && ctest -j20)" - uses: "./.github/actions/post-steps" build-linux-make-with-folly: if: ${{ github.repository_owner == 'facebook' }} runs-on: labels: 16-core-ubuntu container: image: ghcr.io/facebook/rocksdb_ubuntu:22.1 options: --shm-size=16gb steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" - uses: "./.github/actions/setup-folly" - uses: "./.github/actions/cache-folly" id: cache-folly - uses: "./.github/actions/build-folly" with: cache-hit: ${{ steps.cache-folly.outputs.cache-hit }} - run: USE_FOLLY=1 LIB_MODE=static V=1 make -j32 check - uses: "./.github/actions/post-steps" build-linux-make-with-folly-lite-no-test: if: ${{ github.repository_owner == 'facebook' }} runs-on: labels: 16-core-ubuntu container: image: ghcr.io/facebook/rocksdb_ubuntu:22.1 options: --shm-size=16gb steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" - uses: "./.github/actions/setup-folly" - run: USE_FOLLY_LITE=1 EXTRA_CXXFLAGS=-DGLOG_USE_GLOG_EXPORT V=1 make -j32 all - uses: "./.github/actions/post-steps" build-linux-cmake-with-folly-coroutines: if: ${{ github.repository_owner == 'facebook' }} runs-on: labels: 16-core-ubuntu container: image: ghcr.io/facebook/rocksdb_ubuntu:22.1 options: --shm-size=16gb steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" - uses: "./.github/actions/setup-folly" - uses: "./.github/actions/cache-folly" id: cache-folly - uses: "./.github/actions/build-folly" with: cache-hit: ${{ steps.cache-folly.outputs.cache-hit }} - run: "(mkdir build && cd build && cmake -DUSE_COROUTINES=1 -DWITH_GFLAGS=1 -DROCKSDB_BUILD_SHARED=0 .. && make V=1 -j20 && ctest -j20)" - uses: "./.github/actions/post-steps" build-linux-cmake-with-benchmark: if: ${{ github.repository_owner == 'facebook' }} runs-on: labels: 16-core-ubuntu container: image: ghcr.io/facebook/rocksdb_ubuntu:22.1 options: --shm-size=16gb steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" - run: mkdir build && cd build && cmake -DWITH_GFLAGS=1 -DWITH_BENCHMARK=1 .. && make V=1 -j20 && ctest -j20 - uses: "./.github/actions/post-steps" build-linux-encrypted_env-no_compression: if: ${{ github.repository_owner == 'facebook' }} runs-on: labels: 16-core-ubuntu container: image: ghcr.io/facebook/rocksdb_ubuntu:22.1 options: --shm-size=16gb steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" - run: ENCRYPTED_ENV=1 ROCKSDB_DISABLE_SNAPPY=1 ROCKSDB_DISABLE_ZLIB=1 ROCKSDB_DISABLE_BZIP=1 ROCKSDB_DISABLE_LZ4=1 ROCKSDB_DISABLE_ZSTD=1 make V=1 J=32 -j32 check - run: "./sst_dump --help | grep -E -q 'Supported built-in compression types: kNoCompression$' # Verify no compiled in compression\n" - uses: "./.github/actions/post-steps" # ======================== Linux No Test Runs ======================= # build-linux-release: if: ${{ github.repository_owner == 'facebook' }} runs-on: labels: 16-core-ubuntu container: image: ghcr.io/facebook/rocksdb_ubuntu:22.1 options: --shm-size=16gb steps: - uses: actions/checkout@v4.1.0 - run: make V=1 -j32 LIB_MODE=shared release - run: ls librocksdb.so - run: "./trace_analyzer --version" # A tool dependent on gflags that can run in release build - run: make clean - run: USE_RTTI=1 make V=1 -j32 release - run: ls librocksdb.a - run: "./trace_analyzer --version" - run: make clean - run: apt-get remove -y libgflags-dev - run: make V=1 -j32 LIB_MODE=shared release - run: ls librocksdb.so - run: if ./trace_analyzer --version; then false; else true; fi - run: make clean - run: USE_RTTI=1 make V=1 -j32 release - run: ls librocksdb.a - run: if ./trace_analyzer --version; then false; else true; fi - uses: "./.github/actions/post-steps" build-linux-clang-13-no_test_run: if: ${{ github.repository_owner == 'facebook' }} runs-on: labels: 8-core-ubuntu container: image: ghcr.io/facebook/rocksdb_ubuntu:22.1 options: --shm-size=16gb steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" # FIXME: get back to "all microbench" targets - run: CC=clang-13 CXX=clang++-13 USE_CLANG=1 EXTRA_CXXFLAGS=-stdlib=libc++ EXTRA_LDFLAGS=-stdlib=libc++ make -j32 shared_lib - run: make clean # FIXME: get back to "release" target - run: CC=clang-13 CXX=clang++-13 USE_CLANG=1 EXTRA_CXXFLAGS=-stdlib=libc++ EXTRA_LDFLAGS=-stdlib=libc++ DEBUG_LEVEL=0 make -j32 shared_lib - uses: "./.github/actions/post-steps" build-linux-clang-18-no_test_run: if: ${{ github.repository_owner == 'facebook' }} runs-on: labels: 16-core-ubuntu container: image: ghcr.io/facebook/rocksdb_ubuntu:24.0 options: --shm-size=16gb steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" - run: CC=clang-18 CXX=clang++-18 USE_CLANG=1 make -j32 all microbench - run: make clean - run: CC=clang-18 CXX=clang++-18 USE_CLANG=1 DEBUG_LEVEL=0 make -j32 release - uses: "./.github/actions/post-steps" build-linux-gcc-14-no_test_run: if: ${{ github.repository_owner == 'facebook' }} runs-on: labels: 16-core-ubuntu container: image: ghcr.io/facebook/rocksdb_ubuntu:24.0 options: --shm-size=16gb steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" - run: CC=gcc-14 CXX=g++-14 V=1 make -j32 all microbench - uses: "./.github/actions/post-steps" # ======================== Linux Other Checks ======================= # build-linux-clang18-clang-analyze: if: ${{ github.repository_owner == 'facebook' }} runs-on: labels: 16-core-ubuntu container: image: ghcr.io/facebook/rocksdb_ubuntu:24.0 options: --shm-size=16gb steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" - run: CC=clang-18 CXX=clang++-18 ROCKSDB_DISABLE_ALIGNED_NEW=1 CLANG_ANALYZER="/usr/bin/clang++-18" CLANG_SCAN_BUILD=scan-build-18 USE_CLANG=1 make V=1 -j32 analyze - uses: "./.github/actions/post-steps" - name: compress test report run: tar -cvzf scan_build_report.tar.gz scan_build_report if: failure() - uses: actions/upload-artifact@v4.0.0 with: name: scan-build-report path: scan_build_report.tar.gz build-linux-unity-and-headers: if: ${{ github.repository_owner == 'facebook' }} runs-on: labels: 4-core-ubuntu container: image: gcc:latest options: --shm-size=16gb steps: - uses: actions/checkout@v4.1.0 - run: apt-get update -y && apt-get install -y libgflags-dev - name: Unity build run: make V=1 -j8 unity_test - run: make V=1 -j8 -k check-headers - uses: "./.github/actions/post-steps" build-linux-mini-crashtest: if: ${{ github.repository_owner == 'facebook' }} runs-on: labels: 4-core-ubuntu container: image: ghcr.io/facebook/rocksdb_ubuntu:22.1 options: --shm-size=16gb steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" - run: ulimit -S -n `ulimit -H -n` && make V=1 -j8 CRASH_TEST_EXT_ARGS='--duration=960 --max_key=2500000' blackbox_crash_test_with_atomic_flush - uses: "./.github/actions/post-steps" # ======================= Linux with Sanitizers ===================== # build-linux-clang18-asan-ubsan: if: ${{ github.repository_owner == 'facebook' }} runs-on: labels: 32-core-ubuntu container: image: ghcr.io/facebook/rocksdb_ubuntu:24.0 options: --shm-size=16gb steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" - run: COMPILE_WITH_ASAN=1 COMPILE_WITH_UBSAN=1 CC=clang-18 CXX=clang++-18 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j40 check - uses: "./.github/actions/post-steps" build-linux-clang18-mini-tsan: if: ${{ github.repository_owner == 'facebook' }} runs-on: labels: 32-core-ubuntu container: image: ghcr.io/facebook/rocksdb_ubuntu:24.0 options: --shm-size=16gb steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" - run: COMPILE_WITH_TSAN=1 CC=clang-18 CXX=clang++-18 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 check - uses: "./.github/actions/post-steps" build-linux-static_lib-alt_namespace-status_checked: if: ${{ github.repository_owner == 'facebook' }} runs-on: labels: 16-core-ubuntu container: image: ghcr.io/facebook/rocksdb_ubuntu:22.1 options: --shm-size=16gb steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" - run: ASSERT_STATUS_CHECKED=1 TEST_UINT128_COMPAT=1 ROCKSDB_MODIFY_NPHASH=1 LIB_MODE=static OPT="-DROCKSDB_USE_STD_SEMAPHORES -DROCKSDB_NAMESPACE=alternative_rocksdb_ns" make V=1 -j24 check - uses: "./.github/actions/post-steps" # ========================= MacOS build only ======================== # build-macos: if: ${{ github.repository_owner == 'facebook' }} runs-on: macos-15-xlarge env: ROCKSDB_DISABLE_JEMALLOC: 1 steps: - uses: actions/checkout@v4.1.0 - uses: maxim-lobanov/setup-xcode@v1.6.0 with: xcode-version: 16.4.0 - uses: "./.github/actions/increase-max-open-files-on-macos" - uses: "./.github/actions/install-gflags-on-macos" - uses: "./.github/actions/pre-steps-macos" - name: Build run: ulimit -S -n `ulimit -H -n` && make V=1 J=16 -j8 all - uses: "./.github/actions/post-steps" # ========================= MacOS with Tests ======================== # build-macos-cmake: if: ${{ github.repository_owner == 'facebook' }} runs-on: macos-15-xlarge strategy: matrix: run_sharded_tests: [0, 1, 2, 3] steps: - uses: actions/checkout@v4.1.0 - uses: maxim-lobanov/setup-xcode@v1.6.0 with: xcode-version: 16.4.0 - uses: "./.github/actions/increase-max-open-files-on-macos" - uses: "./.github/actions/install-gflags-on-macos" - uses: "./.github/actions/pre-steps-macos" - name: cmake generate project file run: ulimit -S -n `ulimit -H -n` && mkdir build && cd build && cmake -DWITH_GFLAGS=1 .. - name: Build tests run: cd build && make V=1 -j8 - name: Run shard 0 out of 4 test shards run: ulimit -S -n `ulimit -H -n` && cd build && ctest -j8 -I 0,,4 if: ${{ matrix.run_sharded_tests == 0 }} - name: Run shard 1 out of 4 test shards run: ulimit -S -n `ulimit -H -n` && cd build && ctest -j8 -I 1,,4 if: ${{ matrix.run_sharded_tests == 1 }} - name: Run shard 2 out of 4 test shards run: ulimit -S -n `ulimit -H -n` && cd build && ctest -j8 -I 2,,4 if: ${{ matrix.run_sharded_tests == 2 }} - name: Run shard 3 out of 4 test shards run: ulimit -S -n `ulimit -H -n` && cd build && ctest -j8 -I 3,,4 if: ${{ matrix.run_sharded_tests == 3 }} - uses: "./.github/actions/post-steps" # ======================== Windows with Tests ======================= # # NOTE: some windows jobs are in "nightly" to save resources build-windows-vs2022: if: ${{ github.repository_owner == 'facebook' }} runs-on: windows-8-core env: CMAKE_GENERATOR: Visual Studio 17 2022 CMAKE_PORTABLE: 1 steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/windows-build-steps" # ============================ Java Jobs ============================ # build-linux-java: if: ${{ github.repository_owner == 'facebook' }} runs-on: labels: 4-core-ubuntu container: image: ghcr.io/facebook/rocksdb_ubuntu:22.1 options: --shm-size=16gb steps: # The docker image is intentionally based on an OS that has an older GLIBC version. # That GLIBC is incompatibile with GitHub's actions/checkout. Thus we implement a manual checkout step. # NOTE: replaced evolvedbinary/rocksjava:centos7_x64-be with ghcr.io/facebook/rocksdb_ubuntu:22.1 # until a more appropriate docker image with C++20 support is made. - name: Checkout env: GH_TOKEN: ${{ github.token }} run: | chown `whoami` . || true git clone --no-checkout https://oath2:$GH_TOKEN@github.com/${{ github.repository }}.git . git -c protocol.version=2 fetch --update-head-ok --no-tags --prune --no-recurse-submodules --depth=1 origin +${{ github.sha }}:${{ github.ref }} git checkout --progress --force ${{ github.ref }} git log -1 --format='%H' - uses: "./.github/actions/pre-steps" - name: Set Java Environment run: |- echo "JAVA_HOME=${JAVA_HOME}" which java && java -version which javac && javac -version - name: Test RocksDBJava # NOTE: replaced scl enable devtoolset-7 'make V=1 J=8 -j8 jtest' run: make V=1 J=8 -j8 jtest # post-steps skipped because of compatibility issues with docker image build-linux-java-static: if: ${{ github.repository_owner == 'facebook' }} runs-on: labels: 4-core-ubuntu container: image: ghcr.io/facebook/rocksdb_ubuntu:22.1 options: --shm-size=16gb steps: # The docker image is intentionally based on an OS that has an older GLIBC version. # That GLIBC is incompatibile with GitHub's actions/checkout. Thus we implement a manual checkout step. # NOTE: replaced evolvedbinary/rocksjava:centos7_x64-be with ghcr.io/facebook/rocksdb_ubuntu:22.1 # until a more appropriate docker image with C++20 support is made. - name: Checkout env: GH_TOKEN: ${{ github.token }} run: | chown `whoami` . || true git clone --no-checkout https://oath2:$GH_TOKEN@github.com/${{ github.repository }}.git . git -c protocol.version=2 fetch --update-head-ok --no-tags --prune --no-recurse-submodules --depth=1 origin +${{ github.sha }}:${{ github.ref }} git checkout --progress --force ${{ github.ref }} git log -1 --format='%H' - uses: "./.github/actions/pre-steps" - name: Set Java Environment run: |- echo "JAVA_HOME=${JAVA_HOME}" which java && java -version which javac && javac -version - name: Build RocksDBJava Static Library # NOTE: replaced scl enable devtoolset-7 'make V=1 J=8 -j8 rocksdbjavastatic' run: make V=1 J=8 -j8 rocksdbjavastatic # post-steps skipped because of compatibility issues with docker image build-macos-java: if: ${{ github.repository_owner == 'facebook' }} runs-on: macos-15-xlarge env: JAVA_HOME: "/Library/Java/JavaVirtualMachines/liberica-jdk-8.jdk/Contents/Home" ROCKSDB_DISABLE_JEMALLOC: 1 steps: - uses: actions/checkout@v4.1.0 - uses: maxim-lobanov/setup-xcode@v1.6.0 with: xcode-version: 16.4.0 - uses: "./.github/actions/increase-max-open-files-on-macos" - uses: "./.github/actions/install-gflags-on-macos" - uses: "./.github/actions/install-jdk8-on-macos" - uses: "./.github/actions/pre-steps-macos" - name: Set Java Environment run: |- echo "JAVA_HOME=${JAVA_HOME}" which java && java -version which javac && javac -version - name: Test RocksDBJava run: make V=1 J=16 -j16 jtest - uses: "./.github/actions/post-steps" build-macos-java-static: if: ${{ github.repository_owner == 'facebook' }} runs-on: macos-15-xlarge env: JAVA_HOME: "/Library/Java/JavaVirtualMachines/liberica-jdk-8.jdk/Contents/Home" steps: - uses: actions/checkout@v4.1.0 - uses: maxim-lobanov/setup-xcode@v1.6.0 with: xcode-version: 16.4.0 - uses: "./.github/actions/increase-max-open-files-on-macos" - uses: "./.github/actions/install-gflags-on-macos" - uses: "./.github/actions/install-jdk8-on-macos" - uses: "./.github/actions/pre-steps-macos" - name: Set Java Environment run: |- echo "JAVA_HOME=${JAVA_HOME}" which java && java -version which javac && javac -version - name: Build RocksDBJava x86 and ARM Static Libraries run: make V=1 J=16 -j16 rocksdbjavastaticosx - uses: "./.github/actions/post-steps" build-macos-java-static-universal: if: ${{ github.repository_owner == 'facebook' }} runs-on: macos-15-xlarge env: JAVA_HOME: "/Library/Java/JavaVirtualMachines/liberica-jdk-8.jdk/Contents/Home" steps: - uses: actions/checkout@v4.1.0 - uses: maxim-lobanov/setup-xcode@v1.6.0 with: xcode-version: 16.4.0 - uses: "./.github/actions/increase-max-open-files-on-macos" - uses: "./.github/actions/install-gflags-on-macos" - uses: "./.github/actions/install-jdk8-on-macos" - uses: "./.github/actions/pre-steps-macos" - name: Set Java Environment run: |- echo "JAVA_HOME=${JAVA_HOME}" which java && java -version which javac && javac -version - name: Build RocksDBJava Universal Binary Static Library run: make V=1 J=16 -j16 rocksdbjavastaticosx_ub - uses: "./.github/actions/post-steps" build-linux-java-pmd: if: ${{ github.repository_owner == 'facebook' }} runs-on: labels: 4-core-ubuntu container: image: evolvedbinary/rocksjava:alpine3_x64-be options: --shm-size=16gb steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/install-maven" - uses: "./.github/actions/pre-steps" - name: Set Java Environment run: |- echo "JAVA_HOME=${JAVA_HOME}" which java && java -version which javac && javac -version - name: PMD RocksDBJava run: make V=1 J=8 -j8 jpmd - uses: actions/upload-artifact@v4.0.0 with: name: pmd-report path: "${{ github.workspace }}/java/target/pmd.xml" - uses: actions/upload-artifact@v4.0.0 with: name: maven-site path: "${{ github.workspace }}/java/target/site" build-linux-arm: if: ${{ github.repository_owner == 'facebook' }} runs-on: labels: 4-core-ubuntu-arm steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" - run: sudo apt-get update && sudo apt-get install -y build-essential - run: ROCKSDBTESTS_PLATFORM_DEPENDENT=only make V=1 J=4 -j4 all_but_some_tests check_some - uses: "./.github/actions/post-steps"