fix: RocksDB build

Also change some links to the new ones, removing reliance on June's github repos in some places

default.nix

@@ -1,11 +1,10 @@
-(import (
+(import
-  let
+  (
-    lock = builtins.fromJSON (builtins.readFile ./flake.lock);
+    let lock = builtins.fromJSON (builtins.readFile ./flake.lock); in
-  in
+    fetchTarball {
-  fetchTarball {
+      url = lock.nodes.flake-compat.locked.url or "https://github.com/edolstra/flake-compat/archive/${lock.nodes.flake-compat.locked.rev}.tar.gz";
-    url =
+      sha256 = lock.nodes.flake-compat.locked.narHash;
-      lock.nodes.flake-compat.locked.url
+    }
-        or "https://github.com/edolstra/flake-compat/archive/${lock.nodes.flake-compat.locked.rev}.tar.gz";
+  )
-    sha256 = lock.nodes.flake-compat.locked.narHash;
+  { src = ./.; }
-  }
+).defaultNix
-) { src = ./.; }).defaultNix

flake.lock

@@ -10,11 +10,11 @@
         "nixpkgs-stable": "nixpkgs-stable"
       },
       "locked": {
-        "lastModified": 1748532342,
+        "lastModified": 1751403276,
-        "narHash": "sha256-CvaKOUq8G10sghKpZhEB2UYjJoWhEkrDFggDgi7piUI=",
+        "narHash": "sha256-V0EPQNsQko1a8OqIWc2lLviLnMpR1m08Ej00z5RVTfs=",
         "owner": "zhaofengli",
         "repo": "attic",
-        "rev": "ce9373715fe3fac7a174a65a7e6d6baeba8cb4f9",
+        "rev": "896ad88fa57ad5dbcd267c0ac51f1b71ccfcb4dd",
         "type": "github"
       },
       "original": {
@@ -147,15 +147,17 @@
     },
     "fenix": {
       "inputs": {
-        "nixpkgs": "nixpkgs_5",
+        "nixpkgs": [
+          "nixpkgs"
+        ],
         "rust-analyzer-src": "rust-analyzer-src"
       },
       "locked": {
-        "lastModified": 1751352216,
+        "lastModified": 1751525020,
-        "narHash": "sha256-dJj8TUoZGj55Ttro37vvFGF2L+xlYNfspQ9u4BfqTFw=",
+        "narHash": "sha256-oDO6lCYS5Bf4jUITChj9XV7k3TP38DE0Ckz5n5ORCME=",
         "owner": "nix-community",
         "repo": "fenix",
-        "rev": "61b4f1e21bd631da91981f1ed74c959d6993f554",
+        "rev": "a1a5f92f47787e7df9f30e5e5ac13e679215aa1e",
         "type": "github"
       },
       "original": {
@@ -497,27 +499,11 @@
     },
     "nixpkgs_5": {
       "locked": {
-        "lastModified": 1751271578,
+        "lastModified": 1751498133,
-        "narHash": "sha256-P/SQmKDu06x8yv7i0s8bvnnuJYkxVGBWLWHaU+tt4YY=",
+        "narHash": "sha256-QWJ+NQbMU+NcU2xiyo7SNox1fAuwksGlQhpzBl76g1I=",
-        "owner": "nixos",
-        "repo": "nixpkgs",
-        "rev": "3016b4b15d13f3089db8a41ef937b13a9e33a8df",
-        "type": "github"
-      },
-      "original": {
-        "owner": "nixos",
-        "ref": "nixos-unstable",
-        "repo": "nixpkgs",
-        "type": "github"
-      }
-    },
-    "nixpkgs_6": {
-      "locked": {
-        "lastModified": 1751285371,
-        "narHash": "sha256-/hDU+2AUeFFu5qGHO/UyFMc4UG/x5Cw5uXO36KGTk6c=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "b9c03fbbaf84d85bb28eee530c7e9edc4021ca1b",
+        "rev": "d55716bb59b91ae9d1ced4b1ccdea7a442ecbfdb",
         "type": "github"
       },
       "original": {
@@ -553,18 +539,18 @@
         "flake-compat": "flake-compat_3",
         "flake-utils": "flake-utils",
         "nix-filter": "nix-filter",
-        "nixpkgs": "nixpkgs_6",
+        "nixpkgs": "nixpkgs_5",
         "rocksdb": "rocksdb"
       }
     },
     "rust-analyzer-src": {
       "flake": false,
       "locked": {
-        "lastModified": 1751296293,
+        "lastModified": 1751433876,
-        "narHash": "sha256-oaGMVdCcI32y6jQ7RE0+CqshZngfI19XnY31eYjdinI=",
+        "narHash": "sha256-IsdwOcvLLDDlkFNwhdD5BZy20okIQL01+UQ7Kxbqh8s=",
         "owner": "rust-lang",
         "repo": "rust-analyzer",
-        "rev": "eaf37e2c98b66ff7f7a0ac04e4cada39e51fde4b",
+        "rev": "11d45c881389dae90b0da5a94cde52c79d0fc7ef",
         "type": "github"
       },
       "original": {

flake.nix

@@ -7,6 +7,7 @@
     };
     fenix = {
       url = "github:nix-community/fenix?ref=main";
+      inputs.nixpkgs.follows = "nixpkgs";
     };
     flake-compat = {
       url = "github:edolstra/flake-compat?ref=master";
@@ -30,6 +31,7 @@
           inherit system;
         };
 
+        # The Rust toolchain to use
         toolchain = inputs.fenix.packages.${system}.fromToolchainFile {
           file = ./rust-toolchain.toml;
 
@@ -52,66 +54,67 @@
               ];
               buildFlags = [ "library" ];
             };
-            rocksdb-custom = pkgs.rocksdb_9_10.overrideAttrs (oldAttrs: {
+            rocksdb =
-              version = "v9.11.1";
+              (pkgs.rocksdb_9_10.override {
-              src = inputs.rocksdb;
+                # Override the liburing input for the build with our own so
+                # we have it built with the library flag
+                inherit (self) liburing;
+              }).overrideAttrs
+                (old: {
+                  src = inputs.rocksdb;
+                  version = "v9.11.1";
+                  cmakeFlags =
+                    pkgs.lib.subtractLists [
+                      # No real reason to have snappy or zlib, no one uses this
+                      "-DWITH_SNAPPY=1"
+                      "-DZLIB=1"
+                      "-DWITH_ZLIB=1"
+                      # We don't need to use ldb or sst_dump (core_tools)
+                      "-DWITH_CORE_TOOLS=1"
+                      # We don't need to build rocksdb tests
+                      "-DWITH_TESTS=1"
+                      # We use rust-rocksdb via C interface and don't need C++ RTTI
+                      "-DUSE_RTTI=1"
+                      # This doesn't exist in RocksDB, and USE_SSE is deprecated for
+                      # PORTABLE=$(march)
+                      "-DFORCE_SSE42=1"
+                      # PORTABLE will get set in main/default.nix
+                      "-DPORTABLE=1"
+                    ] old.cmakeFlags
+                    ++ [
+                      # No real reason to have snappy, no one uses this
+                      "-DWITH_SNAPPY=0"
+                      "-DZLIB=0"
+                      "-DWITH_ZLIB=0"
+                      # We don't need to use ldb or sst_dump (core_tools)
+                      "-DWITH_CORE_TOOLS=0"
+                      # We don't need trace tools
+                      "-DWITH_TRACE_TOOLS=0"
+                      # We don't need to build rocksdb tests
+                      "-DWITH_TESTS=0"
+                      # We use rust-rocksdb via C interface and don't need C++ RTTI
+                      "-DUSE_RTTI=0"
+                    ];
 
-              cmakeFlags =
+                  # outputs has "tools" which we don't need or use
-                pkgs.lib.subtractLists [
+                  outputs = [ "out" ];
-                  # No real reason to have snappy or zlib, no one uses this
-                  "-DWITH_SNAPPY=1"
-                  "-DZLIB=1"
-                  "-DWITH_ZLIB=1"
-                  # We don't need to use ldb or sst_dump (core_tools)
-                  "-DWITH_CORE_TOOLS=1"
-                  # We don't need to build rocksdb tests
-                  "-DWITH_TESTS=1"
-                  # We use rust-rocksdb via C interface and don't need C++ RTTI
-                  "-DUSE_RTTI=1"
-                  # This doesn't exist in RocksDB, and USE_SSE is deprecated for
-                  # PORTABLE=$(march)
-                  "-DFORCE_SSE42=1"
-                  # PORTABLE will get set in main/default.nix
-                  "-DPORTABLE=1"
-                ] oldAttrs.cmakeFlags
-                ++ [
-                  # No real reason to have snappy, no one uses this
-                  "-DWITH_SNAPPY=0"
-                  "-DZLIB=0"
-                  "-DWITH_ZLIB=0"
-                  # We don't need to use ldb or sst_dump (core_tools)
-                  "-DWITH_CORE_TOOLS=0"
-                  # We don't need trace tools
-                  "-DWITH_TRACE_TOOLS=0"
-                  # We don't need to build rocksdb tests
-                  "-DWITH_TESTS=0"
-                  # We use rust-rocksdb via C interface and don't need C++ RTTI
-                  "-DUSE_RTTI=0"
-                ];
 
-              propagatedBuildInputs = with pkgs; [
+                  # preInstall hooks has stuff for messing with ldb/sst_dump which we don't need or use
-                bzip2
+                  preInstall = "";
-                lz4
-                zstd
-              ];
 
-              # Has "tools" which we don't need or use
+                  # We have this already at https://forgejo.ellis.link/continuwuation/rocksdb/commit/a935c0273e1ba44eacf88ce3685a9b9831486155
-              outputs = [ "out" ];
+                  # Unsetting this so we don't have to revert it and make this nix exclusive
+                  patches = [ ];
 
-              # preInstall hooks has stuff for messing with ldb/sst_dump which we don't need or use
+                  postPatch = ''
-              preInstall = "";
+                    # Fix gcc-13 build failures due to missing <cstdint> and
-
+                    # <system_error> includes, fixed upstream since 8.x
-              # We have this already at https://forgejo.ellis.link/continuwuation/rocksdb/commit/a935c0273e1ba44eacf88ce3685a9b9831486155
+                    sed -e '1i #include <cstdint>' -i db/compaction/compaction_iteration_stats.h
-              patches = [ ];
+                    sed -e '1i #include <cstdint>' -i table/block_based/data_block_hash_index.h
-              postPatch = ''
+                    sed -e '1i #include <cstdint>' -i util/string_util.h
-                # Fix gcc-13 build failures due to missing <cstdint> and
+                    sed -e '1i #include <cstdint>' -i include/rocksdb/utilities/checkpoint.h
-                # <system_error> includes, fixed upstream since 8.x
+                  '';
-                sed -e '1i #include <cstdint>' -i db/compaction/compaction_iteration_stats.h
+                });
-                sed -e '1i #include <cstdint>' -i table/block_based/data_block_hash_index.h
-                sed -e '1i #include <cstdint>' -i util/string_util.h
-                sed -e '1i #include <cstdint>' -i include/rocksdb/utilities/checkpoint.h
-              '';
-            });
           });
 
         scopeHost = mkScope pkgsHost;
@@ -127,19 +130,20 @@
               }).pkgsStatic;
           in
           mkScope pkgsCrossStatic;
+
       in
       {
         packages =
           {
             default = scopeHost.main.override {
               disable_features = [
-                # dont include experimental features
+                # Don't include experimental features
                 "experimental"
                 # jemalloc profiling/stats features are expensive and shouldn't
                 # be expected on non-debug builds.
                 "jemalloc_prof"
                 "jemalloc_stats"
-                # this is non-functional on nix for some reason
+                # This is non-functional on nix for some reason
                 "hardened_malloc"
                 # conduwuit_mods is a development-only hot reload feature
                 "conduwuit_mods"
@@ -147,10 +151,23 @@
             };
             default-debug = scopeHost.main.override {
               profile = "dev";
-              # debug build users expect full logs
+              # Debug build users expect full logs
               disable_release_max_log_level = true;
               disable_features = [
-                # dont include experimental features
+                # Don't include experimental features
+                "experimental"
+                # This is non-functional on nix for some reason
+                "hardened_malloc"
+                # conduwuit_mods is a development-only hot reload feature
+                "conduwuit_mods"
+              ];
+            };
+            # Just a test profile used for things like CI and complement
+            default-test = scopeHost.main.override {
+              profile = "test";
+              disable_release_max_log_level = true;
+              disable_features = [
+                # Don't include experimental features
                 "experimental"
                 # this is non-functional on nix for some reason
                 "hardened_malloc"
@@ -161,13 +178,13 @@
             all-features = scopeHost.main.override {
               all_features = true;
               disable_features = [
-                # dont include experimental features
+                # Don't include experimental features
                 "experimental"
                 # jemalloc profiling/stats features are expensive and shouldn't
                 # be expected on non-debug builds.
                 "jemalloc_prof"
                 "jemalloc_stats"
-                # this is non-functional on nix for some reason
+                # This is non-functional on nix for some reason
                 "hardened_malloc"
                 # conduwuit_mods is a development-only hot reload feature
                 "conduwuit_mods"
@@ -176,12 +193,12 @@
             all-features-debug = scopeHost.main.override {
               profile = "dev";
               all_features = true;
-              # debug build users expect full logs
+              # Debug build users expect full logs
               disable_release_max_log_level = true;
               disable_features = [
-                # dont include experimental features
+                # Don't include experimental features
                 "experimental"
-                # this is non-functional on nix for some reason
+                # This is non-functional on nix for some reason
                 "hardened_malloc"
                 # conduwuit_mods is a development-only hot reload feature
                 "conduwuit_mods"
@@ -233,9 +250,9 @@
                         profile = "test";
                         disable_release_max_log_level = true;
                         disable_features = [
-                          # Don't include experimental features
+                          # dont include experimental features
                           "experimental"
-                          # This is non-functional on nix for some reason
+                          # this is non-functional on nix for some reason
                           "hardened_malloc"
                           # conduwuit_mods is a development-only hot reload feature
                           "conduwuit_mods"
@@ -249,13 +266,13 @@
                       value = scopeCrossStatic.main.override {
                         all_features = true;
                         disable_features = [
-                          # Don't include experimental features
+                          # dont include experimental features
                           "experimental"
                           # jemalloc profiling/stats features are expensive and shouldn't
                           # be expected on non-debug builds.
                           "jemalloc_prof"
                           "jemalloc_stats"
-                          # This is non-functional on nix for some reason
+                          # this is non-functional on nix for some reason
                           "hardened_malloc"
                           # conduwuit_mods is a development-only hot reload feature
                           "conduwuit_mods"
@@ -270,13 +287,13 @@
                       value = scopeCrossStatic.main.override {
                         all_features = true;
                         disable_features = [
-                          # Don't include experimental features
+                          # dont include experimental features
                           "experimental"
                           # jemalloc profiling/stats features are expensive and shouldn't
                           # be expected on non-debug builds.
                           "jemalloc_prof"
                           "jemalloc_stats"
-                          # This is non-functional on nix for some reason
+                          # this is non-functional on nix for some reason
                           "hardened_malloc"
                           # conduwuit_mods is a development-only hot reload feature
                           "conduwuit_mods"
@@ -295,9 +312,9 @@
                         # debug build users expect full logs
                         disable_release_max_log_level = true;
                         disable_features = [
-                          # Don't include experimental features
+                          # dont include experimental features
                           "experimental"
-                          # This is non-functional on nix for some reason
+                          # this is non-functional on nix for some reason
                           "hardened_malloc"
                           # conduwuit_mods is a development-only hot reload feature
                           "conduwuit_mods"
@@ -315,6 +332,8 @@
                   ]
                 )
                 [
+                  #"x86_64-apple-darwin"
+                  #"aarch64-apple-darwin"
                   "x86_64-linux-gnu"
                   "x86_64-linux-musl"
                   "aarch64-linux-musl"

nix/pkgs/main/cross-compilation-env.nix

@@ -2,30 +2,33 @@
 , pkgsBuildHost
 , rust
 , stdenv
-,
 }:
 
 lib.optionalAttrs stdenv.hostPlatform.isStatic
   {
     ROCKSDB_STATIC = "";
   }
-// {
+//
-  CARGO_BUILD_RUSTFLAGS = lib.concatStringsSep " " (
+{
-    lib.optionals stdenv.hostPlatform.isStatic [
+  CARGO_BUILD_RUSTFLAGS =
-      "-C"
+    lib.concatStringsSep
-      "relocation-model=static"
+      " "
-    ]
+      (lib.optionals
-    ++ lib.optionals (stdenv.buildPlatform.config != stdenv.hostPlatform.config) [
+        stdenv.hostPlatform.isStatic
-      "-l"
+        [ "-C" "relocation-model=static" ]
-      "c"
+      ++ lib.optionals
+        (stdenv.buildPlatform.config != stdenv.hostPlatform.config)
+        [
+          "-l"
+          "c"
 
-      "-l"
+          "-l"
-      "stdc++"
+          "stdc++"
 
-      "-L"
+          "-L"
-      "${stdenv.cc.cc.lib}/${stdenv.hostPlatform.config}/lib"
+          "${stdenv.cc.cc.lib}/${stdenv.hostPlatform.config}/lib"
-    ]
+        ]
-  );
+      );
 }
 
   # What follows is stolen from [here][0]. Its purpose is to properly
@@ -34,11 +37,14 @@ lib.optionalAttrs stdenv.hostPlatform.isStatic
   # run on the build platform (I think).
   #
   # [0]: https://github.com/NixOS/nixpkgs/blob/nixpkgs-unstable/pkgs/build-support/rust/lib/default.nix#L48-L68
-  // (
+  //
+(
   let
     inherit (rust.lib) envVars;
   in
-  lib.optionalAttrs (stdenv.targetPlatform.rust.rustcTarget != stdenv.hostPlatform.rust.rustcTarget)
+  lib.optionalAttrs
+    (stdenv.targetPlatform.rust.rustcTarget
+    != stdenv.hostPlatform.rust.rustcTarget)
     (
       let
         inherit (stdenv.targetPlatform.rust) cargoEnvVarTarget;
@@ -49,7 +55,8 @@ lib.optionalAttrs stdenv.hostPlatform.isStatic
         "CARGO_TARGET_${cargoEnvVarTarget}_LINKER" = envVars.ccForTarget;
       }
     )
-  // (
+  //
+  (
     let
       inherit (stdenv.hostPlatform.rust) cargoEnvVarTarget rustcTarget;
     in
@@ -60,7 +67,8 @@ lib.optionalAttrs stdenv.hostPlatform.isStatic
       CARGO_BUILD_TARGET = rustcTarget;
     }
   )
-  // (
+  //
+  (
     let
       inherit (stdenv.buildPlatform.rust) cargoEnvVarTarget;
     in

nix/pkgs/main/default.nix

@@ -7,6 +7,7 @@
 , liburing
 , pkgsBuildHost
 , rocksdb
+, removeReferencesTo
 , rust
 , rust-jemalloc-sys
 , stdenv
@@ -142,6 +143,8 @@ let
       " -Ctarget-cpu=haswell";
   };
 
+
+
   commonAttrs = {
     inherit
       (craneLib.crateNameFromCargoToml {