From c34239476acc13afe57b4e9e1672908975779875 Mon Sep 17 00:00:00 2001 From: hutli Date: Thu, 15 Feb 2024 14:25:04 +0100 Subject: [PATCH 1/7] nix: .#widnows: init initial nix build for windows using zig mingwW64 build removes nix zig windows build removes nix zig windows build removed unnessesary glibc.static removed unnessesary import of pkgs in nix fixed missing trailing newline on non-windows nix builds overriding stdenv when building for crosscompiling to windows in nix better variables when crosscompiling windows in nix cross compile windows on macos removed trailing whitespace remove unnessesary overwrite of "CMAKE_SYSTEM_NAME" in nix windows build nix: keep file extension when copying result files during cross compile for windows nix: better checking for file extensions when using MinGW nix: using hostPlatform instead of targetPlatform when cross compiling for Windows using hostPlatform.extensions.executable to extract executable format --- .devops/nix/package.nix | 6 ++++-- flake.nix | 3 +++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.devops/nix/package.nix b/.devops/nix/package.nix index b651f9e61f7..83858c82951 100644 --- a/.devops/nix/package.nix +++ b/.devops/nix/package.nix @@ -67,6 +67,8 @@ let strings.optionalString (suffices != [ ]) ", accelerated with ${strings.concatStringsSep ", " suffices}"; + executableSuffix = effectiveStdenv.hostPlatform.extensions.executable; + # TODO: package the Python in this repository in a Nix-like way. # It'd be nice to migrate to buildPythonPackage, as well as ensure this repo # is PEP 517-compatible, and ensure the correct .dist-info is generated. @@ -244,8 +246,8 @@ effectiveStdenv.mkDerivation ( # TODO(SomeoneSerge): It's better to add proper install targets at the CMake level, # if they haven't been added yet. postInstall = '' - mv $out/bin/main $out/bin/llama - mv $out/bin/server $out/bin/llama-server + mv $out/bin/main${executableSuffix} $out/bin/llama${executableSuffix} + mv $out/bin/server${executableSuffix} $out/bin/llama-server${executableSuffix} mkdir -p $out/include cp $src/llama.h $out/include/ ''; diff --git a/flake.nix b/flake.nix index 45f9deda0be..6af9bb1e5d8 100644 --- a/flake.nix +++ b/flake.nix @@ -155,6 +155,9 @@ { default = config.legacyPackages.llamaPackages.llama-cpp; vulkan = config.packages.default.override { useVulkan = true; }; + windows = config.legacyPackages.llamaPackages.llama-cpp.override { + stdenv = pkgs.pkgsCross.mingwW64.stdenv; + }; } // lib.optionalAttrs pkgs.stdenv.isLinux { opencl = config.packages.default.override { useOpenCL = true; }; From 07120aed8473f8aadc34bc8dc116cf671eb1fabe Mon Sep 17 00:00:00 2001 From: Someone Serge Date: Tue, 26 Mar 2024 16:22:07 +0000 Subject: [PATCH 2/7] nix: package: don't introduce the dependency on python - The generic /usr/bin/env shebangs are good enough - Python deps are provisioned in the devShells - We need to be able to leave python out at least on windows (currently breaks eval) --- .devops/nix/package.nix | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.devops/nix/package.nix b/.devops/nix/package.nix index 83858c82951..490ad0cc5fa 100644 --- a/.devops/nix/package.nix +++ b/.devops/nix/package.nix @@ -73,6 +73,9 @@ let # It'd be nice to migrate to buildPythonPackage, as well as ensure this repo # is PEP 517-compatible, and ensure the correct .dist-info is generated. # https://peps.python.org/pep-0517/ + # + # TODO: Package up each Python script or service appropriately, by making + # them into "entrypoints" llama-python = python3.withPackages ( ps: [ ps.numpy @@ -161,11 +164,6 @@ effectiveStdenv.mkDerivation ( --replace '[bundle pathForResource:@"ggml-metal" ofType:@"metal"];' "@\"$out/bin/ggml-metal.metal\";" substituteInPlace ./ggml-metal.m \ --replace '[bundle pathForResource:@"default" ofType:@"metallib"];' "@\"$out/bin/default.metallib\";" - - # TODO: Package up each Python script or service appropriately. - # If we were to migrate to buildPythonPackage and prepare the `pyproject.toml`, - # we could make those *.py into setuptools' entrypoints - substituteInPlace ./*.py --replace "/usr/bin/env python" "${llama-python}/bin/python" ''; # With PR#6015 https://github.com/ggerganov/llama.cpp/pull/6015, From 23577d07451502e5745083f41d5b4f9a331ed075 Mon Sep 17 00:00:00 2001 From: Someone Serge Date: Tue, 26 Mar 2024 16:22:42 +0000 Subject: [PATCH 3/7] nix: .#windows: proper cross-compilation set-up Take all dependencies from the cross stage, rather tha only stdenv --- flake.nix | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/flake.nix b/flake.nix index 6af9bb1e5d8..272f06fb48b 100644 --- a/flake.nix +++ b/flake.nix @@ -145,6 +145,7 @@ # the same path you would with an overlay. legacyPackages = { llamaPackages = pkgs.callPackage .devops/nix/scope.nix { inherit llamaVersion; }; + llamaPackagesWindows = pkgs.pkgsCross.mingwW64.callPackage .devops/nix/scope.nix { inherit llamaVersion; }; llamaPackagesCuda = pkgsCuda.callPackage .devops/nix/scope.nix { inherit llamaVersion; }; llamaPackagesRocm = pkgsRocm.callPackage .devops/nix/scope.nix { inherit llamaVersion; }; }; @@ -155,9 +156,7 @@ { default = config.legacyPackages.llamaPackages.llama-cpp; vulkan = config.packages.default.override { useVulkan = true; }; - windows = config.legacyPackages.llamaPackages.llama-cpp.override { - stdenv = pkgs.pkgsCross.mingwW64.stdenv; - }; + windows = config.legacyPackages.llamaPackagesWindows.llama-cpp; } // lib.optionalAttrs pkgs.stdenv.isLinux { opencl = config.packages.default.override { useOpenCL = true; }; From df639e07d742b821100f7a1091cee2a1d217d023 Mon Sep 17 00:00:00 2001 From: hutli Date: Wed, 27 Mar 2024 17:25:05 +0100 Subject: [PATCH 4/7] only using explicit blas if hostPlatform is allowed --- .devops/nix/package.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devops/nix/package.nix b/.devops/nix/package.nix index 490ad0cc5fa..cb6ddb3b482 100644 --- a/.devops/nix/package.nix +++ b/.devops/nix/package.nix @@ -200,7 +200,7 @@ effectiveStdenv.mkDerivation ( ++ optionals useMpi [ mpi ] ++ optionals useOpenCL [ clblast ] ++ optionals useRocm rocmBuildInputs - ++ optionals useBlas [ blas ] + ++ optionals (useBlas && builtins.elem effectiveStdenv.hostPlatform.system blas.meta.platforms) [ blas ] ++ optionals useVulkan vulkanBuildInputs; cmakeFlags = From ae316601c60a880d255d877f167eae7a58cc2e05 Mon Sep 17 00:00:00 2001 From: hutli Date: Wed, 27 Mar 2024 18:10:08 +0100 Subject: [PATCH 5/7] using blas.meta.available to check host platform --- .devops/nix/package.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devops/nix/package.nix b/.devops/nix/package.nix index cb6ddb3b482..77d7e4f5eb4 100644 --- a/.devops/nix/package.nix +++ b/.devops/nix/package.nix @@ -200,7 +200,7 @@ effectiveStdenv.mkDerivation ( ++ optionals useMpi [ mpi ] ++ optionals useOpenCL [ clblast ] ++ optionals useRocm rocmBuildInputs - ++ optionals (useBlas && builtins.elem effectiveStdenv.hostPlatform.system blas.meta.platforms) [ blas ] + ++ optionals (useBlas && blas.meta.available) [ blas ] ++ optionals useVulkan vulkanBuildInputs; cmakeFlags = From e469532ddd98c809afe154a09a40c32c4b44b40d Mon Sep 17 00:00:00 2001 From: hutli Date: Wed, 27 Mar 2024 19:14:28 +0100 Subject: [PATCH 6/7] nix: moved blas availability check to package inputs so it is still overridable --- .devops/nix/package.nix | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.devops/nix/package.nix b/.devops/nix/package.nix index 77d7e4f5eb4..d0db0f10e67 100644 --- a/.devops/nix/package.nix +++ b/.devops/nix/package.nix @@ -18,13 +18,13 @@ vulkan-headers, vulkan-loader, clblast, - useBlas ? builtins.all (x: !x) [ + useBlas ? builtins.all (x: !x) [ useCuda useMetalKit useOpenCL useRocm useVulkan - ], + ] && blas.meta.available, useCuda ? config.cudaSupport, useMetalKit ? stdenv.isAarch64 && stdenv.isDarwin && !useOpenCL, useMpi ? false, # Increases the runtime closure size by ~700M @@ -200,7 +200,7 @@ effectiveStdenv.mkDerivation ( ++ optionals useMpi [ mpi ] ++ optionals useOpenCL [ clblast ] ++ optionals useRocm rocmBuildInputs - ++ optionals (useBlas && blas.meta.available) [ blas ] + ++ optionals useBlas [ blas ] ++ optionals useVulkan vulkanBuildInputs; cmakeFlags = From 7f4d575151dc34bb329c37ed08d25e448c369a48 Mon Sep 17 00:00:00 2001 From: hutli Date: Wed, 27 Mar 2024 19:17:30 +0100 Subject: [PATCH 7/7] nix: removed unnessesary indentation --- .devops/nix/package.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devops/nix/package.nix b/.devops/nix/package.nix index d0db0f10e67..2c0ae4e2a07 100644 --- a/.devops/nix/package.nix +++ b/.devops/nix/package.nix @@ -18,7 +18,7 @@ vulkan-headers, vulkan-loader, clblast, - useBlas ? builtins.all (x: !x) [ + useBlas ? builtins.all (x: !x) [ useCuda useMetalKit useOpenCL