Feat/dynamic import (#473)

* Make the channel importer versions dynamic * few things fromt he list 1. add nixos-org-configurations as input to flake.nix 2. evaluate channels.nix file and export channels via environment variable. that environment variable (lets call it NIXOS_CHANNELS) should be present during the build and inside the nix shell. the content of the variable can be JSON. 3. we pickup the NIXOS_CHANNELS environment variable in frontend/webpack.config.js and pass it further to webpack process, just like we do with ELASTICSEARCH_MAPPING_SCHEMA_VERSION. 4. we forward NIXOS_CHANNELS to Elm via frontend/src/index.js as an Elm application flag. Just like we do with other variables there. * Decode nixosChannels in Elm * Use nixosChannels that came via application flag * read nixos channels in github action * defaultNixOSChannel should be calculated * add two pointers where the check should be added * pass nixosChannels to flake-info and remove title, rather calculate it * Add NixosChannels struct validation and validation Error * Read NIXOS_CHANNEL variable * Check channel * Add channel struct to fix parsing NIXOS_CHANNELS * Use `eachDefaultSystem` instead of listing them manually * Add individual dev shells for frontend and backend * Update .github/workflows/import-to-elasticsearch.yml Co-authored-by: Naïm Favier <n@monade.li> * use both development environments by default (as it was) but still provide devShells for each of the subprojects * pkgs.lib → lib everywhere and define lib = nixpkgs.lib before the call to eachDefaultSystem Also, version = lib.fileContents ./VERSION; * Update flake.nix Co-authored-by: Naïm Favier <n@monade.li> * typo * bumping version to test the changes to import-to-elasticsearch github action * some invisibile characters needed to be removed * typo * will this work * typo * forgot the checkout step * add some debugging * typo * read NIXOS_CHANNELS from environment not via argument * fix for the NIXOS_CHANNELS variable Co-authored-by: Janne Heß <janne@hess.ooo> Co-authored-by: ysndr <me@ysndr.de> Co-authored-by: Naïm Favier <n@monade.li>
NixOS · Apr 24, 2022 · eea6cd3 · eea6cd3
1 parent 492794f
commit eea6cd3
Show file tree

Hide file tree

Showing 15 changed files with 507 additions and 262 deletions.
diff --git a/.github/workflows/import-to-elasticsearch.yml b/.github/workflows/import-to-elasticsearch.yml
@@ -7,15 +7,36 @@ on:
 
 jobs:
 
+  nixos-channels:
+    runs-on: ubuntu-latest
+
+    outputs:
+      matrix: ${{ steps.nixos-channels.outputs.matrix }}
+
+    steps:
+    - name: Checking out the repository
+      uses: actions/checkout@v3
+
+    - name: Setup
+      uses: ./.github/actions/common-setup
+      with:
+        CACHIX_SIGNING_KEY: ${{ secrets.CACHIX_SIGNING_KEY }}
+
+    - name: NixOS Channels
+      id: nixos-channels
+      run: |
+        nix build -L .#nixosChannels
+        channels="{\"channel\": $(< ./result)}"
+        echo $channels
+        echo "::set-output name=matrix::$channels"
+
   import-nixpkgs:
+    needs: nixos-channels
     runs-on: ubuntu-latest
 
     strategy:
       fail-fast: false
-      matrix:
-        channel:
-          - unstable
-          - 21.11
+      matrix: ${{ fromJSON(needs.nixos-channels.outputs.matrix) }}
 
     env:
       RUST_LOG: debug
@@ -46,7 +67,6 @@ jobs:
       if: github.repository_owner == 'NixOS'
 
 
-
   import-flakes:
     runs-on: ubuntu-latest
 

diff --git a/VERSION b/VERSION
@@ -1 +1 @@
-29
+30
diff --git a/flake-info/default.nix b/flake-info/default.nix
@@ -1,5 +1,7 @@
-{ pkgs ? import <nixpkgs> { } }: with pkgs;
-rustPlatform.buildRustPackage rec {
+{ pkgs ? import <nixpkgs> {}
+, nixosChannels ? {}
+}:
+pkgs.rustPlatform.buildRustPackage rec {
   name = "flake-info";
   src = ./.;
   cargoLock = {
@@ -8,11 +10,18 @@ rustPlatform.buildRustPackage rec {
       "elasticsearch-8.0.0-alpha.1" = "sha256-gjmk3Q3LTAvLhzQ+k1knSp1HBwtqNiubjXNnLy/cS5M=";
     };
   };
-  nativeBuildInputs = [ pkg-config ];
-  buildInputs = [ openssl openssl.dev makeWrapper ] 
-              ++ lib.optional pkgs.stdenv.isDarwin [ libiconv darwin.apple_sdk.frameworks.Security ];
+  nativeBuildInputs = with pkgs; [ pkg-config ];
+  buildInputs =
+    with pkgs; [
+      openssl
+      openssl.dev
+      makeWrapper
+    ] ++ lib.optional pkgs.stdenv.isDarwin [
+      libiconv
+      darwin.apple_sdk.frameworks.Security
+    ];
 
-  checkInputs = [ pandoc ];
+  checkInputs = with pkgs; [ pandoc ];
 
   NIXPKGS_PANDOC_FILTERS_PATH = "${pkgs.path + "/doc/build-aux/pandoc-filters"}";
 
@@ -24,6 +33,7 @@ rustPlatform.buildRustPackage rec {
   postInstall = ''
     wrapProgram $out/bin/flake-info \
       --set NIXPKGS_PANDOC_FILTERS_PATH "${NIXPKGS_PANDOC_FILTERS_PATH}" \
-      --prefix PATH : ${pandoc}/bin
+      --set NIXOS_CHANNELS '${builtins.toJSON nixosChannels}' \
+      --prefix PATH : ${pkgs.pandoc}/bin
   '';
 }
diff --git a/flake-info/src/bin/flake-info.rs b/flake-info/src/bin/flake-info.rs
@@ -5,11 +5,14 @@ use flake_info::data::import::{Kind, NixOption};
 use flake_info::data::{self, Export, Nixpkgs, Source};
 use flake_info::elastic::{ElasticsearchError, ExistsStrategy};
 use flake_info::{commands, elastic};
+use lazy_static::lazy_static;
 use log::{debug, error, info, warn};
 use semver::VersionReq;
+use serde::Deserialize;
 use sha2::Digest;
 use std::path::{Path, PathBuf};
 use std::ptr::hash;
+use std::str::FromStr;
 use std::{fs, io};
 use structopt::{clap::ArgGroup, StructOpt};
 use thiserror::Error;
@@ -198,6 +201,9 @@ enum FlakeInfoError {
     #[error("Nix check failed: {0}")]
     NixCheck(#[from] NixCheckError),
 
+    #[error("Nixos Channel `{0}` not among the allowed Channels set by NIXOS_CHANNELS ({:?}", .1.channels)]
+    UnknownNixOSChannel(String, NixosChannels),
+
     #[error("Getting flake info caused an error: {0:?}")]
     Flake(anyhow::Error),
     #[error("Getting nixpkgs info caused an error: {0:?}")]
@@ -237,6 +243,8 @@ async fn run_command(
             Ok((exports, ident))
         }
         Command::Nixpkgs { channel } => {
+            NIXOS_CHANNELS.check_channel(&channel)?;
+
             let nixpkgs = Source::nixpkgs(channel)
                 .await
                 .map_err(FlakeInfoError::Nixpkgs)?;
@@ -251,6 +259,8 @@ async fn run_command(
             Ok((exports, ident))
         }
         Command::NixpkgsArchive { source, channel } => {
+            NIXOS_CHANNELS.check_channel(&channel)?;
+
             let ident = (
                 "nixos".to_string(),
                 channel.to_owned(),
@@ -404,3 +414,36 @@ async fn push_to_elastic(
 
     Ok(())
 }
+
+/// Information about allowed and default nixos channels.
+/// Typyically passed by environment variable NIXOS_CHANNELS.
+/// Used to filter the input arguments for `flake-info nixpkgs` and `flake-info nixpkgs-archive`
+#[derive(Clone, Debug, Deserialize)]
+struct NixosChannels {
+    channels: Vec<Channel>,
+}
+
+#[derive(Clone, Debug, Deserialize)]
+struct Channel {
+    branch: String
+}
+
+impl NixosChannels {
+    fn check_channel(&self, channel: &String) -> Result<(), FlakeInfoError> {
+        self.channels.iter().find(|c| &c.branch == channel).map_or_else(|| Ok(()), |_| Err(FlakeInfoError::UnknownNixOSChannel(channel.clone(), self.clone())))
+    }
+}
+
+impl FromStr for NixosChannels {
+    type Err=serde_json::Error;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        serde_json::from_str(s)
+    }
+}
+
+lazy_static! {
+    static ref NIXOS_CHANNELS: NixosChannels = std::env::var("NIXOS_CHANNELS")
+        .unwrap_or("".to_string())
+        .parse().unwrap();
+}
diff --git a/flake.lock b/flake.lock
diff --git a/flake.nix b/flake.nix
@@ -6,36 +6,119 @@
 
   inputs.nixpkgs.url = "nixpkgs/nixos-unstable";
   inputs.flake-utils.url = "github:numtide/flake-utils";
+  inputs.nixos-org-configurations.url = "github:NixOS/nixos-org-configurations";
+  inputs.nixos-org-configurations.flake = false;
 
   outputs = { self
             , nixpkgs
             , flake-utils
+            , nixos-org-configurations 
             }:
-    flake-utils.lib.eachSystem
-      (with flake-utils.lib.system; [
-        x86_64-linux
-        i686-linux
-        x86_64-darwin
-        aarch64-linux
-      ])
+    flake-utils.lib.eachDefaultSystem
       (system:
         let
           pkgs = nixpkgs.legacyPackages.${system};
-          warnToUpgradeNix = pkgs.lib.warn "Please upgrade Nix to 2.7 or later.";
+          lib = nixpkgs.lib;
+          warnToUpgradeNix = lib.warn "Please upgrade Nix to 2.7 or later.";
+          version = lib.fileContents ./VERSION;
+          nixosChannels =
+            let
+              allChannels = (import "${nixos-org-configurations}/channels.nix").channels;
+              filteredChannels =
+                lib.filterAttrs
+                  (n: v:
+                    builtins.elem v.status ["beta" "stable" "rolling"] &&
+                    lib.hasPrefix "nixos-" n &&
+                    v ? variant && v.variant == "primary"
+                  )
+                  allChannels;
+            in
+            {
+              channels =
+                lib.mapAttrsToList
+                  (n: v:
+                    {
+                      id = lib.removePrefix "nixos-" n;
+                      status = v.status;
+                      jobset =
+                        builtins.concatStringsSep
+                          "/"
+                          (lib.init (lib.splitString "/" v.job));
+                      branch = n;
+                    }
+                  )
+                  filteredChannels;
+              default =
+                builtins.head
+                  (builtins.sort (e1: e2: ! (builtins.lessThan e1 e2))
+                    (builtins.map
+                      (lib.removePrefix "nixos-")
+                      (builtins.attrNames
+                        (lib.filterAttrs (_: v: v.status == "stable") filteredChannels)
+                      )
+                    )
+                  );
+            };
+          nixosChannelsFile = pkgs.runCommand "nixosChannels.json" {} ''
+            echo '${builtins.toJSON (builtins.map (c: c.id) nixosChannels.channels)}' > $out
+          '';
+
+          mkDevShell = { inputsFrom ? [], extraPackages ? [], extraShellHook ? "" }: 
+            pkgs.mkShell {
+              inherit inputsFrom;
+              packages = extraPackages;
+              shellHook = ''
+                export NIXOS_CHANNELS='${builtins.toJSON nixosChannels}';
+                export ELASTICSEARCH_MAPPING_SCHEMA_VERSION="${version}";
+              '' + extraShellHook;
+            };
         in rec {
 
           packages.default = packages.flake-info;
-          packages.flake-info = import ./flake-info { inherit pkgs; };
-          packages.frontend = import ./frontend { inherit pkgs; };
+          packages.flake-info = import ./flake-info { inherit pkgs nixosChannels; };
+          packages.frontend = import ./frontend { inherit pkgs nixosChannels version; };
+          packages.nixosChannels = nixosChannelsFile;
+
+          devShells.default = mkDevShell {
+            inputsFrom = [
+              packages.flake-info
+              packages.frontend
+            ];
+            extraPackages = [pkgs.rustfmt];
+            extraShellHook = ''
+              export RUST_SRC_PATH="${pkgs.rustPlatform.rustLibSrc}";
+              export NIXPKGS_PANDOC_FILTERS_PATH="${packages.flake-info.NIXPKGS_PANDOC_FILTERS_PATH}";
+              export PATH=$PWD/frontend/node_modules/.bin:$PATH
+
+              rm -rf frontend/node_modules
+              ln -sf ${packages.frontend.yarnPkg}/libexec/${(builtins.parseDrvName packages.frontend.name).name}/node_modules frontend/
+              echo "========================================================"
+              echo "= To develop the frontend run: cd frontend && yarn dev ="
+              echo "========================================================"
+            '';
+          };
 
-          devShells.default = pkgs.mkShell {
-            inputsFrom = builtins.attrValues packages;
-            shellHook = ''
+          devShells.flake-info = mkDevShell {
+            inputsFrom = [packages.flake-info];
+            extraPackages = [pkgs.rustfmt];
+            extraShellHook = ''
               export RUST_SRC_PATH="${pkgs.rustPlatform.rustLibSrc}";
               export NIXPKGS_PANDOC_FILTERS_PATH="${packages.flake-info.NIXPKGS_PANDOC_FILTERS_PATH}";
             '';
           };
 
+          devShells.frontend = mkDevShell {
+            inputsFrom = [packages.frontend] ;
+            extraShellHook = ''
+              export PATH=$PWD/frontend/node_modules/.bin:$PATH
+              rm -rf frontend/node_modules
+              ln -sf ${packages.frontend.yarnPkg}/libexec/${(builtins.parseDrvName packages.frontend.name).name}/node_modules frontend/
+              echo "========================================================"
+              echo "= To develop the frontend run: cd frontend && yarn dev ="
+              echo "========================================================"
+            '';
+          };
+
           # XXX: for backwards compatibility
           devShell = warnToUpgradeNix devShells.default;
           defaultPackage = warnToUpgradeNix packages.default;

diff --git a/frontend/default.nix b/frontend/default.nix
@@ -1,5 +1,6 @@
 { pkgs ? import <nixpkgs> { }
-, version ? pkgs.lib.removeSuffix "\n" (builtins.readFile ../VERSION)
+, nixosChannels
+, version
 }:
 let
   package = builtins.fromJSON (builtins.readFile ./package.json);
@@ -52,6 +53,9 @@ pkgs.stdenv.mkDerivation {
       elm-analyse
     ]);
 
+  ELASTICSEARCH_MAPPING_SCHEMA_VERSION = version;
+  NIXOS_CHANNELS = builtins.toJSON nixosChannels;
+
   configurePhase = pkgs.elmPackages.fetchElmDeps {
     elmPackages = import ./elm-srcs.nix;
     elmVersion = pkgs.elmPackages.elm.version;
@@ -66,7 +70,6 @@ pkgs.stdenv.mkDerivation {
   buildPhase = ''
     # Yarn writes cache directories etc to $HOME.
     export HOME=$PWD/yarn_home
-    sed -i -e "s|process.env.ELASTICSEARCH_MAPPING_SCHEMA_VERSION|${version}|" src/index.js
     yarn prod
   '';
 
@@ -75,14 +78,6 @@ pkgs.stdenv.mkDerivation {
     cp -R ./dist/* $out/
     cp netlify.toml $out/
   '';
-  shellHook = ''
-    rm -rf frontend/node_modules
-    ln -sf ${yarnPkg}/libexec/${package.name}/node_modules frontend/
-    export PATH=$PWD/frontend/node_modules/.bin:$PATH
-    export ELASTICSEARCH_MAPPING_SCHEMA_VERSION=${version}
-    echo "============================"
-    echo "= To develop the frontend run: cd frontend && yarn dev ="
-    echo "============================"
-  '';
 
+  passthru.yarnPkg = yarnPkg;
 }