WIP: Relocation based build (inplace free)

README.md

@@ -104,7 +104,7 @@ use `hadrian/UserSettings.hs` for the same purpose, see [documentation](doc/user
 
 #### Clean and full rebuild
 
-* `build clean` removes all build artefacts.
+* `build clean` removes all build artifacts.
 
 * `build -B` forces Shake to rerun all rules, even if the previous build results are
 are still up-to-date.

cabal.project

@@ -1,2 +1,16 @@
 packages: ./
           ../libraries/Cabal/Cabal/
+          ../libraries/hpc/
+          ../libraries/parsec/
+          ../libraries/text/
+
+package hadrian
+  ghc-options: -Werror
+
+repository head.hackage
+   url: http://head.hackage.haskell.org/
+   secure: True
+   root-keys: 07c59cb65787dedfaef5bd5f987ceb5f7e5ebf88b904bbd4c5cbdeb2ff71b740
+              2e8555dde16ebd8df076f1a8ef13b8f14c66bad8eafefd7d9e37d0ed711821fb
+              8f79fd2389ab2967354407ec852cbe73f2e8635793ac446d09461ffb99527f6e
+   key-threshold: 3

cfg/system.config.in

@@ -18,6 +18,7 @@ ranlib         = @REAL_RANLIB_CMD@
 sphinx-build   = @SPHINXBUILD@
 system-ar      = @AR_STAGE0@
 system-cc      = @CC_STAGE0@
+system-ld      = @LD_STAGE0@
 system-ghc     = @WithGhc@
 system-ghc-pkg = @GhcPkgCmd@
 tar            = @TarCmd@
@@ -37,7 +38,8 @@ ar-supports-at-file = @ArSupportsAtFile@
 cc-clang-backend    = @CC_CLANG_BACKEND@
 cc-llvm-backend     = @CC_LLVM_BACKEND@
 gcc-is-clang        = @GccIsClang@
-hs-cpp-args         = @HaskellCPPArgs@
+
+system-ar-supports-at-file = @ArSupportsAtFile_STAGE0@
 
 # Build options:
 #===============
@@ -64,6 +66,7 @@ host-vendor           = @HostVendor_CPP@
 target-platform       = @TargetPlatform@
 target-platform-full  = @TargetPlatformFull@
 target-arch           = @TargetArch_CPP@
+target-arch-arm-isa   = @ARM_ISA@
 target-os             = @TargetOS_CPP@
 target-vendor         = @TargetVendor_CPP@
 llvm-target           = @LLVMTarget_CPP@
@@ -90,6 +93,15 @@ project-git-commit-id = @ProjectGitCommitId@
 # Compilation and linking flags:
 #===============================
 
+# XXX: for most of the tools we have only up to
+#      stage2, as such building stage3 is
+#      is essentially impossible to do right now.
+
+conf-ar-args-stage0         = @AR_OPTS_STAGE0@
+conf-ar-args-stage1         = @ArArgs@
+conf-ar-args-stage2         = @ArArgs@
+conf-ar-args-stage3         = @ArArgs@
+
 conf-cc-args-stage0         = @CONF_CC_OPTS_STAGE0@
 conf-cc-args-stage1         = @CONF_CC_OPTS_STAGE1@
 conf-cc-args-stage2         = @CONF_CC_OPTS_STAGE2@
@@ -106,6 +118,8 @@ conf-ld-linker-args-stage0  = @CONF_LD_LINKER_OPTS_STAGE0@
 conf-ld-linker-args-stage1  = @CONF_LD_LINKER_OPTS_STAGE1@
 conf-ld-linker-args-stage2  = @CONF_LD_LINKER_OPTS_STAGE2@
 
+conf-hs-cpp-args            = @HaskellCPPArgs@
+
 # Include and library directories:
 #=================================
 

hadrian.cabal

@@ -10,6 +10,9 @@ category:            Development
 build-type:          Simple
 cabal-version:       >=1.10
 
+flag llvmng
+    default: false
+
 source-repository head
     type:     git
     location: https://github.com/snowleopard/hadrian
@@ -22,28 +25,35 @@ executable hadrian
                        , Builder
                        , CommandLine
                        , Context
+                       , Context.Paths
+                       , Context.Type
                        , Environment
                        , Expression
+                       , Expression.Type
                        , Flavour
                        , GHC
+                       , GHC.Packages
                        , Hadrian.Builder
                        , Hadrian.Builder.Ar
                        , Hadrian.Builder.Sphinx
                        , Hadrian.Builder.Tar
+                       , Hadrian.Builder.Types
                        , Hadrian.Expression
                        , Hadrian.Haskell.Cabal
+                       , Hadrian.Haskell.Cabal.Configured
                        , Hadrian.Haskell.Cabal.Parse
+                       , Hadrian.Haskell.Cabal.Type
                        , Hadrian.Oracles.ArgsHash
                        , Hadrian.Oracles.DirectoryContents
                        , Hadrian.Oracles.Path
                        , Hadrian.Oracles.TextFile
                        , Hadrian.Package
+                       , Hadrian.Package.Type
                        , Hadrian.Target
                        , Hadrian.Utilities
                        , Oracles.Flag
                        , Oracles.Setting
                        , Oracles.ModuleFiles
-                       , Oracles.PackageData
                        , Rules
                        , Rules.Clean
                        , Rules.Compile
@@ -86,26 +96,23 @@ executable hadrian
                        , Settings.Flavours.Profiled
                        , Settings.Flavours.Quick
                        , Settings.Flavours.QuickCross
+                       , Settings.Flavours.QuickCrossNCG
                        , Settings.Flavours.Quickest
-                       , Settings.Packages.Base
-                       , Settings.Packages.Cabal
-                       , Settings.Packages.Compiler
-                       , Settings.Packages.Ghc
-                       , Settings.Packages.GhcCabal
-                       , Settings.Packages.Ghci
-                       , Settings.Packages.GhcPkg
-                       , Settings.Packages.GhcPrim
-                       , Settings.Packages.Haddock
-                       , Settings.Packages.Haskeline
-                       , Settings.Packages.IntegerGmp
+                       , Settings.Packages
                        , Settings.Packages.Rts
-                       , Settings.Packages.RunGhc
                        , Settings.Warnings
                        , Stage
+                       , Stage.Type
                        , Target
                        , UserSettings
                        , Utilities
                        , Way
+                       , Way.Type
+    if flag(llvmng)
+      other-modules:     Settings.Flavours.QuickWithNG
+                       , Settings.Flavours.QuickCrossNG
+      cpp-options:     -DLLVMNG
+
     default-language:    Haskell2010
     default-extensions:  DeriveFunctor
                        , DeriveGeneric

shell.nix

@@ -2,11 +2,13 @@
 # by only invoking hadrian.
 
 
-{ nixpkgs ? import <nixpkgs> {} }:
+{ nixpkgs ? import <nixpkgs> {}
+, boot-ghc ? "ghc821" }:
 
 let
-  haskellPackages = nixpkgs.haskell.packages.ghc821;
-
+  ourtexlive = nixpkgs.texlive.combine
+    { inherit (nixpkgs.texlive) scheme-small fncychap; };
+  haskellPackages = nixpkgs.haskell.packages.${boot-ghc};
   removeBuild = path: type:
     let baseName = baseNameOf (toString path);
     in
@@ -20,32 +22,49 @@ let
            || nixpkgs.lib.hasSuffix ".sh" baseName
            || !(nixpkgs.lib.cleanSourceFilter path type)) ;
 
-  filterSrc = path: builtins.filterSource removeBuild path;
-
+  filterSrc = path: builtins.filterSource removeBuild path ;
 
-  hadrianPackages = nixpkgs.haskell.packages.ghc821.override {
+  hadrianPackages = haskellPackages.override {
     overrides = self: super: let
-        localPackage = name: path: self.callCabal2nix name (filterSrc path) {};
+        localPackage = name: path: self.callCabal2nix name (filterSrc path) {} ;
+	noCheck = nixpkgs.haskell.lib.dontCheck ;
       in {
         hadrian = localPackage "hadrian" ./. ;
-        shake = self.callHackage "shake" "0.16" {};
-        Cabal = localPackage "Cabal" ./../libraries/Cabal/Cabal ;
-        filepath = localPackage "filepath" ./../libraries/filepath ;
-        text = localPackage "text" ./../libraries/text  ;
-        hpc = localPackage"hpc" ./../libraries/hpc ;
-        parsec = localPackage "parsec" ./../libraries/parsec ;
-        HUnit = nixpkgs.haskell.lib.dontCheck (self.callHackage "HUnit" "1.3.1.2" {});
-        process = localPackage "process" ./../libraries/process ;
-        directory = localPackage "directory" ./../libraries/directory ;
+        shake = noCheck (self.callHackage "shake" "0.16" {}) ;
+        Cabal = noCheck (localPackage "Cabal" ./../libraries/Cabal/Cabal) ;
+        filepath = noCheck (localPackage "filepath" ./../libraries/filepath) ;
+        text = noCheck (localPackage "text" ./../libraries/text) ;
+        hpc = noCheck (localPackage "hpc" ./../libraries/hpc) ;
+        parsec = noCheck (localPackage "parsec" ./../libraries/parsec) ;
+        HUnit = noCheck (self.callHackage "HUnit" "1.3.1.2" {}) ;
+        process = noCheck (localPackage "process" ./../libraries/process) ;
+        directory = noCheck (localPackage "directory" ./../libraries/directory) ;
       }; };
 
 in
-  nixpkgs.lib.overrideDerivation nixpkgs.haskell.packages.ghcHEAD.ghc
+  nixpkgs.lib.overrideDerivation
+    (nixpkgs.haskell.compiler.ghcHEAD.override {
+      bootPkgs = haskellPackages;
+    })
     (drv: {
       name = "ghc-dev";
       buildInputs = drv.buildInputs ++ [
                     hadrianPackages.hadrian
                     nixpkgs.arcanist
-                    ];
-    })
-
+                    haskellPackages.alex
+                    haskellPackages.happy
+                    nixpkgs.python3
+                    nixpkgs.git
+                    nixpkgs.autoconf
+                    nixpkgs.automake
+                    nixpkgs.perl
+                    nixpkgs.gcc
+                    nixpkgs.python3Packages.sphinx
+                    nixpkgs.ncurses
+                    nixpkgs.m4
+                    nixpkgs.gmp
+                    nixpkgs.file
+                    nixpkgs.llvm_5
+		    ourtexlive
+		  ];
+  })

src/Base.hs

@@ -18,12 +18,14 @@ module Base (
     module Stage,
     module Way,
 
+    -- * Files
+    configH, ghcVersionH,
     -- * Paths
-    hadrianPath, configPath, configFile, sourcePath, configH, shakeFilesDir,
-    generatedDir, inplaceBinPath, inplaceLibBinPath, inplaceLibPath,
-    inplaceLibCopyTargets, haddockHtmlResourcesStamp, templateHscPath,
-    stage0PackageDbDir, inplacePackageDbPath, packageDbPath, packageDbStamp,
-    ghcSplitPath
+    hadrianPath, configPath, configFile, sourcePath, shakeFilesDir,
+    generatedDir, generatedPath, inplaceBinPath, inplaceLibBinPath, inplaceLibPath,
+    stageBinPath, stageLibPath,
+    ghcDeps, templateHscPath,
+    inplacePackageDbPath, packageDbPath, packageDbStamp, ghcSplitPath
     ) where
 
 import Control.Applicative
@@ -65,6 +67,9 @@ sourcePath = hadrianPath -/- "src"
 configH :: FilePath
 configH = "mk/config.h"
 
+ghcVersionH :: Action FilePath
+ghcVersionH = generatedPath <&> (-/- "ghcversion.h")
+
 -- | The directory in 'buildRoot' containing the Shake database and other
 -- auxiliary files generated by Hadrian.
 shakeFilesDir :: FilePath
@@ -87,47 +92,39 @@ inplaceLibBinPath = inplaceLibPath -/- "bin"
 generatedDir :: FilePath
 generatedDir = "generated"
 
--- | The directory in 'buildRoot' containing the 'Stage0' package database.
-stage0PackageDbDir :: FilePath
-stage0PackageDbDir = "stage0/bootstrapping.conf"
+generatedPath :: Action FilePath
+generatedPath = buildRoot <&> (-/- generatedDir)
 
--- | Path to the inplace package database used in 'Stage1' and later.
-inplacePackageDbPath :: FilePath
-inplacePackageDbPath = inplaceLibPath -/- "package.conf.d"
+-- | Path to the inplace package database.
+inplacePackageDbPath :: Stage -> FilePath
+inplacePackageDbPath stage = stageString stage -/- "lib" -/- "package.conf.d"
 
 -- | Path to the package database used in a given 'Stage'.
 packageDbPath :: Stage -> Action FilePath
-packageDbPath Stage0 = buildRoot <&> (-/- stage0PackageDbDir)
-packageDbPath _      = return inplacePackageDbPath
+packageDbPath stage  = buildRoot <&> (-/- inplacePackageDbPath stage)
 
 -- | We use a stamp file to track the existence of a package database.
 packageDbStamp :: FilePath
 packageDbStamp = ".stamp"
 
--- ref: GHC_DEPENDENCIES in ghc/ghc.mk
--- ref: INSTALL_LIBS in driver/ghc.mk
--- TODO: Derive this from Builder.runtimeDependencies
--- | Files that need to be copied over to 'inplaceLibPath'.
-inplaceLibCopyTargets :: [FilePath]
-inplaceLibCopyTargets = map (inplaceLibPath -/-)
-    [ "ghc-usage.txt"
-    , "ghci-usage.txt"
-    , "llvm-targets"
-    , "platformConstants"
-    , "settings"
-    , "template-hsc.h" ]
-
--- TODO: This is fragile and will break if @README.md@ is removed. We need to
--- improve the story of program runtime dependencies on directories.
--- See: https://github.com/snowleopard/hadrian/issues/492.
--- | Path to a file in Haddock's HTML resource library.
-haddockHtmlResourcesStamp :: FilePath
-haddockHtmlResourcesStamp = inplaceLibPath -/- "html/README.md"
-
--- ref: utils/hsc2hs/ghc.mk
--- | Path to 'hsc2hs' template.
-templateHscPath :: FilePath
-templateHscPath = inplaceLibPath -/- "template-hsc.h"
+stageBinPath :: Stage -> Action FilePath
+stageBinPath stage = buildRoot <&> (-/- stageString stage -/- "bin")
+
+stageLibPath :: Stage -> Action FilePath
+stageLibPath stage = buildRoot <&> (-/- stageString stage -/- "lib")
+
+-- | Files the `ghc` binary depends on
+ghcDeps :: Stage -> Action [FilePath]
+ghcDeps stage = mapM (\f -> stageLibPath stage <&> (-/- f))
+      [ "ghc-usage.txt"
+      , "ghci-usage.txt"
+      , "llvm-targets"
+      , "platformConstants"
+      , "settings" ]
+
+-- | Path to hsc2hs template.
+templateHscPath :: Stage -> Action FilePath
+templateHscPath stage = stageLibPath stage <&> (-/- "/template-hsc.h")
 
 -- | @ghc-split@ is a Perl script used by GHC when run with @-split-objs@ flag.
 -- It is generated in "Rules.Generate".