From dcadfa8ab486e8902fa153bd5c4b65b0cf5ce9e7 Mon Sep 17 00:00:00 2001 From: Hamish Mackenzie Date: Sat, 13 Jun 2020 01:03:17 +1200 Subject: [PATCH] Use suitable hackage state for index-state (#673) Fixes #672 by using the first index-state that is > than desired index state. It also passes `--index-state` to `cabal v2-cconfigre` to make sure it cabal uses only the packages in the hackage file that are before the specified date. Fixes #540 by returning the `index-state` used back out of `cabalProject` in the same way `plan-nix` is returned. --- lib/call-cabal-project-to-nix.nix | 65 ++++++++++++++++++---- lib/import-and-filter-project.nix | 7 +-- overlays/haskell.nix | 24 ++++++-- test/call-cabal-project-to-nix/default.nix | 10 ++-- test/call-stack-to-nix/default.nix | 8 +-- test/default.nix | 4 ++ test/index-state/default.nix | 39 +++++++++++++ 7 files changed, 126 insertions(+), 31 deletions(-) create mode 100644 test/index-state/default.nix diff --git a/lib/call-cabal-project-to-nix.nix b/lib/call-cabal-project-to-nix.nix index 5f283e73..aa57e820 100644 --- a/lib/call-cabal-project-to-nix.nix +++ b/lib/call-cabal-project-to-nix.nix @@ -122,14 +122,39 @@ let else builtins.trace ("Using latest index state" + (if name == null then "" else " for " + name) + "!") (pkgs.lib.last (builtins.attrNames index-state-hashes)); - # Lookup hash for the index state we found - index-sha256-found = if index-sha256 != null - then index-sha256 - else index-state-hashes.${index-state-found} or null; - in assert (if index-state-found == null then throw "No index state passed and none found in ${cabalProjectFileName}" else true); + + assert (if index-sha256 == null && !(pkgs.lib.hasSuffix "Z" index-state-found) + then throw "Index state found was ${index-state-found} and no `index-sha256` was provided. " + "The index hash lookup code requires zulu time zone (ends in a Z)" else true); + +let + # If a hash was not specified find a suitable cached index state to + # use that will contain all the packages we need. By using the + # first one after the desired index-state we can avoid recalculating + # when new index-state-hashes are added. + # See https://github.com/input-output-hk/haskell.nix/issues/672 + cached-index-state = if index-sha256 != null + then index-state-found + else + let + suitable-index-states = + builtins.filter + (s: s >= index-state-found) # This compare is why we need zulu time + (builtins.attrNames index-state-hashes); + in + if builtins.length suitable-index-states == 0 + then index-state-found + else pkgs.lib.head suitable-index-states; + + # Lookup hash for the index state we found + index-sha256-found = if index-sha256 != null + then index-sha256 + else index-state-hashes.${cached-index-state} or null; + +in assert (if index-sha256-found == null then throw "Unknown index-state ${index-state-found}, the latest index-state I know about is ${pkgs.lib.last (builtins.attrNames index-state-hashes)}. You may need to update to a newer hackage.nix." else true); @@ -389,12 +414,23 @@ let ''} export SSL_CERT_FILE=${cacert}/etc/ssl/certs/ca-bundle.crt export GIT_SSL_CAINFO=${cacert}/etc/ssl/certs/ca-bundle.crt - HOME=${dotCabal { - inherit cabal-install nix-tools extra-hackage-tarballs; - index-state = - builtins.trace ("Using index-state: ${index-state-found}" + (if name == null then "" else " for " + name)) - index-state-found; - sha256 = index-sha256-found; }} cabal v2-configure \ + HOME=${ + # This creates `.cabal` directory that is as it would have + # been at the time `cached-index-state`. We may include + # some packages that will be excluded by `index-state-found` + # which is used by cabal (cached-index-state >= index-state-found). + dotCabal { + inherit cabal-install nix-tools extra-hackage-tarballs; + index-state = cached-index-state; + sha256 = index-sha256-found; + } + } cabal v2-configure \ + --index-state=${ + # Setting the desired `index-state` here in case it was not + # from the cabal.project file. This will further restrict the + # packages used by the solver (cached-index-state >= index-state-found). + builtins.trace ("Using index-state: ${index-state-found}" + (if name == null then "" else " for " + name)) + index-state-found} \ --with-ghc=${ghc.targetPrefix}ghc \ --with-ghc-pkg=${ghc.targetPrefix}ghc-pkg \ --enable-tests \ @@ -437,4 +473,9 @@ let # move pkgs.nix to default.nix ensure we can just nix `import` the result. mv $out${subDir'}/pkgs.nix $out${subDir'}/default.nix ''); -in { projectNix = plan-nix; inherit src; inherit (fixedProject) sourceRepos; } +in { + projectNix = plan-nix; + index-state = index-state-found; + inherit src; + inherit (fixedProject) sourceRepos; +} diff --git a/lib/import-and-filter-project.nix b/lib/import-and-filter-project.nix index 5df939c0..025d59a8 100644 --- a/lib/import-and-filter-project.nix +++ b/lib/import-and-filter-project.nix @@ -19,9 +19,7 @@ let projectSubDir = pkgs.lib.strings.removePrefix "/" projectSubDir'; # Without / projectSubDir'' = if projectSubDir == "" then "" else projectSubDir + "/"; # With trailing / project = import "${projectNix}${projectSubDir'}"; -in { - nix = projectNix; - pkgs = project // { +in project // { extras = hackage: let old = (project.extras hackage).packages; in { packages = pkgs.lib.attrsets.mapAttrs (name: value: if builtins.isFunction value @@ -46,5 +44,4 @@ in { src = (pkgs.lib).mkDefault packageSrc; }) old; }; - }; -} + } diff --git a/overlays/haskell.nix b/overlays/haskell.nix index 98c99bf6..d9904d10 100644 --- a/overlays/haskell.nix +++ b/overlays/haskell.nix @@ -459,9 +459,12 @@ final: prev: { # plan-nix without building the project. cabalProject' = { ... }@args: - let plan = importAndFilterProject (callCabalProjectToNix args); + let + callProjectResults = callCabalProjectToNix args; in let pkg-set = mkCabalProjectPkgSet - { plan-pkgs = plan.pkgs; + { plan-pkgs = importAndFilterProject { + inherit (callProjectResults) projectNix sourceRepos src; + }; pkg-def-extras = args.pkg-def-extras or []; modules = (args.modules or []) ++ final.lib.optional (args ? ghcOverride || args ? ghc) @@ -470,7 +473,12 @@ final: prev: { { compiler.nix-name = args.compiler-nix-name; }; extra-hackages = args.extra-hackages or []; }; - in addProjectAndPackageAttrs { inherit (pkg-set.config) hsPkgs; inherit pkg-set; plan-nix = plan.nix; }; + in addProjectAndPackageAttrs { + inherit (pkg-set.config) hsPkgs; + inherit pkg-set; + plan-nix = callProjectResults.projectNix; + inherit (callProjectResults) index-state; + }; # Take `hsPkgs` from the `rawProject` and update all the packages and # components so they have a `.project` attribute and as well as @@ -509,20 +517,24 @@ final: prev: { stackProject' = { ... }@args: - let stack = importAndFilterProject (callStackToNix ({ inherit cache; } // args)); + let callProjectResults = callStackToNix ({ inherit cache; } // args); generatedCache = genStackCache { inherit (args) src; stackYaml = args.stackYaml or "stack.yaml"; }; cache = args.cache or generatedCache; in let pkg-set = mkStackPkgSet - { stack-pkgs = stack.pkgs; + { stack-pkgs = importAndFilterProject callProjectResults; pkg-def-extras = (args.pkg-def-extras or []); modules = final.lib.singleton (mkCacheModule cache) ++ (args.modules or []) ++ final.lib.optional (args ? ghc) { ghc.package = args.ghc; }; }; - in addProjectAndPackageAttrs { inherit (pkg-set.config) hsPkgs; inherit pkg-set; stack-nix = stack.nix; }; + in addProjectAndPackageAttrs { + inherit (pkg-set.config) hsPkgs; + inherit pkg-set; + stack-nix = callProjectResults.projectNix; + }; stackProject = args: let p = stackProject' args; in p.hsPkgs // { diff --git a/test/call-cabal-project-to-nix/default.nix b/test/call-cabal-project-to-nix/default.nix index b1aa9971..e3b46974 100644 --- a/test/call-cabal-project-to-nix/default.nix +++ b/test/call-cabal-project-to-nix/default.nix @@ -5,19 +5,21 @@ with stdenv.lib; let # This test could use cabalProject', but it does so that it # tests using callCabalProjectToNix and importAndFilterProject - plan = (importAndFilterProject (callCabalProjectToNix { + callProjectResults = callCabalProjectToNix { index-state = "2020-05-25T00:00:00Z"; # reuse the cabal-simple test project src = testSrc "cabal-simple"; - })); + }; pkgSet = mkCabalProjectPkgSet { - plan-pkgs = plan.pkgs; + plan-pkgs = importAndFilterProject { + inherit (callProjectResults) projectNix sourceRepos src; + }; }; packages = pkgSet.config.hsPkgs; in recurseIntoAttrs { ifdInputs = { - plan-nix = plan.nix; + plan-nix = callProjectResults.projectNix; }; run = stdenv.mkDerivation { name = "call-cabal-project-to-nix-test"; diff --git a/test/call-stack-to-nix/default.nix b/test/call-stack-to-nix/default.nix index b78b065d..11552998 100644 --- a/test/call-stack-to-nix/default.nix +++ b/test/call-stack-to-nix/default.nix @@ -3,11 +3,11 @@ with stdenv.lib; let - stack = importAndFilterProject (callStackToNix { + callProjectResults = callStackToNix { src = testSrc "stack-simple"; - }); + }; pkgSet = mkStackPkgSet { - stack-pkgs = stack.pkgs; + stack-pkgs = importAndFilterProject callProjectResults; pkg-def-extras = []; modules = []; }; @@ -15,7 +15,7 @@ let in recurseIntoAttrs { ifdInputs = { - stack-nix = stack.nix; + stack-nix = callProjectResults.projectNix; }; run = stdenv.mkDerivation { name = "callStackToNix-test"; diff --git a/test/default.nix b/test/default.nix index 4a684588..e8f02781 100644 --- a/test/default.nix +++ b/test/default.nix @@ -181,6 +181,10 @@ let # Pandoc does not build with ghcjs or ghc 8.10.1 yet (lookup-sha256 and fully-static build pandoc) lookup-sha256 = callTest ./lookup-sha256 {}; fully-static = callTest ./fully-static { inherit (pkgs) buildPackages; }; + } // lib.optionalAttrs (pkgs.haskell-nix.defaultCompilerNixName != "ghc8101") { + # This test makes a plan for building cabal 3.2 using index-states that will + # never work with ghc 8.10.1 + index-state = callTest ./index-state {}; }; # This is the same as allTests, but filter out all the key/vaules from the diff --git a/test/index-state/default.nix b/test/index-state/default.nix new file mode 100644 index 00000000..76c4da27 --- /dev/null +++ b/test/index-state/default.nix @@ -0,0 +1,39 @@ +# Test a package set +{ stdenv, testSrc, tool }: + +with stdenv.lib; + +let + # The hackage-security 0.6.0.1 was uploaded at 2020-04-06T20:54:35Z + # See https://hackage.haskell.org/package/hackage-security-0.6.0.1 + version-used-at = index-state: (tool "cabal" { + version = "3.2.0.0"; + inherit index-state; + }).project.hsPkgs.hackage-security.components.library.version; + version-before = version-used-at "2020-04-06T20:54:34Z"; + version-after = version-used-at "2020-04-06T20:54:35Z"; + +in + stdenv.mkDerivation { + name = "index-state-test"; + + buildCommand = '' + if [[ "${version-before}" != "0.6.0.0" ]]; then + echo 'Unexpected version ${version-before} (expected "0.6.0.0")' + exit 0 + fi + if [[ "${version-after}" != "0.6.0.1" ]]; then + echo 'Unexpected version ${version-after} (expected "0.6.0.1")' + exit 0 + fi + + touch $out + ''; + + meta.platforms = platforms.all; + + passthru = { + # Used for debugging with nix repl + inherit project packages; + }; + }