Use suitable hackage state for index-state (#673)

Fixes #672 by using the first index-state that is > than desired
index state.  It also passes `--index-state` to `cabal v2-cconfigre`
to make sure it cabal uses only the packages in the hackage
file that are before the specified date.

Fixes #540 by returning the `index-state` used back out of
`cabalProject` in the same way `plan-nix` is returned.

lib/call-cabal-project-to-nix.nix

@@ -122,14 +122,39 @@ let
           else builtins.trace ("Using latest index state" + (if name == null then "" else " for " + name) + "!")
             (pkgs.lib.last (builtins.attrNames index-state-hashes));
 
-  # Lookup hash for the index state we found
-  index-sha256-found = if index-sha256 != null
-    then index-sha256
-    else index-state-hashes.${index-state-found} or null;
-
 in
   assert (if index-state-found == null
     then throw "No index state passed and none found in ${cabalProjectFileName}" else true);
+
+  assert (if index-sha256 == null && !(pkgs.lib.hasSuffix "Z" index-state-found)
+    then throw "Index state found was ${index-state-found} and no `index-sha256` was provided. "
+      "The index hash lookup code requires zulu time zone (ends in a Z)" else true);
+
+let
+  # If a hash was not specified find a suitable cached index state to
+  # use that will contain all the packages we need.  By using the
+  # first one after the desired index-state we can avoid recalculating
+  # when new index-state-hashes are added.
+  # See https://github.com/input-output-hk/haskell.nix/issues/672
+  cached-index-state = if index-sha256 != null
+    then index-state-found
+    else
+      let
+        suitable-index-states =
+          builtins.filter
+            (s: s >= index-state-found) # This compare is why we need zulu time
+            (builtins.attrNames index-state-hashes);
+      in
+        if builtins.length suitable-index-states == 0
+          then index-state-found
+          else pkgs.lib.head suitable-index-states;
+
+  # Lookup hash for the index state we found
+  index-sha256-found = if index-sha256 != null
+    then index-sha256
+    else index-state-hashes.${cached-index-state} or null;
+
+in
   assert (if index-sha256-found == null
     then throw "Unknown index-state ${index-state-found}, the latest index-state I know about is ${pkgs.lib.last (builtins.attrNames index-state-hashes)}. You may need to update to a newer hackage.nix." else true);
 
@@ -389,12 +414,23 @@ let
     ''}
     export SSL_CERT_FILE=${cacert}/etc/ssl/certs/ca-bundle.crt
     export GIT_SSL_CAINFO=${cacert}/etc/ssl/certs/ca-bundle.crt
-    HOME=${dotCabal {
-      inherit cabal-install nix-tools extra-hackage-tarballs;
-      index-state =
-        builtins.trace ("Using index-state: ${index-state-found}" + (if name == null then "" else " for " + name))
-          index-state-found;
-      sha256 = index-sha256-found; }} cabal v2-configure \
+    HOME=${
+      # This creates `.cabal` directory that is as it would have
+      # been at the time `cached-index-state`.  We may include
+      # some packages that will be excluded by `index-state-found`
+      # which is used by cabal (cached-index-state >= index-state-found).  
+      dotCabal {
+        inherit cabal-install nix-tools extra-hackage-tarballs;
+        index-state = cached-index-state;
+        sha256 = index-sha256-found;
+      }
+    } cabal v2-configure \
+        --index-state=${
+            # Setting the desired `index-state` here in case it was not
+            # from the cabal.project file. This will further restrict the
+            # packages used by the solver (cached-index-state >= index-state-found).
+            builtins.trace ("Using index-state: ${index-state-found}" + (if name == null then "" else " for " + name))
+              index-state-found} \
         --with-ghc=${ghc.targetPrefix}ghc \
         --with-ghc-pkg=${ghc.targetPrefix}ghc-pkg \
         --enable-tests \
@@ -437,4 +473,9 @@ let
     # move pkgs.nix to default.nix ensure we can just nix `import` the result.
     mv $out${subDir'}/pkgs.nix $out${subDir'}/default.nix
   '');
-in { projectNix = plan-nix; inherit src; inherit (fixedProject) sourceRepos; }
+in {
+  projectNix = plan-nix;
+  index-state = index-state-found;
+  inherit src;
+  inherit (fixedProject) sourceRepos;
+}

lib/import-and-filter-project.nix

@@ -19,9 +19,7 @@ let
   projectSubDir = pkgs.lib.strings.removePrefix "/" projectSubDir';          # Without /
   projectSubDir'' = if projectSubDir == "" then "" else projectSubDir + "/"; # With trailing /
   project = import "${projectNix}${projectSubDir'}";
-in {
-  nix = projectNix;
-  pkgs = project // {
+in project // {
     extras = hackage: let old = (project.extras hackage).packages; in {
       packages = pkgs.lib.attrsets.mapAttrs (name: value:
         if builtins.isFunction value
@@ -46,5 +44,4 @@ in {
               src = (pkgs.lib).mkDefault packageSrc;
             }) old;
     };
-  };
-}
+  }

overlays/haskell.nix

@@ -459,9 +459,12 @@ final: prev: {
         # plan-nix without building the project.
         cabalProject' =
             { ... }@args:
-            let plan = importAndFilterProject (callCabalProjectToNix args);
+            let
+              callProjectResults = callCabalProjectToNix args;
             in let pkg-set = mkCabalProjectPkgSet
-                { plan-pkgs = plan.pkgs;
+                { plan-pkgs = importAndFilterProject {
+                    inherit (callProjectResults) projectNix sourceRepos src;
+                  };
                   pkg-def-extras = args.pkg-def-extras or [];
                   modules = (args.modules or [])
                           ++ final.lib.optional (args ? ghcOverride || args ? ghc)
@@ -470,7 +473,12 @@ final: prev: {
                               { compiler.nix-name = args.compiler-nix-name; };
                   extra-hackages = args.extra-hackages or [];
                 };
-            in addProjectAndPackageAttrs { inherit (pkg-set.config) hsPkgs; inherit pkg-set; plan-nix = plan.nix; };
+            in addProjectAndPackageAttrs {
+              inherit (pkg-set.config) hsPkgs;
+              inherit pkg-set;
+              plan-nix = callProjectResults.projectNix;
+              inherit (callProjectResults) index-state;
+            };
 
         # Take `hsPkgs` from the `rawProject` and update all the packages and
         # components so they have a `.project` attribute and as well as
@@ -509,20 +517,24 @@ final: prev: {
 
         stackProject' =
             { ... }@args:
-            let stack = importAndFilterProject (callStackToNix ({ inherit cache; } // args));
+            let callProjectResults = callStackToNix ({ inherit cache; } // args);
                 generatedCache = genStackCache {
                     inherit (args) src;
                     stackYaml = args.stackYaml or "stack.yaml";
                 };
                 cache = args.cache or generatedCache;
             in let pkg-set = mkStackPkgSet
-                { stack-pkgs = stack.pkgs;
+                { stack-pkgs = importAndFilterProject callProjectResults;
                   pkg-def-extras = (args.pkg-def-extras or []);
                   modules =  final.lib.singleton (mkCacheModule cache)
                              ++ (args.modules or [])
                              ++ final.lib.optional (args ? ghc) { ghc.package = args.ghc; };
                 };
-            in addProjectAndPackageAttrs { inherit (pkg-set.config) hsPkgs; inherit pkg-set; stack-nix = stack.nix; };
+            in addProjectAndPackageAttrs {
+              inherit (pkg-set.config) hsPkgs;
+              inherit pkg-set;
+              stack-nix = callProjectResults.projectNix;
+            };
 
         stackProject = args: let p = stackProject' args;
             in p.hsPkgs // {

test/call-cabal-project-to-nix/default.nix

@@ -5,19 +5,21 @@ with stdenv.lib;
 let
   # This test could use cabalProject', but it does so that it
   # tests using callCabalProjectToNix and importAndFilterProject
-  plan = (importAndFilterProject (callCabalProjectToNix {
+  callProjectResults = callCabalProjectToNix {
     index-state = "2020-05-25T00:00:00Z";
     # reuse the cabal-simple test project
     src = testSrc "cabal-simple";
-  }));
+  };
   pkgSet = mkCabalProjectPkgSet {
-    plan-pkgs = plan.pkgs;
+    plan-pkgs = importAndFilterProject {
+      inherit (callProjectResults) projectNix sourceRepos src;
+    };
   };
   packages = pkgSet.config.hsPkgs;
 
 in recurseIntoAttrs {
   ifdInputs = {
-    plan-nix = plan.nix;
+    plan-nix = callProjectResults.projectNix;
   };
   run = stdenv.mkDerivation {
     name = "call-cabal-project-to-nix-test";

test/call-stack-to-nix/default.nix

@@ -3,11 +3,11 @@
 with stdenv.lib;
 
 let
-  stack = importAndFilterProject (callStackToNix {
+  callProjectResults = callStackToNix {
     src = testSrc "stack-simple";
-  });
+  };
   pkgSet = mkStackPkgSet {
-    stack-pkgs = stack.pkgs;
+    stack-pkgs = importAndFilterProject callProjectResults;
     pkg-def-extras = [];
     modules = [];
   };
@@ -15,7 +15,7 @@ let
 
 in recurseIntoAttrs {
   ifdInputs = {
-    stack-nix = stack.nix;
+    stack-nix = callProjectResults.projectNix;
   };
   run = stdenv.mkDerivation {
     name = "callStackToNix-test";

test/default.nix

@@ -181,6 +181,10 @@ let
     # Pandoc does not build with ghcjs or ghc 8.10.1 yet (lookup-sha256 and fully-static build pandoc)
     lookup-sha256 = callTest ./lookup-sha256 {};
     fully-static = callTest ./fully-static { inherit (pkgs) buildPackages; };
+  } // lib.optionalAttrs (pkgs.haskell-nix.defaultCompilerNixName != "ghc8101") {
+    # This test makes a plan for building cabal 3.2 using index-states that will
+    # never work with ghc 8.10.1
+    index-state = callTest ./index-state {};
   };
 
   # This is the same as allTests, but filter out all the key/vaules from the

test/index-state/default.nix

@@ -0,0 +1,39 @@
+# Test a package set
+{ stdenv, testSrc, tool }:
+
+with stdenv.lib;
+
+let
+  # The hackage-security 0.6.0.1 was uploaded at 2020-04-06T20:54:35Z
+  # See https://hackage.haskell.org/package/hackage-security-0.6.0.1
+  version-used-at = index-state: (tool "cabal" {
+      version = "3.2.0.0";
+      inherit index-state;
+    }).project.hsPkgs.hackage-security.components.library.version;
+  version-before = version-used-at "2020-04-06T20:54:34Z";
+  version-after = version-used-at "2020-04-06T20:54:35Z";
+
+in
+  stdenv.mkDerivation {
+    name = "index-state-test";
+
+    buildCommand = ''
+      if [[ "${version-before}" != "0.6.0.0" ]]; then
+        echo 'Unexpected version ${version-before} (expected "0.6.0.0")'
+        exit 0
+      fi
+      if [[ "${version-after}" != "0.6.0.1" ]]; then
+        echo 'Unexpected version ${version-after} (expected "0.6.0.1")'
+        exit 0
+      fi
+
+      touch $out
+    '';
+
+    meta.platforms = platforms.all;
+
+    passthru = {
+      # Used for debugging with nix repl
+      inherit project packages;
+    };
+  }