diff --git a/docs/user-guide/clean-git.md b/docs/user-guide/clean-git.md index 6b8177dd..8127b8f2 100644 --- a/docs/user-guide/clean-git.md +++ b/docs/user-guide/clean-git.md @@ -38,3 +38,53 @@ components.tests.test.src = haskell-nix.haskellLib.cleanSourceWith { }; ``` +## Multiple Git Repositories with cleanGits + +Some times it is handy to temporarily use a relative path between git +repos. If the repos are individually cleaned this is not possible +(since the cleaned version of one repo will never include the files +of the other). + +There are 3 options: + +* We could `symlinkJoin` the cleaned directories together, but the + result could not be cleaned and any change would to either + repo would result in a rebuild of everything. + +* We could add one repo to the other as a submodule, + but adding and then removing a submodule is a pain and it does not + work well if you have more than one repo that needs to share the + submodule. + +* We could add a `source-repository-package` but then we would have + to commit each change before testing. + +`cleanGits` allows us to specify a root directory and any number of +sub directories containing git repos. + +For example if `repoA` and `repoB` are two git repos with +cabal packages and want to use the `repoB` package when building +`repoA. First we can add `../repoB` to `repoA/cabal.project`: + +``` +packages: + ./. + ../repoB +``` + +Then in `repoA/default.nix` we can use: + +``` +haskell-nix.cabalProject { + src = haskell-nix.haskellLib.cleanSourceWith { + src = haskell-nix.haskellLib.cleanGits { + name = "root"; + src = ../.; # Parent dir that contains repoA and repoB + gitDirs = [ "repoA" "repoB" ]; + }; + subDir = "repoA"; # Where to look for the `cabal.project` + includeSiblings = true; # Tells it not to exclude `repoB` dir + }; +} +``` + diff --git a/lib/call-cabal-project-to-nix.nix b/lib/call-cabal-project-to-nix.nix index d5e3b98d..25e1a0c0 100644 --- a/lib/call-cabal-project-to-nix.nix +++ b/lib/call-cabal-project-to-nix.nix @@ -54,20 +54,23 @@ in let ghc = ghc'; + subDir' = src.origSubDir or ""; + subDir = pkgs.lib.strings.removePrefix "/" subDir'; maybeCleanedSource = if haskellLib.canCleanSource src - then haskellLib.cleanSourceWith { - inherit src; - filter = path: type: - type == "directory" || - pkgs.lib.any (i: (pkgs.lib.hasSuffix i path)) [ ".project" ".cabal" ".freeze" "package.yaml" ]; } - else src; + then (haskellLib.cleanSourceWith { + name = src.name + "-root-cabal-files"; + src = src.origSrc; + filter = path: type: src.filter path type && ( + type == "directory" || + pkgs.lib.any (i: (pkgs.lib.hasSuffix i path)) [ ".project" ".cabal" ".freeze" "package.yaml" ]); }) + else src.origSrc or src; # Using origSrcSubDir bypasses any cleanSourceWith so that it will work when # access to the store is restricted. If origSrc was already in the store # you can pass the project in as a string. rawCabalProject = - let origSrcDir = maybeCleanedSource.origSrcSubDir or maybeCleanedSource; + let origSrcDir = (maybeCleanedSource.origSrcSubDir or maybeCleanedSource) + subDir'; in if cabalProject != null then cabalProject else @@ -321,12 +324,13 @@ let fi cp -r ${maybeCleanedSource}/* . chmod +w -R . - ${fixedProject.makeFixedProjectFile} # warning: this may not generate the proper cabal file. # hpack allows globbing, and turns that into module lists # without the source available (we cleaneSourceWith'd it), # this may not produce the right result. find . -name package.yaml -exec hpack "{}" \; + ${pkgs.lib.optionalString (subDir != "") "cd ${subDir}"} + ${fixedProject.makeFixedProjectFile} export SSL_CERT_FILE=${cacert}/etc/ssl/certs/ca-bundle.crt export GIT_SSL_CAINFO=${cacert}/etc/ssl/certs/ca-bundle.crt HOME=${dotCabal { @@ -357,24 +361,24 @@ let # make sure the path's in the plan.json are relative to $out instead of $tmp # this is necessary so that plan-to-nix relative path logic can work. - substituteInPlace $tmp/dist-newstyle/cache/plan.json --replace "$tmp" "$out" + substituteInPlace $tmp${subDir'}/dist-newstyle/cache/plan.json --replace "$tmp" "$out" # run `plan-to-nix` in $out. This should produce files right there with the # proper relative paths. - (cd $out && plan-to-nix --full --plan-json $tmp/dist-newstyle/cache/plan.json -o .) + (cd $out${subDir'} && plan-to-nix --full --plan-json $tmp${subDir'}/dist-newstyle/cache/plan.json -o .) # Remove the non nix files ".project" ".cabal" "package.yaml" files # as they should not be in the output hash (they may change slightly # without affecting the nix). - if [ -d $out/.source-repository-packages ]; then - chmod +w -R $out/.source-repository-packages - rm -rf $out/.source-repository-packages + if [ -d $out${subDir'}/.source-repository-packages ]; then + chmod +w -R $out${subDir'}/.source-repository-packages + rm -rf $out${subDir'}/.source-repository-packages fi find $out \( -type f -or -type l \) ! -name '*.nix' -delete # Remove empty dirs find $out -type d -empty -delete # move pkgs.nix to default.nix ensure we can just nix `import` the result. - mv $out/pkgs.nix $out/default.nix + mv $out${subDir'}/pkgs.nix $out${subDir'}/default.nix ''); in { projectNix = plan-nix; inherit src; inherit (fixedProject) sourceRepos; } diff --git a/lib/call-stack-to-nix.nix b/lib/call-stack-to-nix.nix index 2613245d..8e58514d 100644 --- a/lib/call-stack-to-nix.nix +++ b/lib/call-stack-to-nix.nix @@ -16,6 +16,7 @@ , checkMaterialization ? null # If true the nix files will be generated used to check plan-sha256 and material , ... }: let + subDir' = src.origSubDir or ""; stackToNixArgs = builtins.concatStringsSep " " [ "--full" "--stack-yaml=${src}/${if stackYaml == null then "stack.yaml" else stackYaml}" @@ -37,11 +38,11 @@ let LC_ALL = "en_US.UTF-8"; preferLocalBuild = false; } ('' - mkdir -p $out + mkdir -p $out${subDir'} '' + pkgs.lib.optionalString (cache != null) '' - cp ${mkCacheFile cache}/.stack-to-nix.cache* $out + cp ${mkCacheFile cache}/.stack-to-nix.cache* $out${subDir'} '' + '' - (cd $out && stack-to-nix ${stackToNixArgs}) + (cd $out${subDir'} && stack-to-nix ${stackToNixArgs}) # We need to strip out any references to $src, as those won't # be accessable in restricted mode. @@ -50,6 +51,6 @@ let done # move pkgs.nix to default.nix ensure we can just nix `import` the result. - mv $out/pkgs.nix $out/default.nix + mv $out${subDir'}/pkgs.nix $out${subDir'}/default.nix '')); in { projectNix = stack; inherit src; sourceRepos = []; } diff --git a/lib/clean-git.nix b/lib/clean-git.nix index f460dcea..f7122665 100644 --- a/lib/clean-git.nix +++ b/lib/clean-git.nix @@ -33,7 +33,9 @@ let whitelist_set = listToAttrs ( concatMap (p: - let full_path = toString (root + "/${p}"); in + # Using `origSrcSubDir` (if present) makes it possible to cleanGit src that + # has already been cleaned with cleanSrcWith. + let full_path = root.origSrcSubDir or (toString root) + "/${p}"; in map (p': { name = p'; value = true; }) (all_paren_dirs full_path) ) files ); diff --git a/lib/clean-source-with.nix b/lib/clean-source-with.nix index 44af0e23..6cc2e9f2 100644 --- a/lib/clean-source-with.nix +++ b/lib/clean-source-with.nix @@ -35,6 +35,10 @@ # It will be as if `src = src + "/${subDir}` and filters # already applied to `src` will be respected. # + # includeSiblings: By default the siblings trees of `subDir` are excluded. + # In some cases it is useful to include these so that + # relative references to those siblings will work. + # # name: Optional name to use as part of the store path. # If you do not provide a `name` it wil be derived # from the `subDir`. You should provide `name` or @@ -47,7 +51,7 @@ # the message to the use more meaningful. # cleanSourceWith = { filter ? _path: _type: true, src, subDir ? "", name ? null - , caller ? "cleanSourceWith" }: + , caller ? "cleanSourceWith", includeSiblings ? false }: let subDir' = if subDir == "" then "" else "/" + subDir; subDirName = __replaceStrings ["/"] ["-"] subDir; @@ -61,12 +65,16 @@ then path: type: src.filter path type else path: type: true; filter' = path: type: - # Include parent paths based on the parent filter - (lib.strings.hasPrefix (path + "/") (origSrcSubDir + "/") - && parentFilter path type) - # Children only if both filters return true - || (lib.strings.hasPrefix (origSrcSubDir + "/") path - && (filter path type && parentFilter path type)); + # Respect the parent filter + parentFilter path type && ( + # Must include parent paths of the subdir. + (lib.strings.hasPrefix (path + "/") (origSrcSubDir + "/")) + || + # Everything else is either the child tree or sibling tree. + ((includeSiblings || lib.strings.hasPrefix (origSrcSubDir + "/") path) + && filter path type # Use the filter function to decide if we need it + ) + ); name' = if name != null then name else diff --git a/lib/default.nix b/lib/default.nix index 9ec61ce1..f2c823d2 100644 --- a/lib/default.nix +++ b/lib/default.nix @@ -208,6 +208,31 @@ in { inherit (pkgs.evalPackages) runCommand; }; + # Some times it is handy to temporarily use a relative path between git + # repos. If the repos are individually cleaned this is not possible + # (since the cleaned version of one repo will never include the files + # of the other). + # + # `cleanGits` allows us to specify a root directory and any number of + # sub directories containing git repos. + # + # See docs/user-guide/clean-git.md for details of how to use this + # with `cabalProject`. + cleanGits = { src, gitDirs, name ? null, caller ? "cleanGits" }@args: + let + # List of filters, one for each git directory. + filters = builtins.map (subDir: + (pkgs.haskell-nix.haskellLib.cleanGit { + src = pkgs.haskell-nix.haskellLib.cleanSourceWith { + inherit src subDir; + }; + }).filter) gitDirs; + in pkgs.haskell-nix.haskellLib.cleanSourceWith { + inherit src name caller; + # Keep files that match any of the filters + filter = path: type: pkgs.lib.any (f: f path type) filters; + }; + # Check a test component check = import ./check.nix { inherit stdenv lib haskellLib srcOnly; diff --git a/lib/import-and-filter-project.nix b/lib/import-and-filter-project.nix index 652fa1aa..5df939c0 100644 --- a/lib/import-and-filter-project.nix +++ b/lib/import-and-filter-project.nix @@ -4,7 +4,21 @@ { pkgs, haskellLib }: { projectNix, sourceRepos, src }: let - project = import "${projectNix}"; + # Full source including possible relartive paths form the + # project directory. + srcRoot = + if haskellLib.canCleanSource src + then haskellLib.cleanSourceWith { + name = src.name + "-root"; + src = src.origSrc or src; + inherit (src) filter; + } + else src.origSrc or src; + # The sub directory containing the cabal.project or stack.yaml file + projectSubDir' = src.origSubDir or ""; # With leading / + projectSubDir = pkgs.lib.strings.removePrefix "/" projectSubDir'; # Without / + projectSubDir'' = if projectSubDir == "" then "" else projectSubDir + "/"; # With trailing / + project = import "${projectNix}${projectSubDir'}"; in { nix = projectNix; pkgs = project // { @@ -20,14 +34,14 @@ in { subDir = pkgs.lib.strings.removePrefix "/" ( pkgs.lib.strings.removePrefix (toString projectNix) (toString oldPkg.src.content)); - srcRepoPrefix = ".source-repository-packages/"; + srcRepoPrefix = projectSubDir'' + ".source-repository-packages/"; in if pkgs.lib.strings.hasPrefix srcRepoPrefix subDir then pkgs.lib.lists.elemAt sourceRepos ( pkgs.lib.strings.toInt (pkgs.lib.strings.removePrefix srcRepoPrefix subDir)) - else if haskellLib.canCleanSource src - then haskellLib.cleanSourceWith { inherit src subDir; } - else src + (if subDir == "" then "" else "/" + subDir); + else if haskellLib.canCleanSource srcRoot + then haskellLib.cleanSourceWith { src = srcRoot; inherit subDir; } + else srcRoot + (if subDir == "" then "" else "/" + subDir); in oldPkg // { src = (pkgs.lib).mkDefault packageSrc; }) old;