resholve: init at 0.4.0 (#85827)
resholve: init at 0.4.0 resholve attempts to resolve executables in shell scripts. Includes Nix builder for resolving dependencies in Nix-built shell projects.gstqt5
Travis A. Everett
GitHub
parent
645f39f33e
commit
6fd9283bba
|
|
@ -0,0 +1,138 @@
|
|||
# Using resholve's Nix API
|
||||
|
||||
resholve converts bare executable references in shell scripts to absolute
|
||||
paths. This will hopefully make its way into the Nixpkgs manual soon, but
|
||||
until then I'll outline how to use the `resholvePackage` function.
|
||||
|
||||
> Fair warning: resholve does *not* aspire to resolving all valid Shell
|
||||
> scripts. It depends on the OSH/Oil parser, which aims to support most (but
|
||||
> not all) Bash, and aims to be a ~90% sort of solution.
|
||||
|
||||
Let's start with a simple example from one of my own projects:
|
||||
|
||||
```nix
|
||||
{ stdenv, lib, resholvePackage, fetchFromGitHub, bashup-events44, bashInteractive_5, doCheck ? true, shellcheck }:
|
||||
|
||||
resholvePackage rec {
|
||||
pname = "shellswain";
|
||||
version = "unreleased";
|
||||
|
||||
src = fetchFromGitHub {
|
||||
# ...
|
||||
};
|
||||
|
||||
solutions = {
|
||||
profile = {
|
||||
# the only *required* arguments
|
||||
scripts = [ "bin/shellswain.bash" ];
|
||||
interpreter = "none";
|
||||
inputs = [ bashup-events44 ];
|
||||
};
|
||||
};
|
||||
|
||||
makeFlags = [ "prefix=${placeholder "out"}" ];
|
||||
|
||||
inherit doCheck;
|
||||
checkInputs = [ shellcheck ];
|
||||
|
||||
# ...
|
||||
}
|
||||
```
|
||||
|
||||
I'll focus on the `solutions` attribute, since this is the only part
|
||||
that differs from other derivations.
|
||||
|
||||
Each "solution" (k=v pair)
|
||||
describes one resholve invocation. For most shell packages, one
|
||||
invocation will probably be enough. resholve will make you be very
|
||||
explicit about your script's dependencies, and it may also need your
|
||||
help sorting out some references or problems that it can't safely
|
||||
handle on its own.
|
||||
|
||||
If you have more than one script, and your scripts need conflicting
|
||||
directives, you can specify more than one solution to resolve the
|
||||
scripts separately, but still produce a single package.
|
||||
|
||||
Let's take a closer look:
|
||||
|
||||
```nix
|
||||
solutions = {
|
||||
# each solution has a short name; this is what you'd use to
|
||||
# override the settings of this solution, and it may also show up
|
||||
# in (some) error messages.
|
||||
profile = {
|
||||
# specify one or more $out-relative script paths (unlike many
|
||||
# builders, resholve will modify the output files during fixup
|
||||
# to correctly resolve scripts that source within the package)
|
||||
scripts = [ "bin/shellswain.bash" ];
|
||||
# "none" for no shebang, "${bash}/bin/bash" for bash, etc.
|
||||
interpreter = "none";
|
||||
# packages resholve should resolve executables from
|
||||
inputs = [ bashup-events44 ];
|
||||
};
|
||||
};
|
||||
```
|
||||
|
||||
resholve has a (growing) number of options for handling more complex
|
||||
scripts. I won't cover these in excruciating detail here. You can find
|
||||
more information about these in `man resholve` via `nixpkgs.resholve`.
|
||||
|
||||
Instead, we'll look at the general form of the solutions attrset:
|
||||
|
||||
```nix
|
||||
solutions = {
|
||||
shortname = {
|
||||
# required
|
||||
# $out-relative paths to try resolving
|
||||
scripts = [ "bin/shunit2" ];
|
||||
# packages to resolve executables from
|
||||
inputs = [ coreutils gnused gnugrep findutils ];
|
||||
# path for shebang, or 'none' to omit shebang
|
||||
interpreter = "${bash}/bin/bash";
|
||||
|
||||
# optional
|
||||
fake = { fake directives };
|
||||
fix = { fix directives };
|
||||
keep = { keep directives };
|
||||
# file to inject before first code-line of script
|
||||
prologue = file;
|
||||
# file to inject after last code-line of script
|
||||
epilogue = file;
|
||||
# extra command-line flags passed to resholve; generally this API
|
||||
# should align with what resholve supports, but flags may help if
|
||||
# you need to override the version of resholve.
|
||||
flags = [ ];
|
||||
};
|
||||
};
|
||||
```
|
||||
|
||||
The main way you'll adjust how resholve handles your scripts are the
|
||||
fake, fix, and keep directives. The manpage covers their purpose and
|
||||
how to format them on the command-line, so I'll focus on how you'll
|
||||
need to translate them into Nix types.
|
||||
|
||||
```nix
|
||||
# --fake 'f:setUp;tearDown builtin:setopt source:/etc/bashrc'
|
||||
fake = {
|
||||
function = [ "setUp" "tearDown" ];
|
||||
builtin = [ "setopt" ];
|
||||
source = [ "/etc/bashrc" ];
|
||||
};
|
||||
|
||||
# --fix 'aliases xargs:ls $GIT:gix'
|
||||
fix = {
|
||||
# all single-word directives use `true` as value
|
||||
aliases = true;
|
||||
xargs = [ "ls" ];
|
||||
"$GIT" = [ "gix" ];
|
||||
};
|
||||
|
||||
# --keep 'which:git;ls .:$HOME $LS:exa /etc/bashrc ~/.bashrc'
|
||||
keep = {
|
||||
which = [ "git" "ls" ];
|
||||
"." = [ "$HOME" ];
|
||||
"$LS" = [ "exa" ];
|
||||
"/etc/bashrc" = true;
|
||||
"~/.bashrc" = true;
|
||||
};
|
||||
```
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
{ callPackage
|
||||
, doCheck ? true
|
||||
}:
|
||||
|
||||
rec {
|
||||
resholve = callPackage ./resholve.nix { inherit doCheck; };
|
||||
resholvePackage =
|
||||
callPackage ./resholve-package.nix { inherit resholve; };
|
||||
}
|
||||
|
|
@ -0,0 +1,120 @@
|
|||
{ stdenv
|
||||
, python27Packages
|
||||
, fetchFromGitHub
|
||||
, makeWrapper
|
||||
, # re2c deps
|
||||
autoreconfHook
|
||||
, # py-yajl deps
|
||||
git
|
||||
, # oil deps
|
||||
readline
|
||||
, cmark
|
||||
, file
|
||||
, glibcLocales
|
||||
, oilPatches ? [ ]
|
||||
}:
|
||||
|
||||
/*
|
||||
Notes on specific dependencies:
|
||||
- if/when python2.7 is removed from nixpkgs, this may need to figure
|
||||
out how to build oil's vendored python2
|
||||
- I'm not sure if glibcLocales is worth the addition here. It's to fix
|
||||
a libc test oil runs. My oil fork just disabled the libc tests, but
|
||||
I haven't quite decided if that's the right long-term call, so I
|
||||
didn't add a patch for it here yet.
|
||||
*/
|
||||
|
||||
rec {
|
||||
# had to add this as well; 1.3 causes a break here; sticking
|
||||
# to oil's official 1.0.3 dep for now.
|
||||
re2c = stdenv.mkDerivation rec {
|
||||
pname = "re2c";
|
||||
version = "1.0.3";
|
||||
sourceRoot = "${src.name}/re2c";
|
||||
src = fetchFromGitHub {
|
||||
owner = "skvadrik";
|
||||
repo = "re2c";
|
||||
rev = version;
|
||||
sha256 = "0grx7nl9fwcn880v5ssjljhcb9c5p2a6xpwil7zxpmv0rwnr3yqi";
|
||||
};
|
||||
nativeBuildInputs = [ autoreconfHook ];
|
||||
preCheck = ''
|
||||
patchShebangs run_tests.sh
|
||||
'';
|
||||
};
|
||||
|
||||
py-yajl = python27Packages.buildPythonPackage rec {
|
||||
pname = "oil-pyyajl-unstable";
|
||||
version = "2019-12-05";
|
||||
src = fetchFromGitHub {
|
||||
owner = "oilshell";
|
||||
repo = "py-yajl";
|
||||
rev = "eb561e9aea6e88095d66abcc3990f2ee1f5339df";
|
||||
sha256 = "17hcgb7r7cy8r1pwbdh8di0nvykdswlqj73c85k6z8m0filj3hbh";
|
||||
fetchSubmodules = true;
|
||||
};
|
||||
# just for submodule IIRC
|
||||
nativeBuildInputs = [ git ];
|
||||
};
|
||||
|
||||
# resholve's primary dependency is this developer build of the oil shell.
|
||||
oildev = python27Packages.buildPythonPackage rec {
|
||||
pname = "oildev-unstable";
|
||||
version = "2020-03-31";
|
||||
|
||||
src = fetchFromGitHub {
|
||||
owner = "oilshell";
|
||||
repo = "oil";
|
||||
rev = "ea80cdad7ae1152a25bd2a30b87fe3c2ad32394a";
|
||||
sha256 = "0pxn0f8qbdman4gppx93zwml7s5byqfw560n079v68qjgzh2brq2";
|
||||
|
||||
/*
|
||||
It's not critical to drop most of these; the primary target is
|
||||
the vendored fork of Python-2.7.13, which is ~ 55M and over 3200
|
||||
files, dozens of which get interpreter script patches in fixup.
|
||||
*/
|
||||
extraPostFetch = ''
|
||||
rm -rf Python-2.7.13 benchmarks metrics py-yajl rfc gold web testdata services demo devtools cpp
|
||||
'';
|
||||
};
|
||||
|
||||
# TODO: not sure why I'm having to set this for nix-build...
|
||||
# can anyone tell if I'm doing something wrong?
|
||||
SOURCE_DATE_EPOCH = 315532800;
|
||||
|
||||
# These aren't, strictly speaking, nix/nixpkgs specific, but I've
|
||||
# had hell upstreaming them. Pulling from resholve source and
|
||||
# passing in from resholve.nix
|
||||
patches = oilPatches;
|
||||
|
||||
buildInputs = [ readline cmark py-yajl ];
|
||||
|
||||
nativeBuildInputs = [ re2c file makeWrapper ];
|
||||
|
||||
propagatedBuildInputs = with python27Packages; [ six typing ];
|
||||
|
||||
doCheck = true;
|
||||
|
||||
preBuild = ''
|
||||
build/dev.sh all
|
||||
'';
|
||||
|
||||
postPatch = ''
|
||||
patchShebangs asdl build core doctools frontend native oil_lang
|
||||
'';
|
||||
|
||||
_NIX_SHELL_LIBCMARK = "${cmark}/lib/libcmark${stdenv.hostPlatform.extensions.sharedLibrary}";
|
||||
|
||||
# See earlier note on glibcLocales
|
||||
LOCALE_ARCHIVE = stdenv.lib.optionalString (stdenv.buildPlatform.libc == "glibc") "${glibcLocales}/lib/locale/locale-archive";
|
||||
|
||||
meta = {
|
||||
description = "A new unix shell";
|
||||
homepage = "https://www.oilshell.org/";
|
||||
license = with stdenv.lib.licenses; [
|
||||
psfl # Includes a portion of the python interpreter and standard library
|
||||
asl20 # Licence for Oil itself
|
||||
];
|
||||
};
|
||||
};
|
||||
}
|
||||
|
|
@ -0,0 +1,97 @@
|
|||
{ stdenv, lib, resholve }:
|
||||
|
||||
{ pname
|
||||
, src
|
||||
, version
|
||||
, passthru ? { }
|
||||
, solutions
|
||||
, ...
|
||||
}@attrs:
|
||||
let
|
||||
inherit stdenv;
|
||||
/* These functions break up the work of partially validating the
|
||||
* 'solutions' attrset and massaging it into env/cli args.
|
||||
*
|
||||
* Note: some of the left-most args do not *have* to be passed as
|
||||
* deep as they are, but I've done so to provide more error context
|
||||
*/
|
||||
|
||||
# for brevity / line length
|
||||
spaces = l: builtins.concatStringsSep " " l;
|
||||
semicolons = l: builtins.concatStringsSep ";" l;
|
||||
|
||||
/* Throw a fit with dotted attr path context */
|
||||
nope = path: msg:
|
||||
throw "${builtins.concatStringsSep "." path}: ${msg}";
|
||||
|
||||
/* Special-case directive value representations by type */
|
||||
makeDirective = solution: env: name: val:
|
||||
if builtins.isInt val then builtins.toString val
|
||||
else if builtins.isString val then name
|
||||
else if true == val then name
|
||||
else if false == val then "" # omit!
|
||||
else if null == val then "" # omit!
|
||||
else if builtins.isList val then "${name}:${semicolons val}"
|
||||
else nope [ solution env name ] "unexpected type: ${builtins.typeOf val}";
|
||||
|
||||
/* Build fake/fix/keep directives from Nix types */
|
||||
makeDirectives = solution: env: val:
|
||||
lib.mapAttrsToList (makeDirective solution env) val;
|
||||
|
||||
/* Special-case value representation by type/name */
|
||||
makeEnvVal = solution: env: val:
|
||||
if env == "inputs" then lib.makeBinPath val
|
||||
else if builtins.isString val then val
|
||||
else if builtins.isList val then spaces val
|
||||
else if builtins.isAttrs val then spaces (makeDirectives solution env val)
|
||||
else nope [ solution env ] "unexpected type: ${builtins.typeOf val}";
|
||||
|
||||
/* Shell-format each env value */
|
||||
shellEnv = solution: env: value:
|
||||
lib.escapeShellArg (makeEnvVal solution env value);
|
||||
|
||||
/* Build a single ENV=val pair */
|
||||
makeEnv = solution: env: value:
|
||||
"RESHOLVE_${lib.toUpper env}=${shellEnv solution env value}";
|
||||
|
||||
/* Discard attrs claimed by makeArgs */
|
||||
removeCliArgs = value:
|
||||
removeAttrs value [ "scripts" "flags" ];
|
||||
|
||||
/* Verify required arguments are present */
|
||||
validateSolution = { scripts, inputs, interpreter, ... }: true;
|
||||
|
||||
/* Pull out specific solution keys to build ENV=val pairs */
|
||||
makeEnvs = solution: value:
|
||||
spaces (lib.mapAttrsToList (makeEnv solution) (removeCliArgs value));
|
||||
|
||||
/* Pull out specific solution keys to build CLI argstring */
|
||||
makeArgs = { flags ? [ ], scripts, ... }:
|
||||
spaces (flags ++ scripts);
|
||||
|
||||
/* Build a single resholve invocation */
|
||||
makeInvocation = solution: value:
|
||||
if validateSolution value then
|
||||
"${makeEnvs solution value} resholve --overwrite ${makeArgs value}"
|
||||
else throw "invalid solution"; # shouldn't trigger for now
|
||||
|
||||
/* Build resholve invocation for each solution. */
|
||||
makeCommands = solutions:
|
||||
lib.mapAttrsToList makeInvocation solutions;
|
||||
|
||||
self = (stdenv.mkDerivation ((removeAttrs attrs [ "solutions" ])
|
||||
// {
|
||||
inherit pname version src;
|
||||
buildInputs = [ resholve ];
|
||||
|
||||
# enable below for verbose debug info if needed
|
||||
# supports default python.logging levels
|
||||
# LOGLEVEL="INFO";
|
||||
preFixup = ''
|
||||
pushd "$out"
|
||||
${builtins.concatStringsSep "\n" (makeCommands solutions)}
|
||||
popd
|
||||
'';
|
||||
}));
|
||||
in
|
||||
lib.extendDerivation true passthru self
|
||||
|
|
@ -0,0 +1,74 @@
|
|||
{ stdenv
|
||||
, callPackage
|
||||
, python27Packages
|
||||
, installShellFiles
|
||||
, fetchFromGitHub
|
||||
, file
|
||||
, findutils
|
||||
, gettext
|
||||
, bats
|
||||
, bash
|
||||
, doCheck ? true
|
||||
}:
|
||||
let
|
||||
version = "0.4.0";
|
||||
rSrc = fetchFromGitHub {
|
||||
owner = "abathur";
|
||||
repo = "resholve";
|
||||
rev = "v${version}";
|
||||
hash = "sha256-wfxcX3wMZqoi5bWjXYRa21UDDJmTDfE+21p4mL2IJog=";
|
||||
};
|
||||
deps = callPackage ./deps.nix {
|
||||
/*
|
||||
resholve needs to patch Oil, but trying to avoid adding
|
||||
them all *to* nixpkgs, since they aren't specific to
|
||||
nix/nixpkgs.
|
||||
*/
|
||||
oilPatches = [
|
||||
"${rSrc}/0001-add_setup_py.patch"
|
||||
"${rSrc}/0002-add_MANIFEST_in.patch"
|
||||
"${rSrc}/0003-fix_codegen_shebang.patch"
|
||||
"${rSrc}/0004-disable-internal-py-yajl-for-nix-built.patch"
|
||||
];
|
||||
};
|
||||
in
|
||||
python27Packages.buildPythonApplication {
|
||||
pname = "resholve";
|
||||
inherit version;
|
||||
src = rSrc;
|
||||
format = "other";
|
||||
|
||||
nativeBuildInputs = [ installShellFiles ];
|
||||
|
||||
propagatedBuildInputs = [ deps.oildev python27Packages.ConfigArgParse ];
|
||||
|
||||
patchPhase = ''
|
||||
for file in resholve; do
|
||||
substituteInPlace $file --subst-var-by version ${version}
|
||||
done
|
||||
'';
|
||||
|
||||
installPhase = ''
|
||||
install -Dm755 resholve $out/bin/resholve
|
||||
installManPage resholve.1
|
||||
'';
|
||||
|
||||
inherit doCheck;
|
||||
checkInputs = [ bats ];
|
||||
RESHOLVE_PATH = "${stdenv.lib.makeBinPath [ file findutils gettext ]}";
|
||||
|
||||
checkPhase = ''
|
||||
# explicit interpreter for test suite
|
||||
export INTERP="${bash}/bin/bash" PATH="$out/bin:$PATH"
|
||||
patchShebangs .
|
||||
./test.sh
|
||||
'';
|
||||
|
||||
meta = with stdenv.lib; {
|
||||
description = "Resolve external shell-script dependencies";
|
||||
homepage = "https://github.com/abathur/resholve";
|
||||
license = with licenses; [ mit ];
|
||||
maintainers = with maintainers; [ abathur ];
|
||||
platforms = platforms.all;
|
||||
};
|
||||
}
|
||||
|
|
@ -7099,6 +7099,9 @@ in
|
|||
|
||||
rescuetime = libsForQt5.callPackage ../applications/misc/rescuetime { };
|
||||
|
||||
inherit (callPackage ../development/misc/resholve { })
|
||||
resholve resholvePackage;
|
||||
|
||||
reuse = callPackage ../tools/package-management/reuse { };
|
||||
|
||||
rewritefs = callPackage ../os-specific/linux/rewritefs { };
|
||||
|
|
|
|||
Loading…
Reference in New Issue