prefetch-yarn-deps, fetchYarnDeps: init

pkgs/build-support/node/fetch-yarn-deps/default.nix

@@ -0,0 +1,74 @@
+{ stdenv, lib, makeWrapper, coreutils, nix-prefetch-git, fetchurl, nodejs-slim, prefetch-yarn-deps, cacert, callPackage }:
+
+let
+  yarnpkg-lockfile-tar = fetchurl {
+    url = "https://registry.yarnpkg.com/@yarnpkg/lockfile/-/lockfile-1.1.0.tgz";
+    sha512 = "sha512-GpSwvyXOcOOlV70vbnzjj4fW5xW/FdUF6nQEt1ENy7m4ZCczi1+/buVUPAqmGfqznsORNFzUMjctTIp8a9tuCQ==";
+  };
+
+in {
+  prefetch-yarn-deps = stdenv.mkDerivation {
+    name = "prefetch-yarn-deps";
+
+    dontUnpack = true;
+
+    nativeBuildInputs = [ makeWrapper ];
+    buildInputs = [ coreutils nix-prefetch-git nodejs-slim ];
+
+    buildPhase = ''
+      runHook preBuild
+
+      mkdir libexec
+      tar --strip-components=1 -xf ${yarnpkg-lockfile-tar} package/index.js
+      mv index.js libexec/yarnpkg-lockfile.js
+      cp ${./index.js} libexec/index.js
+      patchShebangs libexec/index.js
+
+      runHook postBuild
+    '';
+
+    installPhase = ''
+      runHook preInstall
+
+      mkdir -p $out/bin
+      cp -r libexec $out
+      makeWrapper $out/libexec/index.js $out/bin/prefetch-yarn-deps \
+        --prefix PATH : ${lib.makeBinPath [ coreutils nix-prefetch-git ]}
+
+      runHook postInstall
+    '';
+  };
+
+  fetchYarnDeps = let
+    f = {
+      name ? "offline",
+      yarnLock,
+      hash ? "",
+      sha256 ? "",
+    }: let
+      hash_ =
+        if hash != "" then { outputHashAlgo = null; outputHash = hash; }
+        else if sha256 != "" then { outputHashAlgo = "sha256"; outputHash = sha256; }
+        else throw "fetchYarnDeps requires a hash";
+    in stdenv.mkDerivation {
+      inherit name;
+
+      dontUnpack = true;
+      dontInstall = true;
+
+      nativeBuildInputs = [ prefetch-yarn-deps ];
+      GIT_SSL_CAINFO = "${cacert}/etc/ssl/certs/ca-bundle.crt";
+
+      buildPhase = ''
+        mkdir -p $out
+        (cd $out; prefetch-yarn-deps --verbose --builder ${yarnLock})
+      '';
+
+      outputHashMode = "recursive";
+      inherit (hash_) outputHashAlgo outputHash;
+    };
+
+  in lib.setFunctionArgs f (lib.functionArgs f) // {
+    tests = callPackage ./tests {};
+  };
+}

pkgs/build-support/node/fetch-yarn-deps/index.js

@@ -0,0 +1,168 @@
+#!/usr/bin/env node
+'use strict'
+
+const fs = require('fs')
+const crypto = require('crypto')
+const process = require('process')
+const https = require('https')
+const child_process = require('child_process')
+const path = require('path')
+const lockfile = require('./yarnpkg-lockfile.js')
+const { promisify } = require('util')
+
+const execFile = promisify(child_process.execFile)
+
+const exec = async (...args) => {
+	const res = await execFile(...args)
+	if (res.error) throw new Error(res.stderr)
+	return res
+}
+
+const downloadFileHttps = (fileName, url, expectedHash) => {
+	return new Promise((resolve, reject) => {
+		https.get(url, (res) => {
+			const file = fs.createWriteStream(fileName)
+			const hash = crypto.createHash('sha1')
+			res.pipe(file)
+			res.pipe(hash).setEncoding('hex')
+			res.on('end', () => {
+				file.close()
+				const h = hash.read()
+				if (h != expectedHash) return reject(new Error(`hash mismatch, expected ${expectedHash}, got ${h}`))
+				resolve()
+			})
+                        res.on('error', e => reject(e))
+		})
+	})
+}
+
+const downloadGit = async (fileName, url, rev) => {
+	await exec('nix-prefetch-git', [
+		'--out', fileName + '.tmp',
+		'--url', url,
+		'--rev', rev,
+		'--builder'
+	])
+
+	await exec('tar', [
+		// hopefully make it reproducible across runs and systems
+		'--owner=0', '--group=0', '--numeric-owner', '--format=gnu', '--sort=name', '--mtime=@1',
+
+		// Set u+w because tar-fs can't unpack archives with read-only dirs: https://github.com/mafintosh/tar-fs/issues/79
+		'--mode', 'u+w',
+
+		'-C', fileName + '.tmp',
+		'-cf', fileName, '.'
+	])
+
+	await exec('rm', [ '-rf', fileName + '.tmp', ])
+}
+
+const downloadPkg = (pkg, verbose) => {
+	const [ url, hash ] = pkg.resolved.split('#')
+	if (verbose) console.log('downloading ' + url)
+	if (url.startsWith('https://codeload.github.com/') && url.includes('/tar.gz/')) {
+		const fileName = path.basename(url)
+		const s = url.split('/')
+		downloadGit(fileName, `https://github.com/${s[3]}/${s[4]}.git`, s[6])
+	} else if (url.startsWith('https://')) {
+		const fileName = url
+			.replace(/https:\/\/(.)*(.com)\//g, '') // prevents having long directory names
+			.replace(/[@/%:-]/g, '_') // replace @ and : and - and % characters with underscore
+
+		return downloadFileHttps(fileName, url, hash)
+	} else if (url.startsWith('git+')) {
+		const fileName = path.basename(url)
+		return downloadGit(fileName, url.replace(/^git\+/, ''), hash)
+	} else {
+		throw new Error('don\'t know how to download "' + url + '"')
+	}
+}
+
+const performParallel = tasks => {
+	const worker = async () => {
+		while (tasks.length > 0) await tasks.shift()()
+	}
+
+	const workers = []
+	for (let i = 0; i < 4; i++) {
+		workers.push(worker())
+	}
+
+	return Promise.all(workers)
+}
+
+const prefetchYarnDeps = async (lockData, verbose) => {
+	const tasks = Object.values(
+		Object.entries(lockData.object)
+		.map(([key, value]) => {
+			return { key, ...value }
+		})
+		.reduce((out, pkg) => {
+			out[pkg.resolved] = pkg
+			return out
+		}, {})
+	)
+		.map(pkg => () => downloadPkg(pkg, verbose))
+
+	await performParallel(tasks)
+	if (verbose) console.log('Done')
+}
+
+const showUsage = async () => {
+	process.stderr.write(`
+syntax: prefetch-yarn-deps [path to yarn.lock] [options]
+
+Options:
+  -h --help         Show this help
+  -v --verbose      Verbose output
+  --builder         Only perform the download to current directory, then exit
+`)
+	process.exit(1)
+}
+
+const main = async () => {
+	const args = process.argv.slice(2)
+	let next, lockFile, verbose, isBuilder
+	while (next = args.shift()) {
+		if (next == '--builder') {
+			isBuilder = true
+		} else if (next == '--verbose' || next == '-v') {
+			verbose = true
+		} else if (next == '--help' || next == '-h') {
+			showUsage()
+		} else if (!lockFile) {
+			lockFile = next
+		} else {
+			showUsage()
+		}
+	}
+	let lockContents
+	try {
+		lockContents = await fs.promises.readFile(lockFile || 'yarn.lock', 'utf-8')
+	} catch(e) {
+		showUsage()
+	}
+	const lockData = lockfile.parse(lockContents)
+
+	if (isBuilder) {
+		await prefetchYarnDeps(lockData, verbose)
+	} else {
+		const { stdout: tmpDir } = await exec('mktemp', [ '-d' ])
+
+		try {
+			process.chdir(tmpDir.trim())
+			await prefetchYarnDeps(lockData, verbose)
+			const { stdout: hash } = await exec('nix-hash', [ '--type', 'sha256', '--base32', tmpDir.trim() ])
+			console.log(hash)
+		} finally {
+			await exec('rm', [ '-rf', tmpDir.trim() ])
+		}
+	}
+}
+
+main()
+	.catch(e => {
+		console.error(e)
+		process.exit(1)
+	})

pkgs/build-support/node/fetch-yarn-deps/tests/default.nix

@@ -0,0 +1,16 @@
+{ invalidateFetcherByDrvHash, fetchYarnDeps, ... }:
+
+{
+  simple = invalidateFetcherByDrvHash fetchYarnDeps {
+    yarnLock = ./simple.lock;
+    sha256 = "sha256-Erdkw2E8wWT09jFNLXGkrdwKl0HuSZWnUDJUrV95vSE=";
+  };
+  gitDep = invalidateFetcherByDrvHash fetchYarnDeps {
+    yarnLock = ./git.lock;
+    sha256 = "sha256-lAqN4LpoE+jgsQO1nDtuORwcVEO7ogEV53jCu2jFJUI=";
+  };
+  githubDep = invalidateFetcherByDrvHash fetchYarnDeps {
+    yarnLock = ./github.lock;
+    sha256 = "sha256-Tsfgyjxz8x6gNmfN0xR7G/NQNoEs4svxRN/N+26vfJU=";
+  };
+}

pkgs/build-support/node/fetch-yarn-deps/tests/git.lock

@@ -0,0 +1,7 @@
+# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
+# yarn lockfile v1
+
+
+"async@git+https://github.com/caolan/async":
+  version "3.2.1"
+  resolved "git+https://github.com/caolan/async#fc9ba651341af5ab974aade6b1640e345912be83"

pkgs/build-support/node/fetch-yarn-deps/tests/github.lock

@@ -0,0 +1,7 @@
+# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
+# yarn lockfile v1
+
+
+"async@github:caolan/async":
+  version "3.2.1"
+  resolved "https://codeload.github.com/caolan/async/tar.gz/fc9ba651341af5ab974aade6b1640e345912be83"

pkgs/build-support/node/fetch-yarn-deps/tests/simple.lock

@@ -0,0 +1,8 @@
+# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
+# yarn lockfile v1
+
+
+lit-html@1:
+  version "1.4.1"
+  resolved "https://registry.yarnpkg.com/lit-html/-/lit-html-1.4.1.tgz#0c6f3ee4ad4eb610a49831787f0478ad8e9ae5e0"
+  integrity sha512-B9btcSgPYb1q4oSOb/PrOT6Z/H+r6xuNzfH4lFli/AWhYwdtrgQkQWBbIc6mdnf6E2IL3gDXdkkqNktpU0OZQA==

pkgs/top-level/all-packages.nix

@@ -468,6 +468,10 @@ with pkgs;
 
   fetchMavenArtifact = callPackage ../build-support/fetchmavenartifact { };
 
+  inherit (callPackage ../build-support/node/fetch-yarn-deps { })
+    prefetch-yarn-deps
+    fetchYarnDeps;
+
   find-cursor = callPackage ../tools/X11/find-cursor { };
 
   flare-floss = callPackage ../tools/security/flare-floss { };