diff options
| author | Maxim Cournoyer <maxim.cournoyer@gmail.com> | 2021-04-08 15:47:00 -0400 |
|---|---|---|
| committer | Maxim Cournoyer <maxim.cournoyer@gmail.com> | 2021-04-08 15:47:00 -0400 |
| commit | 533a893cc6b03f100566760d6e0c8e0500ed7082 (patch) | |
| tree | 0ecbf89895a400c43d16bb1f6539f1a88aeaa4cf /guix | |
| parent | 6a2546f92d07df04e5d700924edd027ed1e2ef11 (diff) | |
| parent | c762df54786fd6f005f3b5307323f1d2df3cbf0b (diff) | |
Merge branch 'master' into staging
Conflicts:
.guix-authorizations
gnu/ci.scm
gnu/packages/base.scm
gnu/packages/databases.scm
gnu/packages/gnome.scm
gnu/packages/gtk.scm
gnu/packages/imagemagick.scm
gnu/packages/password-utils.scm
gnu/packages/python-xyz.scm
gnu/packages/web-browsers.scm
Diffstat (limited to 'guix')
67 files changed, 2639 insertions, 1169 deletions
diff --git a/guix/build-system/cargo.scm b/guix/build-system/cargo.scm index 6c8edf6bac..e53d2a7523 100644 --- a/guix/build-system/cargo.scm +++ b/guix/build-system/cargo.scm @@ -5,6 +5,7 @@ ;;; Copyright © 2016 David Craven <david@craven.ch> ;;; Copyright © 2019 Ivan Petkov <ivanppetkov@gmail.com> ;;; Copyright © 2020 Jakub KÄ…dzioÅ‚ka <kuba@kadziolka.net> +;;; Copyright © 2021 Efraim Flashner <efraim@flashner.co.il> ;;; ;;; This file is part of GNU Guix. ;;; @@ -77,8 +78,10 @@ to NAME and VERSION." (vendor-dir "guix-vendor") (cargo-build-flags ''("--release")) (cargo-test-flags ''("--release")) + (cargo-package-flags ''("--no-metadata" "--no-verify")) (features ''()) (skip-build? #f) + (install-source? #t) (phases '(@ (guix build cargo-build-system) %standard-phases)) (outputs '("out")) @@ -106,8 +109,10 @@ to NAME and VERSION." #:vendor-dir ,vendor-dir #:cargo-build-flags ,cargo-build-flags #:cargo-test-flags ,cargo-test-flags + #:cargo-package-flags ,cargo-package-flags #:features ,features #:skip-build? ,skip-build? + #:install-source? ,install-source? #:tests? ,(and tests? (not skip-build?)) #:phases ,phases #:outputs %outputs diff --git a/guix/build-system/go.scm b/guix/build-system/go.scm index f8ebaefb27..0e2c1cd2ee 100644 --- a/guix/build-system/go.scm +++ b/guix/build-system/go.scm @@ -26,9 +26,12 @@ #:use-module (guix build-system gnu) #:use-module (guix packages) #:use-module (ice-9 match) + #:use-module (ice-9 regex) #:export (%go-build-system-modules go-build - go-build-system)) + go-build-system + + go-version->git-ref)) ;; Commentary: ;; @@ -37,6 +40,36 @@ ;; ;; Code: +(define %go-version-rx + (make-regexp (string-append + "(v?[0-9]\\.[0-9]\\.[0-9])" ;"v" prefix can be omitted in version prefix + "(-|-pre\\.0\\.|-0\\.)" ;separator + "([0-9]{14})-" ;timestamp + "([0-9A-Fa-f]{12})"))) ;commit hash + +(define (go-version->git-ref version) + "Parse VERSION, a \"pseudo-version\" as defined at +<https://golang.org/ref/mod#pseudo-versions>, and extract the commit hash from +it, defaulting to full VERSION if a pseudo-version pattern is not recognized." + ;; A module version like v1.2.3 is introduced by tagging a revision in the + ;; underlying source repository. Untagged revisions can be referred to + ;; using a "pseudo-version" like v0.0.0-yyyymmddhhmmss-abcdefabcdef, where + ;; the time is the commit time in UTC and the final suffix is the prefix of + ;; the commit hash (see: https://golang.org/ref/mod#pseudo-versions). + (let* ((version + ;; If a source code repository has a v2.0.0 or later tag for a file + ;; tree with no go.mod, the version is considered to be part of the + ;; v1 module's available versions and is given an +incompatible + ;; suffix + ;; (see:https://golang.org/cmd/go/#hdr-Module_compatibility_and_semantic_versioning). + (if (string-suffix? "+incompatible" version) + (string-drop-right version 13) + version)) + (match (regexp-exec %go-version-rx version))) + (if match + (match:substring match 4) + version))) + (define %go-build-system-modules ;; Build-side modules imported and used by default. `((guix build go-build-system) diff --git a/guix/build-system/node.scm b/guix/build-system/node.scm index 05c24c47d5..98f63f87ef 100644 --- a/guix/build-system/node.scm +++ b/guix/build-system/node.scm @@ -1,5 +1,6 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2016 Jelle Licht <jlicht@fsfe.org> +;;; Copyright © 2019 Timothy Sample <samplet@ngyro.com> ;;; ;;; This file is part of GNU Guix. ;;; @@ -17,9 +18,6 @@ ;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>. (define-module (guix build-system node) - #:use-module (guix store) - #:use-module (guix build json) - #:use-module (guix build union) #:use-module (guix utils) #:use-module (guix packages) #:use-module (guix derivations) @@ -27,28 +25,21 @@ #:use-module (guix build-system) #:use-module (guix build-system gnu) #:use-module (ice-9 match) - #:export (npm-meta-uri - %node-build-system-modules + #:export (%node-build-system-modules node-build node-build-system)) -(define (npm-meta-uri name) - "Return a URI string for the metadata of node module NAME found in the npm -registry." - (string-append "https://registry.npmjs.org/" name)) - (define %node-build-system-modules ;; Build-side modules imported by default. `((guix build node-build-system) (guix build json) - (guix build union) - ,@%gnu-build-system-modules)) ;; TODO: Might be not needed + ,@%gnu-build-system-modules)) (define (default-node) "Return the default Node package." ;; Lazily resolve the binding to avoid a circular dependency. (let ((node (resolve-interface '(gnu packages node)))) - (module-ref node 'node))) + (module-ref node 'node-lts))) (define* (lower name #:key source inputs native-inputs outputs system target @@ -78,7 +69,7 @@ registry." (define* (node-build store name inputs #:key - (npm-flags ''()) + (test-target "test") (tests? #t) (phases '(@ (guix build node-build-system) %standard-phases)) @@ -88,8 +79,6 @@ registry." (guile #f) (imported-modules %node-build-system-modules) (modules '((guix build node-build-system) - (guix build json) - (guix build union) (guix build utils)))) "Build SOURCE using NODE and INPUTS." (define builder @@ -99,12 +88,10 @@ registry." #:source ,(match (assoc-ref inputs "source") (((? derivation? source)) (derivation->output-path source)) - ((source) - source) - (source - source)) + ((source) source) + (source source)) #:system ,system - #:npm-flags ,npm-flags + #:test-target ,test-target #:tests? ,tests? #:phases ,phases #:outputs %outputs @@ -131,5 +118,5 @@ registry." (define node-build-system (build-system (name 'node) - (description "The standard Node build system") + (description "The Node build system") (lower lower))) diff --git a/guix/build-system/python.scm b/guix/build-system/python.scm index e39c06528e..80895162f8 100644 --- a/guix/build-system/python.scm +++ b/guix/build-system/python.scm @@ -98,8 +98,7 @@ pre-defined variants." ;; Otherwise build the new package object graph. ((eq? (package-build-system p) python-build-system) - (package - (inherit p) + (package/inherit p (location (package-location p)) (name (let ((name (package-name p))) (string-append new-prefix @@ -131,8 +130,7 @@ pre-defined variants." (define (strip-python2-variant p) "Remove the 'python2-variant' property from P." - (package - (inherit p) + (package/inherit p (properties (alist-delete 'python2-variant (package-properties p))))) (define* (lower name diff --git a/guix/build/cargo-build-system.scm b/guix/build/cargo-build-system.scm index 1d21b33895..0a95672b00 100644 --- a/guix/build/cargo-build-system.scm +++ b/guix/build/cargo-build-system.scm @@ -2,7 +2,7 @@ ;;; Copyright © 2016 David Craven <david@craven.ch> ;;; Copyright © 2017 Mathieu Othacehe <m.othacehe@gmail.com> ;;; Copyright © 2019 Ivan Petkov <ivanppetkov@gmail.com> -;;; Copyright © 2019, 2020 Efraim Flashner <efraim@flashner.co.il> +;;; Copyright © 2019, 2020, 2021 Efraim Flashner <efraim@flashner.co.il> ;;; Copyright © 2020 Jakub KÄ…dzioÅ‚ka <kuba@kadziolka.net> ;;; Copyright © 2020 Marius Bakke <marius@gnu.org> ;;; @@ -73,6 +73,44 @@ Cargo.toml file present at its root." " | cut -d/ -f2" " | grep -q '^Cargo.toml$'"))))) +(define* (unpack-rust-crates #:key inputs vendor-dir #:allow-other-keys) + (define (inputs->rust-inputs inputs) + "Filter using the label part from INPUTS." + (filter (lambda (input) + (match input + ((name . _) (rust-package? name)))) + inputs)) + (define (inputs->directories inputs) + "Extract the directory part from INPUTS." + (match inputs + (((names . directories) ...) + directories))) + + (let ((rust-inputs (inputs->directories (inputs->rust-inputs inputs)))) + (unless (null? rust-inputs) + (mkdir-p "target/package") + (mkdir-p vendor-dir) + ;; TODO: copy only regular inputs to target/package, not native-inputs. + (for-each + (lambda (input-crate) + (for-each + (lambda (packaged-crate) + (unless + (file-exists? + (string-append "target/package/" (basename packaged-crate))) + (install-file packaged-crate "target/package/"))) + (find-files + (string-append input-crate "/share/cargo/registry") "\\.crate$"))) + (delete-duplicates rust-inputs)) + + (for-each (lambda (crate) + (invoke "tar" "xzf" crate "-C" vendor-dir)) + (find-files "target/package" "\\.crate$")))) + #t) + +(define (rust-package? name) + (string-prefix? "rust-" name)) + (define* (configure #:key inputs (vendor-dir "guix-vendor") #:allow-other-keys) @@ -170,9 +208,27 @@ directory = '" port) (apply invoke "cargo" "test" cargo-test-flags) #t)) -(define* (install #:key inputs outputs skip-build? features #:allow-other-keys) +(define* (package #:key + install-source? + (cargo-package-flags '("--no-metadata" "--no-verify")) + #:allow-other-keys) + "Run 'cargo-package' for a given Cargo package." + (if install-source? + (apply invoke `("cargo" "package" ,@cargo-package-flags)) + (format #t "Not installing cargo sources, skipping `cargo package`.~%")) + #t) + +(define* (install #:key + inputs + outputs + skip-build? + install-source? + features + #:allow-other-keys) "Install a given Cargo package." - (let* ((out (assoc-ref outputs "out"))) + (let* ((out (assoc-ref outputs "out")) + (registry (string-append out "/share/cargo/registry")) + (sources (string-append out "/share/cargo/src"))) (mkdir-p out) ;; Make cargo reuse all the artifacts we just built instead @@ -186,6 +242,18 @@ directory = '" port) (invoke "cargo" "install" "--no-track" "--path" "." "--root" out "--features" (string-join features))) + (when install-source? + ;; Install crate tarballs and unpacked sources for later use. + ;; TODO: Is there a better format/directory for these files? + (mkdir-p sources) + (for-each (lambda (crate) + (install-file crate registry)) + (find-files "target/package" "\\.crate$")) + + (for-each (lambda (crate) + (invoke "tar" "xzf" crate "-C" sources)) + (find-files registry "\\.crate$"))) + #t)) (define %standard-phases @@ -195,6 +263,8 @@ directory = '" port) (replace 'build build) (replace 'check check) (replace 'install install) + (add-after 'build 'package package) + (add-after 'unpack 'unpack-rust-crates unpack-rust-crates) (add-after 'patch-generated-file-shebangs 'patch-cargo-checksums patch-cargo-checksums))) (define* (cargo-build #:key inputs (phases %standard-phases) diff --git a/guix/build/download.scm b/guix/build/download.scm index 46af149b2f..a22d4064ca 100644 --- a/guix/build/download.scm +++ b/guix/build/download.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021 Ludovic Courtès <ludo@gnu.org> ;;; Copyright © 2015 Mark H Weaver <mhw@netris.org> ;;; Copyright © 2017 Tobias Geerinckx-Rice <me@tobias.gr> ;;; @@ -28,7 +28,6 @@ #:use-module (guix build utils) #:use-module (guix progress) #:use-module (rnrs io ports) - #:use-module ((ice-9 binary-ports) #:select (unget-bytevector)) #:use-module (rnrs bytevectors) #:use-module (srfi srfi-1) #:use-module (srfi srfi-11) @@ -306,14 +305,22 @@ host name without trailing dot." (let ((record (session-record-port session))) (define (read! bv start count) - (define read-bv (get-bytevector-some record)) - (if (eof-object? read-bv) - 0 ; read! returns 0 on eof-object - (let ((read-bv-len (bytevector-length read-bv))) - (bytevector-copy! read-bv 0 bv start (min read-bv-len count)) - (when (< count read-bv-len) - (unget-bytevector record bv count (- read-bv-len count))) - read-bv-len))) + (define read + (catch 'gnutls-error + (lambda () + (get-bytevector-n! record bv start count)) + (lambda (key err proc . rest) + ;; When responding to "Connection: close" requests, some + ;; servers close the connection abruptly after sending the + ;; response body, without doing a proper TLS connection + ;; termination. Treat it as EOF. + (if (eq? err error/premature-termination) + the-eof-object + (apply throw key err proc rest))))) + + (if (eof-object? read) + 0 + read)) (define (write! bv start count) (put-bytevector record bv start count) (force-output record) @@ -328,17 +335,24 @@ host name without trailing dot." (unless (port-closed? record) (close-port record))) + (define (unbuffered port) + (setvbuf port 'none) + port) + (setvbuf record 'block) ;; Return a port that wraps RECORD to ensure that closing it also ;; closes PORT, the actual socket port, and its file descriptor. + ;; Make sure it does not introduce extra buffering (custom ports + ;; are buffered by default as of Guile 3.0.5). ;; XXX: This wrapper would be unnecessary if GnuTLS could ;; automatically close SESSION's file descriptor when RECORD is ;; closed, but that doesn't seem to be possible currently (as of ;; 3.6.9). - (make-custom-binary-input/output-port "gnutls wrapped port" read! write! - get-position set-position! - close)))) + (unbuffered + (make-custom-binary-input/output-port "gnutls wrapped port" read! write! + get-position set-position! + close))))) (define (ensure-uri uri-or-string) ;XXX: copied from (web http) (cond diff --git a/guix/build/julia-build-system.scm b/guix/build/julia-build-system.scm index 8f57045a8c..d74acf2a05 100644 --- a/guix/build/julia-build-system.scm +++ b/guix/build/julia-build-system.scm @@ -101,7 +101,7 @@ Project.toml)." (or (getenv "JULIA_LOAD_PATH") ""))) (setenv "HOME" "/tmp") - (invoke "julia" + (invoke "julia" "--depwarn=yes" (string-append builddir "packages/" package "/test/runtests.jl")))) #t) diff --git a/guix/build/node-build-system.scm b/guix/build/node-build-system.scm index 7799f03595..a55cab237c 100644 --- a/guix/build/node-build-system.scm +++ b/guix/build/node-build-system.scm @@ -1,6 +1,7 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2015 David Thompson <davet@gnu.org> -;;; Copyright © 2016 Jelle Licht <jlicht@fsfe.org> +;;; Copyright © 2016, 2020 Jelle Licht <jlicht@fsfe.org> +;;; Copyright © 2019, 2021 Timothy Sample <samplet@ngyro.com> ;;; ;;; This file is part of GNU Guix. ;;; @@ -19,144 +20,130 @@ (define-module (guix build node-build-system) #:use-module ((guix build gnu-build-system) #:prefix gnu:) - #:use-module (guix build json) - #:use-module (guix build union) #:use-module (guix build utils) + #:use-module (guix build json) + #:use-module (ice-9 ftw) #:use-module (ice-9 match) - #:use-module (ice-9 popen) - #:use-module (ice-9 regex) #:use-module (srfi srfi-1) - #:use-module (srfi srfi-26) #:export (%standard-phases node-build)) ;; Commentary: ;; -;; Builder-side code of the standard Node/npm package build procedure. +;; Builder-side code of the standard Node/NPM package install procedure. ;; ;; Code: -(define* (read-package-data #:key (filename "package.json")) - (call-with-input-file filename - (lambda (port) - (read-json port)))) +(define (set-home . _) + (with-directory-excursion ".." + (let loop ((i 0)) + (let ((dir (string-append "npm-home-" (number->string i)))) + (if (directory-exists? dir) + (loop (1+ i)) + (begin + (mkdir dir) + (setenv "HOME" (string-append (getcwd) "/" dir)) + (format #t "set HOME to ~s~%" (getenv "HOME"))))))) + #t) -(define* (build #:key inputs #:allow-other-keys) - (define (build-from-package-json? package-file) - (let* ((package-data (read-package-data #:filename package-file)) - (scripts (assoc-ref package-data "scripts"))) - (assoc-ref scripts "build"))) - "Build a new node module using the appropriate build system." - ;; XXX: Develop a more robust heuristic, allow override - (cond ((file-exists? "gulpfile.js") - (invoke "gulp")) - ((file-exists? "gruntfile.js") - (invoke "grunt")) - ((file-exists? "Makefile") - (invoke "make")) - ((and (file-exists? "package.json") - (build-from-package-json? "package.json")) - (invoke "npm" "run" "build"))) +(define (module-name module) + (let* ((package.json (string-append module "/package.json")) + (package-meta (call-with-input-file package.json read-json))) + (assoc-ref package-meta "name"))) + +(define (index-modules input-paths) + (define (list-modules directory) + (append-map (lambda (x) + (if (string-prefix? "@" x) + (list-modules (string-append directory "/" x)) + (list (string-append directory "/" x)))) + (filter (lambda (x) + (not (member x '("." "..")))) + (or (scandir directory) '())))) + (let ((index (make-hash-table (* 2 (length input-paths))))) + (for-each (lambda (dir) + (let ((nm (string-append dir "/lib/node_modules"))) + (for-each (lambda (module) + (hash-set! index (module-name module) module)) + (list-modules nm)))) + input-paths) + index)) + +(define* (patch-dependencies #:key inputs #:allow-other-keys) + + (define index (index-modules (map cdr inputs))) + + (define (resolve-dependencies package-meta meta-key) + (fold (lambda (key+value acc) + (match key+value + ('@ acc) + ((key . value) (acons key (hash-ref index key value) acc)))) + '() + (or (assoc-ref package-meta meta-key) '()))) + + (with-atomic-file-replacement "package.json" + (lambda (in out) + (let ((package-meta (read-json in))) + (assoc-set! package-meta "dependencies" + (append + '(@) + (resolve-dependencies package-meta "dependencies") + (resolve-dependencies package-meta "peerDependencies"))) + (assoc-set! package-meta "devDependencies" + (append + '(@) + (resolve-dependencies package-meta "devDependencies"))) + (write-json package-meta out)))) #t) -(define* (link-npm-dependencies #:key inputs #:allow-other-keys) - (define (inputs->node-inputs inputs) - "Filter the directory part from INPUTS." - (filter (lambda (input) - (match input - ((name . _) (node-package? name)))) - inputs)) - (define (inputs->directories inputs) - "Extract the directory part from INPUTS." - (match inputs - (((names . directories) ...) - directories))) - (define (make-node-path root) - (string-append root "/lib/node_modules/")) - - (let ((input-node-directories (inputs->directories - (inputs->node-inputs inputs)))) - (union-build "node_modules" - (map make-node-path input-node-directories)) +(define* (configure #:key outputs inputs #:allow-other-keys) + (let ((npm (string-append (assoc-ref inputs "node") "/bin/npm"))) + (invoke npm "--offline" "--ignore-scripts" "install") #t)) -(define configure link-npm-dependencies) +(define* (build #:key inputs #:allow-other-keys) + (let ((package-meta (call-with-input-file "package.json" read-json))) + (if (and=> (assoc-ref package-meta "scripts") + (lambda (scripts) + (assoc-ref scripts "build"))) + (let ((npm (string-append (assoc-ref inputs "node") "/bin/npm"))) + (invoke npm "run" "build")) + (format #t "there is no build script to run~%")) + #t)) -(define* (check #:key tests? #:allow-other-keys) +(define* (check #:key tests? inputs #:allow-other-keys) "Run 'npm test' if TESTS?" (if tests? - ;; Should only be enabled once we know that there are tests - (invoke "npm" "test")) + (let ((npm (string-append (assoc-ref inputs "node") "/bin/npm"))) + (invoke npm "test")) + (format #t "test suite not run~%")) #t) -(define (node-package? name) - "Check if NAME correspond to the name of an Node package." - (string-prefix? "node-" name)) +(define* (repack #:key inputs #:allow-other-keys) + (invoke "tar" "-czf" "../package.tgz" ".") + #t) (define* (install #:key outputs inputs #:allow-other-keys) - "Install the node module to the output store item. The module itself is -installed in a subdirectory of @file{node_modules} and its runtime dependencies -as defined by @file{package.json} are symlinked into a @file{node_modules} -subdirectory of the module's directory. Additionally, binaries are installed in -the @file{bin} directory." - (let* ((out (assoc-ref outputs "out")) - (target (string-append out "/lib")) - (binaries (string-append out "/bin")) - (data (read-package-data)) - (modulename (assoc-ref data "name")) - (binary-configuration (match (assoc-ref data "bin") - (('@ configuration ...) configuration) - ((? string? configuration) configuration) - (#f #f))) - (dependencies (match (assoc-ref data "dependencies") - (('@ deps ...) deps) - (#f #f)))) - (mkdir-p target) - (copy-recursively "." (string-append target "/node_modules/" modulename)) - ;; Remove references to dependencies - (delete-file-recursively - (string-append target "/node_modules/" modulename "/node_modules")) - (cond - ((string? binary-configuration) - (begin - (mkdir-p binaries) - (symlink (string-append target "/node_modules/" modulename "/" - binary-configuration) - (string-append binaries "/" modulename)))) - ((list? binary-configuration) - (for-each - (lambda (conf) - (match conf - ((key . value) - (begin - (mkdir-p (dirname (string-append binaries "/" key))) - (symlink (string-append target "/node_modules/" modulename "/" - value) - (string-append binaries "/" key)))))) - binary-configuration))) - (when dependencies - (mkdir-p - (string-append target "/node_modules/" modulename "/node_modules")) - (for-each - (lambda (dependency) - (let ((dependency (car dependency))) - (symlink - (string-append (assoc-ref inputs (string-append "node-" dependency)) - "/lib/node_modules/" dependency) - (string-append target "/node_modules/" modulename - "/node_modules/" dependency)))) - dependencies)) + "Install the node module to the output store item." + (let ((out (assoc-ref outputs "out")) + (npm (string-append (assoc-ref inputs "node") "/bin/npm"))) + (invoke npm "--prefix" out + "--global" + "--offline" + "--loglevel" "info" + "--production" + "install" "../package.tgz") #t)) - (define %standard-phases (modify-phases gnu:%standard-phases + (add-after 'unpack 'set-home set-home) + (add-before 'configure 'patch-dependencies patch-dependencies) (replace 'configure configure) (replace 'build build) - (replace 'install install) - (delete 'check) - (add-after 'install 'check check) - (delete 'strip))) + (replace 'check check) + (add-before 'install 'repack repack) + (replace 'install install))) (define* (node-build #:key inputs (phases %standard-phases) #:allow-other-keys #:rest args) diff --git a/guix/build/profiles.scm b/guix/build/profiles.scm index b42f498a80..a40c3f96de 100644 --- a/guix/build/profiles.scm +++ b/guix/build/profiles.scm @@ -170,8 +170,8 @@ SEARCH-PATHS." (display "\ ;; This file was automatically generated and is for internal use only. ;; It cannot be passed to the '--manifest' option. -;; Run 'guix package --export-manifest' if to export a file suitable -;; for '--manifest'.\n\n" +;; Run 'guix package --export-manifest' if you want to export a file +;; suitable for '--manifest'.\n\n" p) (pretty-print manifest p))) diff --git a/guix/build/renpy-build-system.scm b/guix/build/renpy-build-system.scm index 464fc97b13..66683971c5 100644 --- a/guix/build/renpy-build-system.scm +++ b/guix/build/renpy-build-system.scm @@ -57,7 +57,7 @@ (delete-file (string-append data "/renpy-build.json")) (call-with-output-file launcher (lambda (port) - (format port "#!~a~%~a ~a \"$@\"" + (format port "#!~a~%~a ~s \"$@\"" (which "bash") (which "renpy") data))) @@ -77,8 +77,9 @@ (string-append out "/share/applications/" executable-name ".desktop") #:name (assoc-ref json-dump "name") #:generic-name (assoc-ref build "display_name") - #:exec (string-append (which "renpy") " " - out "/share/renpy/" directory-name) + #:exec (format #f "~a ~s" + (which "renpy") + (string-append out "/share/renpy/" directory-name)) #:categories '("Game" "Visual Novel"))) #t) diff --git a/guix/build/syscalls.scm b/guix/build/syscalls.scm index 85c1c45f81..8886fc0fb9 100644 --- a/guix/build/syscalls.scm +++ b/guix/build/syscalls.scm @@ -1,11 +1,12 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2014, 2015, 2016, 2017, 2018, 2019, 2020 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021 Ludovic Courtès <ludo@gnu.org> ;;; Copyright © 2015 David Thompson <davet@gnu.org> ;;; Copyright © 2015 Mark H Weaver <mhw@netris.org> ;;; Copyright © 2017 Mathieu Othacehe <m.othacehe@gmail.com> ;;; Copyright © 2019 Guillaume Le Vaillant <glv@posteo.net> ;;; Copyright © 2020 Julien Lepiller <julien@lepiller.eu> ;;; Copyright © 2020 Jan (janneke) Nieuwenhuizen <janneke@gnu.org> +;;; Copyright © 2021 Chris Marusich <cmmarusich@gmail.com> ;;; ;;; This file is part of GNU Guix. ;;; @@ -43,9 +44,10 @@ MS_NOEXEC MS_REMOUNT MS_NOATIME + MS_STRICTATIME + MS_RELATIME MS_BIND MS_MOVE - MS_STRICTATIME MS_LAZYTIME MNT_FORCE MNT_DETACH @@ -53,7 +55,18 @@ UMOUNT_NOFOLLOW restart-on-EINTR + + mount? + mount-device-number + mount-source + mount-point + mount-type + mount-options + mount-flags + + mounts mount-points + swapon swapoff @@ -70,6 +83,21 @@ file-system-fragment-size file-system-mount-flags statfs + + ST_RDONLY + ST_NOSUID + ST_NODEV + ST_NOEXEC + ST_SYNCHRONOUS + ST_MANDLOCK + ST_WRITE + ST_APPEND + ST_IMMUTABLE + ST_NOATIME + ST_NODIRATIME + ST_RELATIME + statfs-flags->mount-flags + free-disk-space device-in-use? add-to-entropy-count @@ -466,6 +494,7 @@ the returned procedure is called." (define MS_NOATIME 1024) (define MS_BIND 4096) (define MS_MOVE 8192) +(define MS_RELATIME 2097152) (define MS_STRICTATIME 16777216) (define MS_LAZYTIME 33554432) @@ -519,17 +548,107 @@ constants from <sys/mount.h>." (when update-mtab? (remove-from-mtab target))))) -(define (mount-points) - "Return the mounts points for currently mounted file systems." - (call-with-input-file "/proc/mounts" +;; Mount point information. +(define-record-type <mount> + (%mount source point devno type options) + mount? + (devno mount-device-number) ;st_dev + (source mount-source) ;string + (point mount-point) ;string + (type mount-type) ;string + (options mount-options)) ;string + +(define (option-string->mount-flags str) + "Parse the \"option string\" STR as it appears in /proc/mounts and similar, +and return two values: a mount bitmask (inclusive or of MS_* constants), and +the remaining unprocessed options." + ;; Why do we need to do this? Because mount flags and mount options are + ;; often lumped together; this is the case in /proc/mounts & co., so we need + ;; to extract the bits that actually correspond to mount flags. + + (define not-comma + (char-set-complement (char-set #\,))) + + (define lst + (string-tokenize str not-comma)) + + (let loop ((options lst) + (mask 0) + (remainder '())) + (match options + (() + (values mask (string-concatenate-reverse remainder))) + ((head . tail) + (letrec-syntax ((match-options (syntax-rules (=>) + ((_) + (loop tail mask + (cons head remainder))) + ((_ (str => bit) rest ...) + (if (string=? str head) + (loop tail (logior bit mask) + remainder) + (match-options rest ...)))))) + (match-options ("rw" => 0) + ("ro" => MS_RDONLY) + ("nosuid" => MS_NOSUID) + ("nodev" => MS_NODEV) + ("noexec" => MS_NOEXEC) + ("relatime" => MS_RELATIME) + ("noatime" => MS_NOATIME))))))) + +(define (mount-flags mount) + "Return the mount flags of MOUNT, a <mount> record, as an inclusive or of +MS_* constants." + (option-string->mount-flags (mount-options mount))) + +(define (octal-decode str) + "Decode octal escapes from STR and return the corresponding string. STR may +look like this: \"white\\040space\", which is decoded as \"white space\"." + (define char-set:octal + (char-set #\0 #\1 #\2 #\3 #\4 #\5 #\6 #\7)) + (define (octal? c) + (char-set-contains? char-set:octal c)) + + (let loop ((chars (string->list str)) + (result '())) + (match chars + (() + (list->string (reverse result))) + ((#\\ (? octal? a) (? octal? b) (? octal? c) . rest) + (loop rest + (cons (integer->char + (string->number (list->string (list a b c)) 8)) + result))) + ((head . tail) + (loop tail (cons head result)))))) + +(define (mounts) + "Return the list of mounts (<mount> records) visible in the namespace of the +current process." + (define (string->device-number str) + (match (string-split str #\:) + (((= string->number major) (= string->number minor)) + (+ (* major 256) minor)))) + + (call-with-input-file "/proc/self/mountinfo" (lambda (port) (let loop ((result '())) (let ((line (read-line port))) (if (eof-object? line) (reverse result) (match (string-tokenize line) - ((source mount-point _ ...) - (loop (cons mount-point result)))))))))) + ;; See the proc(5) man page for a description of the columns. + ((id parent-id major:minor root mount-point + options _ ... "-" type source _) + (let ((devno (string->device-number major:minor))) + (loop (cons (%mount (octal-decode source) + (octal-decode mount-point) + devno type options) + result))))))))))) + +(define (mount-points) + "Return the mounts points for currently mounted file systems." + (map mount-point (mounts))) (define swapon (let ((proc (syscall->procedure int "swapon" (list '* int)))) @@ -652,6 +771,56 @@ fdatasync(2) on the underlying file descriptor." (define-syntax fsword ;fsword_t (identifier-syntax long)) +(define linux? (string-contains %host-type "linux-gnu")) + +(define-syntax define-statfs-flags + (syntax-rules (linux hurd) + "Define the statfs mount flags." + ((_ (name (linux linux-value) (hurd hurd-value)) rest ...) + (begin + (define name + (if linux? linux-value hurd-value)) + (define-statfs-flags rest ...))) + ((_ (name value) rest ...) + (begin + (define name value) + (define-statfs-flags rest ...))) + ((_) #t))) + +(define-statfs-flags ;<bits/statfs.h> + (ST_RDONLY 1) + (ST_NOSUID 2) + (ST_NODEV (linux 4) (hurd 0)) + (ST_NOEXEC 8) + (ST_SYNCHRONOUS 16) + (ST_MANDLOCK (linux 64) (hurd 0)) + (ST_WRITE (linux 128) (hurd 0)) + (ST_APPEND (linux 256) (hurd 0)) + (ST_IMMUTABLE (linux 512) (hurd 0)) + (ST_NOATIME (linux 1024) (hurd 32)) + (ST_NODIRATIME (linux 2048) (hurd 0)) + (ST_RELATIME (linux 4096) (hurd 64))) + +(define (statfs-flags->mount-flags flags) + "Convert FLAGS, a logical or of ST_* constants as returned by +'file-system-mount-flags', to the corresponding logical or of MS_* constants." + (letrec-syntax ((match-flags (syntax-rules (=>) + ((_ (statfs => mount) rest ...) + (logior (if (zero? (logand flags statfs)) + 0 + mount) + (match-flags rest ...))) + ((_) + 0)))) + (match-flags + (ST_RDONLY => MS_RDONLY) + (ST_NOSUID => MS_NOSUID) + (ST_NODEV => MS_NODEV) + (ST_NOEXEC => MS_NOEXEC) + (ST_NOATIME => MS_NOATIME) + (ST_NODIRATIME => 0) ;FIXME + (ST_RELATIME => MS_RELATIME)))) + (define-c-struct %statfs ;<bits/statfs.h> sizeof-statfs ;slightly overestimated file-system @@ -667,7 +836,7 @@ fdatasync(2) on the underlying file descriptor." (identifier (array int 2)) (name-length fsword) (fragment-size fsword) - (mount-flags fsword) + (mount-flags fsword) ;ST_* (spare (array fsword 4))) (define statfs @@ -774,7 +943,11 @@ backend device." ;;; ;; From <uapi/linux/random.h>. -(define RNDADDTOENTCNT #x40045201) +(define RNDADDTOENTCNT + ;; Avoid using %current-system here to avoid depending on host-side code. + (if (string-prefix? "powerpc64le" %host-type) + #x80045201 + #x40045201)) (define (add-to-entropy-count port-or-fd n) "Add N to the kernel's entropy count (the value that can be read from @@ -853,6 +1026,7 @@ Turning finalization off shuts down the finalization thread as a side effect." ("mips64" 5055) ("armv7l" 120) ("aarch64" 220) + ("ppc64le" 120) (_ #f)))) (lambda (flags) "Create a new child process by duplicating the current parent process. diff --git a/guix/channels.scm b/guix/channels.scm index 05226e766b..b812c1b6e5 100644 --- a/guix/channels.scm +++ b/guix/channels.scm @@ -86,6 +86,7 @@ latest-channel-instances checkout->channel-instance latest-channel-derivation + channel-instance->sexp channel-instances->manifest %channel-profile-hooks channel-instances->derivation @@ -948,6 +949,7 @@ does not have the expected structure." (#f name) (('name name) name))) (url url) + (branch branch) (commit commit) (introduction (match (assq 'introduction rest) diff --git a/guix/describe.scm b/guix/describe.scm index 6a31c707f0..0683ad8a27 100644 --- a/guix/describe.scm +++ b/guix/describe.scm @@ -23,7 +23,9 @@ #:use-module ((guix utils) #:select (location-file)) #:use-module ((guix store) #:select (%store-prefix store-path?)) #:use-module ((guix config) #:select (%state-directory)) - #:autoload (guix channels) (sexp->channel manifest-entry-channel) + #:autoload (guix channels) (channel-name + sexp->channel + manifest-entry-channel) #:use-module (srfi srfi-1) #:use-module (ice-9 match) #:export (current-profile @@ -33,6 +35,7 @@ package-path-entries package-provenance + package-channels manifest-entry-with-provenance manifest-entry-provenance)) @@ -144,6 +147,26 @@ when applicable." "/site-ccache"))) (current-channel-entries)))) +(define (package-channels package) + "Return the list of channels providing PACKAGE or an empty list if it could +not be determined." + (match (and=> (package-location package) location-file) + (#f '()) + (file + (let ((file (if (string-prefix? "/" file) + file + (search-path %load-path file)))) + (if (and file + (string-prefix? (%store-prefix) file)) + (filter-map + (lambda (entry) + (let ((item (manifest-entry-item entry))) + (and (or (string-prefix? item file) + (string=? "guix" (manifest-entry-name entry))) + (manifest-entry-channel entry)))) + (current-profile-entries)) + '()))))) + (define (package-provenance package) "Return the provenance of PACKAGE as an sexp for use as the 'provenance' property of manifest entries, or #f if it could not be determined." @@ -153,36 +176,31 @@ property of manifest entries, or #f if it could not be determined." (('source value) value) (_ #f))) - (match (and=> (package-location package) location-file) - (#f #f) - (file - (let ((file (if (string-prefix? "/" file) - file - (search-path %load-path file)))) - (and file - (string-prefix? (%store-prefix) file) - - ;; Always store information about the 'guix' channel and - ;; optionally about the specific channel FILE comes from. - (or (let ((main (and=> (find (lambda (entry) - (string=? "guix" - (manifest-entry-name entry))) - (current-profile-entries)) - entry-source)) - (extra (any (lambda (entry) - (let ((item (manifest-entry-item entry))) - (and (string-prefix? item file) - (entry-source entry)))) - (current-profile-entries)))) - (and main - `(,main - ,@(if extra (list extra) '())))))))))) + (let* ((channels (package-channels package)) + (names (map (compose symbol->string channel-name) channels))) + ;; Always store information about the 'guix' channel and + ;; optionally about the specific channel FILE comes from. + (or (let ((main (and=> (find (lambda (entry) + (string=? "guix" + (manifest-entry-name entry))) + (current-profile-entries)) + entry-source)) + (extra (any (lambda (entry) + (let ((item (manifest-entry-item entry)) + (name (manifest-entry-name entry))) + (and (member name names) + (not (string=? name "guix")) + (entry-source entry)))) + (current-profile-entries)))) + (and main + `(,main + ,@(if extra (list extra) '()))))))) (define (manifest-entry-with-provenance entry) "Return ENTRY with an additional 'provenance' property if it's not already there." (let ((properties (manifest-entry-properties entry))) - (if (assq 'properties properties) + (if (assq 'provenance properties) entry (let ((item (manifest-entry-item entry))) (manifest-entry diff --git a/guix/download.scm b/guix/download.scm index 494825860e..30f69c0325 100644 --- a/guix/download.scm +++ b/guix/download.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021 Ludovic Courtès <ludo@gnu.org> ;;; Copyright © 2013, 2014, 2015 Andreas Enge <andreas@enge.fr> ;;; Copyright © 2015 Federico Beffa <beffa@fbengineering.ch> ;;; Copyright © 2016 Alex Griffin <a@ajgrf.com> @@ -27,7 +27,7 @@ #:use-module (guix derivations) #:use-module (guix packages) #:use-module (guix store) - #:use-module ((guix build download) #:prefix build:) + #:autoload (guix build download) (url-fetch) #:use-module (guix monads) #:use-module (guix gexp) #:use-module (guix utils) @@ -35,7 +35,7 @@ #:use-module (srfi srfi-1) #:use-module (srfi srfi-26) #:export (%mirrors - url-fetch + (url-fetch* . url-fetch) url-fetch/executable url-fetch/tarbomb url-fetch/zipbomb @@ -449,11 +449,11 @@ download by itself using its own dependencies." ;; for that built-in is widespread. #:local-build? #t))) -(define* (url-fetch url hash-algo hash - #:optional name - #:key (system (%current-system)) - (guile (default-guile)) - executable?) +(define* (url-fetch* url hash-algo hash + #:optional name + #:key (system (%current-system)) + (guile (default-guile)) + executable?) "Return a fixed-output derivation that fetches data from URL (a string, or a list of strings denoting alternate URLs), which is expected to have hash HASH of type HASH-ALGO (a symbol). By default, the file name is the base name of @@ -499,10 +499,10 @@ name in the store." #:key (system (%current-system)) (guile (default-guile))) "Like 'url-fetch', but make the downloaded file executable." - (url-fetch url hash-algo hash name - #:system system - #:guile guile - #:executable? #t)) + (url-fetch* url hash-algo hash name + #:system system + #:guile guile + #:executable? #t)) (define* (url-fetch/tarbomb url hash-algo hash #:optional name @@ -521,11 +521,11 @@ own. This helper makes it easier to deal with \"tar bombs\"." (define tar (module-ref (resolve-interface '(gnu packages base)) 'tar)) - (mlet %store-monad ((drv (url-fetch url hash-algo hash - (string-append "tarbomb-" - (or name file-name)) - #:system system - #:guile guile)) + (mlet %store-monad ((drv (url-fetch* url hash-algo hash + (string-append "tarbomb-" + (or name file-name)) + #:system system + #:guile guile)) (guile (package->derivation guile system))) ;; Take the tar bomb, and simply unpack it as a directory. ;; Use ungrafted tar/gzip so that the resulting tarball doesn't depend on @@ -535,9 +535,9 @@ own. This helper makes it easier to deal with \"tar bombs\"." #~(begin (use-modules (guix build utils)) (mkdir #$output) - (setenv "PATH" (string-append #$gzip "/bin")) + (setenv "PATH" (string-append #+gzip "/bin")) (chdir #$output) - (invoke (string-append #$tar "/bin/tar") + (invoke (string-append #+tar "/bin/tar") "xf" #$drv))) #:system system #:guile-for-build guile @@ -559,11 +559,11 @@ own. This helper makes it easier to deal with \"zip bombs\"." (define unzip (module-ref (resolve-interface '(gnu packages compression)) 'unzip)) - (mlet %store-monad ((drv (url-fetch url hash-algo hash - (string-append "zipbomb-" - (or name file-name)) - #:system system - #:guile guile)) + (mlet %store-monad ((drv (url-fetch* url hash-algo hash + (string-append "zipbomb-" + (or name file-name)) + #:system system + #:guile guile)) (guile (package->derivation guile system))) ;; Take the zip bomb, and simply unpack it as a directory. ;; Use ungrafted unzip so that the resulting tarball doesn't depend on @@ -574,7 +574,7 @@ own. This helper makes it easier to deal with \"zip bombs\"." (use-modules (guix build utils)) (mkdir #$output) (chdir #$output) - (invoke (string-append #$unzip "/bin/unzip") + (invoke (string-append #+unzip "/bin/unzip") #$drv))) #:system system #:guile-for-build guile @@ -598,10 +598,9 @@ whether or not to validate HTTPS server certificates." (lambda (temp port) (let ((result (parameterize ((current-output-port log)) - (build:url-fetch url temp - #:mirrors %mirrors - #:verify-certificate? - verify-certificate?)))) + (url-fetch url temp + #:mirrors %mirrors + #:verify-certificate? verify-certificate?)))) (close port) (and result (add-to-store store name recursive? "sha256" temp))))))) diff --git a/guix/ftp-client.scm b/guix/ftp-client.scm index 8d5adcb8ed..9cc34cc7ec 100644 --- a/guix/ftp-client.scm +++ b/guix/ftp-client.scm @@ -216,6 +216,17 @@ TIMEOUT, an ETIMEDOUT error is raised." (else (throw 'ftp-error conn "PASV" 227 message))))) +(define (ftp-epsv conn) + (let* ((message (%ftp-command "EPSV" 229 (ftp-connection-socket conn)))) + (string->number + (match:substring (string-match "\\(...([0-9]+).\\)" message) 1)))) + +(define (ftp-passive conn) + "Enter passive mode using EPSV or PASV, return a data connection port on +success." + ;; IPv6 only works with EPSV, so try it first. + (or (false-if-exception (ftp-epsv conn)) (ftp-pasv conn))) + (define (address-with-port sa port) "Return a socket-address object based on SA, but with PORT." (let ((fam (sockaddr:fam sa)) @@ -232,7 +243,7 @@ TIMEOUT, an ETIMEDOUT error is raised." (if directory (ftp-chdir conn directory)) - (let* ((port (ftp-pasv conn)) + (let* ((port (ftp-passive conn)) (ai (ftp-connection-addrinfo conn)) (s (socket (addrinfo:fam ai) (addrinfo:socktype ai) (addrinfo:protocol ai)))) @@ -281,7 +292,7 @@ must be closed before CONN can be used for other purposes." ;; Ask for "binary mode". (%ftp-command "TYPE I" 200 (ftp-connection-socket conn)) - (let* ((port (ftp-pasv conn)) + (let* ((port (ftp-passive conn)) (ai (ftp-connection-addrinfo conn)) (s (with-fluids ((%default-port-encoding #f)) (socket (addrinfo:fam ai) (addrinfo:socktype ai) diff --git a/guix/gexp.scm b/guix/gexp.scm index 764c89a187..afb935761e 100644 --- a/guix/gexp.scm +++ b/guix/gexp.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2014, 2015, 2016, 2017, 2018, 2019, 2020 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021 Ludovic Courtès <ludo@gnu.org> ;;; Copyright © 2018 Clément Lassieur <clement@lassieur.org> ;;; Copyright © 2018 Jan Nieuwenhuizen <janneke@gnu.org> ;;; Copyright © 2019, 2020 Mathieu Othacehe <m.othacehe@gmail.com> @@ -120,8 +120,6 @@ file-like? lower-object - lower-inputs - &gexp-error gexp-error? &gexp-input-error @@ -759,19 +757,28 @@ attribute that is traversed." (append (let ((attribute (self-attribute gexp))) (validate gexp attribute) attribute) - (append-map (match-lambda - (($ <gexp-input> (? gexp? exp)) - (gexp-attribute exp self-attribute - #:validate validate)) - (($ <gexp-input> (lst ...)) - (append-map (lambda (item) - (gexp-attribute item self-attribute - #:validate - validate)) - lst)) - (_ - '())) - (gexp-references gexp))) + (reverse + (fold (lambda (input result) + (match input + (($ <gexp-input> (? gexp? exp)) + (append (gexp-attribute exp self-attribute + #:validate validate) + result)) + (($ <gexp-input> (lst ...)) + (fold/tree (lambda (obj result) + (match obj + ((? gexp? exp) + (append (gexp-attribute exp self-attribute + #:validate validate) + result)) + (_ + result))) + result + lst)) + (_ + result))) + '() + (gexp-references gexp)))) equal?) '())) ;plain Scheme data type @@ -828,8 +835,7 @@ list." (one-of symbol? string? keyword? pair? null? array? number? boolean? char?))) -(define* (lower-inputs inputs - #:key system target) +(define (lower-inputs inputs system target) "Turn any object from INPUTS into a derivation input for SYSTEM or a store item (a \"source\"); return the corresponding input list as a monadic value. When TARGET is true, use it as the cross-compilation target triplet." @@ -842,24 +848,23 @@ When TARGET is true, use it as the cross-compilation target triplet." (with-monad %store-monad (>>= (mapm/accumulate-builds (match-lambda - (((? struct? thing) sub-drv ...) - (mlet %store-monad ((obj (lower-object - thing system #:target target))) + (($ <gexp-input> (? store-item? item)) + (return item)) + (($ <gexp-input> thing output native?) + (mlet %store-monad ((obj (lower-object thing system + #:target + (and (not native?) + target)))) (return (match obj ((? derivation? drv) - (let ((outputs (if (null? sub-drv) - '("out") - sub-drv))) - (derivation-input drv outputs))) + (derivation-input drv (list output))) ((? store-item? item) item) ((? self-quoting?) ;; Some inputs such as <system-binding> can lower to ;; a self-quoting object that FILTERM will filter ;; out. - #f))))) - (((? store-item? item)) - (return item))) + #f)))))) inputs) filterm))) @@ -867,11 +872,17 @@ When TARGET is true, use it as the cross-compilation target triplet." "Given GRAPHS, a list of (FILE-NAME INPUT ...) lists for use as a #:reference-graphs argument, lower it such that each INPUT is replaced by the corresponding <derivation-input> or store item." + (define tuple->gexp-input + (match-lambda + ((thing) + (%gexp-input thing "out" (not target))) + ((thing output) + (%gexp-input thing output (not target))))) + (match graphs (((file-names . inputs) ...) - (mlet %store-monad ((inputs (lower-inputs inputs - #:system system - #:target target))) + (mlet %store-monad ((inputs (lower-inputs (map tuple->gexp-input inputs) + system target))) (return (map cons file-names inputs)))))) (define* (lower-references lst #:key system target) @@ -925,6 +936,7 @@ second element is the derivation to compile them." (mcached equal? (mlet %store-monad ((modules (if (pair? modules) (imported-modules modules + #:guile guile #:system system #:module-path module-path) (return #f))) @@ -941,6 +953,15 @@ second element is the derivation to compile them." modules system extensions guile deprecation-warnings module-path)) +(define (sexp->string sexp) + "Like 'object->string', but deterministic and slightly faster." + ;; Explicitly use UTF-8 for determinism, and also because UTF-8 output is + ;; faster. + (with-fluids ((%default-port-encoding "UTF-8")) + (call-with-output-string + (lambda (port) + (write sexp port))))) + (define* (lower-gexp exp #:key (module-path %load-path) @@ -991,16 +1012,9 @@ derivations--e.g., code evaluated for its side effects." (guile (if guile-for-build (return guile-for-build) (default-guile-derivation system))) - (normals (lower-inputs (gexp-inputs exp) - #:system system - #:target target)) - (natives (lower-inputs (gexp-native-inputs exp) - #:system system - #:target #f)) - (inputs -> (append normals natives)) - (sexp (gexp->sexp exp - #:system system - #:target target)) + (inputs (lower-inputs (gexp-inputs exp) + system target)) + (sexp (gexp->sexp exp system target)) (extensions -> (gexp-extensions exp)) (exts (mapm %store-monad (lambda (obj) @@ -1159,7 +1173,7 @@ The other arguments are as for 'derivation'." (return #f))) (guile -> (lowered-gexp-guile lowered)) (builder (text-file script-name - (object->string + (sexp->string (lowered-gexp-sexp lowered))))) (mbegin %store-monad (set-grafting graft?) ;restore the initial setting @@ -1203,42 +1217,60 @@ The other arguments are as for 'derivation'." #:substitutable? substitutable? #:properties properties)))) -(define* (gexp-inputs exp #:key native?) - "Return the input list for EXP. When NATIVE? is true, return only native -references; otherwise, return only non-native references." - ;; TODO: Return <gexp-input> records instead of tuples. +(define (fold/tree proc seed lst) + "Like 'fold', but recurse into sub-lists of LST and accept improper lists." + (let loop ((obj lst) + (result seed)) + (match obj + ((head . tail) + (loop tail (loop head result))) + (_ + (proc obj result))))) + +(define (gexp-inputs exp) + "Return the list of <gexp-input> for EXP." + (define set-gexp-input-native? + (match-lambda + (($ <gexp-input> thing output) + (%gexp-input thing output #t)))) + + (define (interesting? obj) + (or (file-like? obj) + (and (string? obj) (direct-store-path? obj)))) + (define (add-reference-inputs ref result) (match ref (($ <gexp-input> (? gexp? exp) _ #t) - (if native? - (append (gexp-inputs exp) - (gexp-inputs exp #:native? #t) - result) - result)) - (($ <gexp-input> (? gexp? exp) _ #f) - (append (gexp-inputs exp #:native? native?) + (append (map set-gexp-input-native? (gexp-inputs exp)) result)) + (($ <gexp-input> (? gexp? exp) _ #f) + (append (gexp-inputs exp) result)) (($ <gexp-input> (? string? str)) (if (direct-store-path? str) - (cons `(,str) result) + (cons ref result) result)) (($ <gexp-input> (? struct? thing) output n?) - (if (and (eqv? n? native?) (lookup-compiler thing)) + (if (lookup-compiler thing) ;; THING is a derivation, or a package, or an origin, etc. - (cons `(,thing ,output) result) + (cons ref result) result)) - (($ <gexp-input> (lst ...) output n?) - (fold-right add-reference-inputs result - ;; XXX: For now, automatically convert LST to a list of - ;; gexp-inputs. Inherit N?. - (map (match-lambda - ((? gexp-input? x) - (%gexp-input (gexp-input-thing x) - (gexp-input-output x) - n?)) - (x - (%gexp-input x "out" n?))) - lst))) + (($ <gexp-input> (? pair? lst) output n?) + ;; XXX: Scan LST for inputs. Inherit N?. + (fold/tree (lambda (obj result) + (match obj + ((? gexp-input? x) + (cons (%gexp-input (gexp-input-thing x) + (gexp-input-output x) + n?) + result)) + ((? interesting? x) + (cons (%gexp-input x "out" n?) result)) + ((? gexp? x) + (append (gexp-inputs x) result)) + (_ + result))) + result + lst)) (_ ;; Ignore references to other kinds of objects. result))) @@ -1247,9 +1279,6 @@ references; otherwise, return only non-native references." '() (gexp-references exp))) -(define gexp-native-inputs - (cut gexp-inputs <> #:native? #t)) - (define (gexp-outputs exp) "Return the outputs referred to by EXP as a list of strings." (define (add-reference-output ref result) @@ -1258,24 +1287,22 @@ references; otherwise, return only non-native references." (cons name result)) (($ <gexp-input> (? gexp? exp)) (append (gexp-outputs exp) result)) - (($ <gexp-input> (lst ...) output native?) - ;; XXX: Automatically convert LST. - (add-reference-output (map (match-lambda - ((? gexp-input? x) x) - (x (%gexp-input x "out" native?))) - lst) - result)) - ((lst ...) - (fold-right add-reference-output result lst)) + (($ <gexp-input> (? pair? lst)) + ;; XXX: Scan LST for outputs. + (fold/tree (lambda (obj result) + (match obj + (($ <gexp-output> name) (cons name result)) + ((? gexp? x) (append (gexp-outputs x) result)) + (_ result))) + result + lst)) (_ result))) (delete-duplicates - (add-reference-output (gexp-references exp) '()))) + (fold add-reference-output '() (gexp-references exp)))) -(define* (gexp->sexp exp #:key - (system (%current-system)) - (target (%current-target-system))) +(define (gexp->sexp exp system target) "Return (monadically) the sexp corresponding to EXP for the given OUTPUT, and in the current monad setting (system type, etc.)" (define* (reference->sexp ref #:optional native?) @@ -1288,17 +1315,19 @@ and in the current monad setting (system type, etc.)" (return `((@ (guile) getenv) ,output))) (($ <gexp-input> (? gexp? exp) output n?) (gexp->sexp exp - #:system system - #:target (if (or n? native?) #f target))) + system (if (or n? native?) #f target))) (($ <gexp-input> (refs ...) output n?) (mapm %store-monad (lambda (ref) ;; XXX: Automatically convert REF to an gexp-input. - (reference->sexp - (if (gexp-input? ref) - ref - (%gexp-input ref "out" n?)) - (or n? native?))) + (if (or (symbol? ref) (number? ref) + (boolean? ref) (null? ref) (array? ref)) + (return ref) + (reference->sexp + (if (gexp-input? ref) + ref + (%gexp-input ref "out" n?)) + (or n? native?)))) refs)) (($ <gexp-input> (? struct? thing) output n?) (let ((target (if (or n? native?) #f target))) @@ -1680,6 +1709,7 @@ TARGET, a GNU triplet." ;; TODO: Pass MODULES as an environment variable. (gexp->derivation name build #:system system + #:target target #:guile-for-build guile #:local-build? #t #:env-vars @@ -1705,21 +1735,26 @@ TARGET, a GNU triplet." 'guile-3.0)) (define* (load-path-expression modules #:optional (path %load-path) - #:key (extensions '()) system target) + #:key (extensions '()) system target + (guile (default-guile))) "Return as a monadic value a gexp that sets '%load-path' and '%load-compiled-path' to point to MODULES, a list of module names. MODULES -are searched for in PATH. Return #f when MODULES and EXTENSIONS are empty." +are searched for in PATH. Return #f when MODULES and EXTENSIONS are empty. +Assume MODULES are compiled with GUILE." (if (and (null? modules) (null? extensions)) (with-monad %store-monad (return #f)) - (mlet %store-monad ((modules (imported-modules modules - #:module-path path - #:system system)) - (compiled (compiled-modules modules - #:extensions extensions - #:module-path path - #:system system - #:target target))) + (mlet* %store-monad ((guile (lower-object guile system #:target #f)) + (compiled (compiled-modules modules + #:guile guile + #:extensions extensions + #:module-path path + #:system system + #:target target)) + (modules (imported-modules modules + #:guile guile + #:module-path path + #:system system))) (return (gexp (eval-when (expand load eval) ;; Augment the load paths and delete duplicates. Do that @@ -1765,10 +1800,13 @@ imported modules in its search path. Look up EXP's modules in MODULE-PATH." (set-load-path (load-path-expression (gexp-modules exp) module-path + #:guile guile #:extensions (gexp-extensions exp) #:system system - #:target target))) + #:target target)) + (guile-for-build + (lower-object guile system #:target #f))) (gexp->derivation name (gexp (call-with-output-file (ungexp output) @@ -1791,6 +1829,7 @@ imported modules in its search path. Look up EXP's modules in MODULE-PATH." #:system system #:target target #:module-path module-path + #:guile-for-build guile-for-build ;; These derivations are not worth offloading or ;; substituting. diff --git a/guix/git-download.scm b/guix/git-download.scm index 8e575e3b5f..425184717a 100644 --- a/guix/git-download.scm +++ b/guix/git-download.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2014, 2015, 2016, 2017, 2018, 2019, 2020 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021 Ludovic Courtès <ludo@gnu.org> ;;; Copyright © 2017 Mathieu Lirzin <mthl@gnu.org> ;;; Copyright © 2017 Christopher Baines <mail@cbaines.net> ;;; Copyright © 2020 Jakub KÄ…dzioÅ‚ka <kuba@kadziolka.net> @@ -27,7 +27,14 @@ #:use-module (guix packages) #:use-module (guix modules) #:autoload (guix build-system gnu) (standard-packages) - #:use-module (git) + #:autoload (git repository) (repository-open + repository-close! + repository-discover + repository-head + repository-working-directory) + #:autoload (git commit) (commit-lookup commit-tree) + #:autoload (git reference) (reference-target) + #:autoload (git tree) (tree-list) #:use-module (ice-9 match) #:use-module (ice-9 vlist) #:use-module (srfi srfi-1) diff --git a/guix/git.scm b/guix/git.scm index a5103547d3..1820036f25 100644 --- a/guix/git.scm +++ b/guix/git.scm @@ -185,10 +185,9 @@ make sure no empty directory is left behind." (lambda () (mkdir-p directory) - (let ((auth-method (%make-auth-ssh-agent))) - (clone url directory - (make-clone-options - #:fetch-options (make-default-fetch-options))))) + (clone url directory + (make-clone-options + #:fetch-options (make-default-fetch-options)))) (lambda _ (false-if-exception (rmdir directory))))) @@ -389,9 +388,8 @@ it unchanged." ;; Only fetch remote if it has not been cloned just before. (when (and cache-exists? (not (reference-available? repository ref))) - (let ((auth-method (%make-auth-ssh-agent))) - (remote-fetch (remote-lookup repository "origin") - #:fetch-options (make-default-fetch-options)))) + (remote-fetch (remote-lookup repository "origin") + #:fetch-options (make-default-fetch-options))) (when recursive? (update-submodules repository #:log-port log-port)) diff --git a/guix/gnu-maintenance.scm b/guix/gnu-maintenance.scm index 0da6fc19b6..ba659c0a60 100644 --- a/guix/gnu-maintenance.scm +++ b/guix/gnu-maintenance.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021 Ludovic Courtès <ludo@gnu.org> ;;; Copyright © 2012, 2013 Nikita Karetnikov <nikita@karetnikov.org> ;;; Copyright © 2021 Simon Tournier <zimon.toutoune@gmail.com> ;;; @@ -28,6 +28,7 @@ #:use-module (srfi srfi-1) #:use-module (srfi srfi-11) #:use-module (srfi srfi-26) + #:use-module (srfi srfi-34) #:use-module (rnrs io ports) #:use-module (system foreign) #:use-module (guix http-client) @@ -37,7 +38,8 @@ #:use-module (guix records) #:use-module (guix upstream) #:use-module (guix packages) - #:use-module (zlib) + #:autoload (zlib) (call-with-gzip-input-port) + #:autoload (htmlprag) (html->sxml) ;from Guile-Lib #:export (gnu-package-name gnu-package-mundane-name gnu-package-copyright-holder @@ -64,8 +66,10 @@ %gnu-updater %gnu-ftp-updater %savannah-updater + %sourceforge-updater %xorg-updater - %kernel.org-updater)) + %kernel.org-updater + %generic-html-updater)) ;;; Commentary: ;;; @@ -238,7 +242,8 @@ network to check in GNU's database." ;; The .zip extensions is notably used for freefont-ttf. ;; The "-src" pattern is for "TeXmacs-1.0.7.9-src.tar.gz". ;; The "-gnu[0-9]" pattern is for "icecat-38.4.0-gnu1.tar.bz2". - (make-regexp "^([^.]+)-([0-9]|[^-])+(-(src|gnu[0-9]))?\\.(tar\\.|zip$)")) + ;; Accept underscores as in "PKG_1.2.tar.gz" for some non-GNU packages. + (make-regexp "^([^.]+)[-_]([0-9]|[^-])+(-(src|[sS]ource|gnu[0-9]))?\\.(tar\\.|zip$)")) (define %alpha-tarball-rx (make-regexp "^.*-.*[0-9](-|~)?(alpha|beta|rc|RC|cvs|svn|git)-?[0-9\\.]*\\.tar\\.")) @@ -246,7 +251,9 @@ network to check in GNU's database." (define (release-file? project file) "Return #f if FILE is not a release tarball of PROJECT, otherwise return true." - (and (not (member (file-extension file) '("sig" "sign" "asc"))) + (and (not (member (file-extension file) + '("sig" "sign" "asc" + "md5sum" "sha1sum" "sha256sum"))) (and=> (regexp-exec %tarball-rx file) (lambda (match) ;; Filter out unrelated files, like `guile-www-1.1.1'. @@ -322,16 +329,11 @@ name/directory pairs." #:key (server "ftp.gnu.org") (directory (string-append "/gnu/" project)) - (keep-file? (const #t)) - (file->signature (cut string-append <> ".sig")) - (ftp-open ftp-open) (ftp-close ftp-close)) + (file->signature (cut string-append <> ".sig"))) "Return an <upstream-source> for the latest release of PROJECT on SERVER under DIRECTORY, or #f. Use FTP-OPEN and FTP-CLOSE to open (resp. close) FTP connections; this can be useful to reuse connections. -KEEP-FILE? is a predicate to decide whether to enter a directory and to -consider a given file (source tarball) as a valid candidate based on its name. - FILE->SIGNATURE must be a procedure; it is passed a source file URL and must return the corresponding signature URL, or #f it signatures are unavailable." (define (latest a b) @@ -345,7 +347,7 @@ return the corresponding signature URL, or #f it signatures are unavailable." ;; Return #t for patch directory names such as 'bash-4.2-patches'. (cut string-suffix? "patches" <>)) - (define conn (ftp-open server)) + (define conn (ftp-open server #:timeout 5)) (define (file->url directory file) (string-append "ftp://" server directory "/" file)) @@ -389,7 +391,6 @@ return the corresponding signature URL, or #f it signatures are unavailable." (releases (filter-map (match-lambda ((file 'file . _) (and (release-file? project file) - (keep-file? file) (file->source directory file))) (_ #f)) entries))) @@ -447,18 +448,6 @@ hosted on ftp.gnu.org, or not under that name (this is the case for ;;; Latest HTTP release. ;;; -(define (html->sxml port) - "Read HTML from PORT and return the corresponding SXML tree." - (let ((str (get-string-all port))) - (catch #t - (lambda () - ;; XXX: This is the poor developer's HTML-to-XML converter. It's good - ;; enough for directory listings at <https://kernel.org/pub> but if - ;; needed we could resort to (htmlprag) from Guile-Lib. - (call-with-input-string (string-replace-substring str "<hr>" "<hr />") - xml->sxml)) - (const '(html))))) ;parse error - (define (html-links sxml) "Return the list of links found in SXML, the SXML tree of an HTML page." (let loop ((sxml sxml) @@ -479,33 +468,47 @@ hosted on ftp.gnu.org, or not under that name (this is the case for #:key (base-url "https://kernel.org/pub") (directory (string-append "/" package)) - (file->signature (cut string-append <> ".sig"))) + file->signature) "Return an <upstream-source> for the latest release of PACKAGE (a string) on SERVER under DIRECTORY, or #f. BASE-URL should be the URL of an HTML page, typically a directory listing as found on 'https://kernel.org/pub'. -FILE->SIGNATURE must be a procedure; it is passed a source file URL and must -return the corresponding signature URL, or #f it signatures are unavailable." - (let* ((uri (string->uri (string-append base-url directory "/"))) - (port (http-fetch/cached uri #:ttl 3600)) - (sxml (html->sxml port))) +When FILE->SIGNATURE is omitted or #f, guess the detached signature file name, +if any. Otherwise, FILE->SIGNATURE must be a procedure; it is passed a source +file URL and must return the corresponding signature URL, or #f it signatures +are unavailable." + (let* ((uri (string->uri (if (string-null? directory) + base-url + (string-append base-url directory "/")))) + (port (http-fetch/cached uri #:ttl 3600)) + (sxml (html->sxml port)) + (links (delete-duplicates (html-links sxml)))) + (define (file->signature/guess url) + (let ((base (basename url))) + (any (lambda (link) + (any (lambda (extension) + (and (string=? (string-append base extension) + (basename link)) + (string-append url extension))) + '(".asc" ".sig" ".sign"))) + links))) + (define (url->release url) - (and (string=? url (basename url)) ;relative reference? - (release-file? package url) - (let-values (((name version) - (package-name->name+version - (tarball-sans-extension url) - #\-))) - (upstream-source - (package name) - (version version) - (urls (list (string-append base-url directory "/" url))) - (signature-urls - (list (file->signature - (string-append base-url directory "/" url)))))))) + (let* ((base (basename url)) + (url (if (string=? base url) + (string-append base-url directory "/" url) + url))) + (and (release-file? package base) + (let ((version (tarball->version base))) + (upstream-source + (package package) + (version version) + (urls (list url)) + (signature-urls + (list ((or file->signature file->signature/guess) url)))))))) (define candidates - (filter-map url->release (html-links sxml))) + (filter-map url->release links)) (close-port port) (match candidates @@ -593,7 +596,7 @@ list available from %GNU-FILE-LIST-URI over HTTP(S)." (define %package-name-rx ;; Regexp for a package name, e.g., "foo-X.Y". Since TeXmacs uses ;; "TeXmacs-X.Y-src", the `-src' suffix is allowed. - (make-regexp "^(.*)-(([0-9]|\\.)+)(-src)?")) + (make-regexp "^(.*)[-_](([0-9]|\\.)+)(-src|\\.src|\\.orig)?")) (define (gnu-package-name->name+version name+version) "Return the package name and version number extracted from NAME+VERSION." @@ -608,11 +611,12 @@ list available from %GNU-FILE-LIST-URI over HTTP(S)." (define (pure-gnu-package? package) "Return true if PACKAGE is a non-Emacs and non-GNOME GNU package. This excludes AucTeX, for instance, whose releases are now uploaded to -elpa.gnu.org, and all the GNOME packages; EMMS is included though, because its -releases are on gnu.org." +elpa.gnu.org, GNU Radio, which has releases at www.gnuradio.org, and all the +GNOME packages; EMMS is included though, because its releases are on gnu.org." (and (or (not (string-prefix? "emacs-" (package-name package))) (gnu-hosted? package)) (not (gnome-package? package)) + (not (string-prefix? "gnuradio" (package-name package))) (gnu-package? package))) (define gnu-hosted? @@ -621,7 +625,7 @@ releases are on gnu.org." (define (url-prefix-rewrite old new) "Return a one-argument procedure that rewrites URL prefix OLD to NEW." (lambda (url) - (if (string-prefix? old url) + (if (and url (string-prefix? old url)) (string-append new (string-drop url (string-length old))) url))) @@ -634,9 +638,6 @@ releases are on gnu.org." (lambda (urls) (map rewrite-url urls)))))) -(define savannah-package? - (url-prefix-predicate "mirror://savannah/")) - (define %savannah-base ;; One of the Savannah mirrors listed at ;; <http://download0.savannah.gnu.org/mirmon/savannah/> that serves valid @@ -653,14 +654,57 @@ releases are on gnu.org." (directory (dirname (uri-path uri))) (rewrite (url-prefix-rewrite %savannah-base "mirror://savannah"))) - ;; Note: We use the default 'file->signature', which adds ".sig", but not - ;; all projects on Savannah follow that convention: some use ".asc" and - ;; perhaps some lack signatures altogether. + ;; Note: We use the default 'file->signature', which adds ".sig", ".asc", + ;; or whichever detached signature naming scheme PACKAGE uses. (and=> (latest-html-release package #:base-url %savannah-base #:directory directory) (cut adjusted-upstream-source <> rewrite)))) +(define (latest-sourceforge-release package) + "Return the latest release of PACKAGE." + (define (uri-append uri extension) + ;; Return URI with EXTENSION appended. + (build-uri (uri-scheme uri) + #:host (uri-host uri) + #:path (string-append (uri-path uri) extension))) + + (define (valid-uri? uri) + ;; Return true if URI is reachable. + (false-if-exception + (case (response-code (http-head uri)) + ((200 302) #t) + (else #f)))) + + (let* ((name (package-upstream-name package)) + (base (string-append "https://sourceforge.net/projects/" + name "/files")) + (url (string-append base "/latest/download")) + (response (false-if-exception (http-head url)))) + (and response + (= 302 (response-code response)) + (response-location response) + (match (string-tokenize (uri-path (response-location response)) + (char-set-complement (char-set #\/))) + ((_ components ...) + (let* ((path (string-join components "/")) + (url (string-append "mirror://sourceforge/" path))) + (and (release-file? name (basename path)) + + ;; Take the heavy-handed approach of probing 3 additional + ;; URLs. XXX: Would be nicer if this could be avoided. + (let* ((loc (response-location response)) + (sig (any (lambda (extension) + (let ((uri (uri-append loc extension))) + (and (valid-uri? uri) + (string-append url extension)))) + '(".asc" ".sig" ".sign")))) + (upstream-source + (package name) + (version (tarball->version (basename path))) + (urls (list url)) + (signature-urls (and sig (list sig)))))))))))) + (define (latest-xorg-release package) "Return the latest release of PACKAGE." (let ((uri (string->uri (origin-uri (package-source package))))) @@ -695,6 +739,60 @@ releases are on gnu.org." #:file->signature file->signature) (cut adjusted-upstream-source <> rewrite)))) +(define html-updatable-package? + ;; Return true if the given package may be handled by the generic HTML + ;; updater. + (let ((hosting-sites '("github.com" "github.io" "gitlab.com" + "notabug.org" "sr.ht" + "gforge.inria.fr" "gitlab.inria.fr" + "ftp.gnu.org" "download.savannah.gnu.org" + "pypi.org" "crates.io" "rubygems.org" + "bioconductor.org"))) + (define http-url? + (url-predicate (lambda (url) + (match (string->uri url) + (#f #f) + (uri + (let ((scheme (uri-scheme uri)) + (host (uri-host uri))) + (and (memq scheme '(http https)) + (not (member host hosting-sites))))))))) + + (lambda (package) + (or (assoc-ref (package-properties package) 'release-monitoring-url) + (http-url? package))))) + +(define (latest-html-updatable-release package) + "Return the latest release of PACKAGE. Do that by crawling the HTML page of +the directory containing its source tarball." + (let* ((uri (string->uri + (match (origin-uri (package-source package)) + ((? string? url) url) + ((url _ ...) url)))) + (custom (assoc-ref (package-properties package) + 'release-monitoring-url)) + (base (or custom + (string-append (symbol->string (uri-scheme uri)) + "://" (uri-host uri)))) + (directory (if custom + "" + (dirname (uri-path uri)))) + (package (package-upstream-name package))) + (catch #t + (lambda () + (guard (c ((http-get-error? c) #f)) + (latest-html-release package + #:base-url base + #:directory directory))) + (lambda (key . args) + ;; Return false and move on upon connection failures and bogus HTTP + ;; servers. + (unless (memq key '(gnutls-error tls-certificate-error + system-error + bad-header bad-header-component)) + (apply throw key args)) + #f)))) + (define %gnu-updater ;; This is for everything at ftp.gnu.org. (upstream-updater @@ -721,6 +819,13 @@ releases are on gnu.org." (pred (url-prefix-predicate "mirror://savannah/")) (latest latest-savannah-release))) +(define %sourceforge-updater + (upstream-updater + (name 'sourceforge) + (description "Updater for packages hosted on sourceforge.net") + (pred (url-prefix-predicate "mirror://sourceforge/")) + (latest latest-sourceforge-release))) + (define %xorg-updater (upstream-updater (name 'xorg) @@ -735,4 +840,11 @@ releases are on gnu.org." (pred (url-prefix-predicate "mirror://kernel.org/")) (latest latest-kernel.org-release))) +(define %generic-html-updater + (upstream-updater + (name 'generic-html) + (description "Updater that crawls HTML pages.") + (pred html-updatable-package?) + (latest latest-html-updatable-release))) + ;;; gnu-maintenance.scm ends here diff --git a/guix/grafts.scm b/guix/grafts.scm index 910dcadc8a..fd8a108092 100644 --- a/guix/grafts.scm +++ b/guix/grafts.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2014, 2015, 2016, 2017, 2018, 2019, 2020 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021 Ludovic Courtès <ludo@gnu.org> ;;; ;;; This file is part of GNU Guix. ;;; @@ -323,14 +323,14 @@ DRV, and graft DRV itself to refer to those grafted dependencies." ;; Whether to honor package grafts by default. (make-parameter #t)) -(define (set-grafting enable?) - "This monadic procedure enables grafting when ENABLE? is true, and disables -it otherwise. It returns the previous setting." +(define-inlinable (set-grafting enable?) + ;; This monadic procedure enables grafting when ENABLE? is true, and + ;; disables it otherwise. It returns the previous setting. (lambda (store) (values (%graft? enable?) store))) -(define (grafting?) - "Return a Boolean indicating whether grafting is enabled." +(define-inlinable (grafting?) + ;; Return a Boolean indicating whether grafting is enabled. (lambda (store) (values (%graft?) store))) diff --git a/guix/http-client.scm b/guix/http-client.scm index 553640fe9e..a2e11a1b73 100644 --- a/guix/http-client.scm +++ b/guix/http-client.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2020 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2020, 2021 Ludovic Courtès <ludo@gnu.org> ;;; Copyright © 2015 Mark H Weaver <mhw@netris.org> ;;; Copyright © 2012, 2015 Free Software Foundation, Inc. ;;; Copyright © 2017 Tobias Geerinckx-Rice <me@tobias.gr> @@ -21,8 +21,11 @@ (define-module (guix http-client) #:use-module (web uri) + #:use-module (web http) #:use-module ((web client) #:hide (open-socket-for-uri)) + #:use-module (web request) #:use-module (web response) + #:use-module (srfi srfi-1) #:use-module (srfi srfi-11) #:use-module (srfi srfi-19) #:use-module (srfi srfi-26) @@ -35,6 +38,7 @@ #:use-module (guix utils) #:use-module (guix base64) #:autoload (gcrypt hash) (sha256) + #:autoload (gnutls) (error/invalid-session) #:use-module ((guix build utils) #:select (mkdir-p dump-port)) #:use-module ((guix build download) @@ -50,6 +54,7 @@ http-get-error-reason http-fetch + http-multiple-get %http-cache-ttl http-fetch/cached)) @@ -70,9 +75,11 @@ (define* (http-fetch uri #:key port (text? #f) (buffered? #t) + (open-connection guix:open-connection-for-uri) (keep-alive? #f) (verify-certificate? #t) (headers '((user-agent . "GNU Guile"))) + (log-port (current-error-port)) timeout) "Return an input port containing the data at URI, and the expected number of bytes available or #f. If TEXT? is true, the data at URI is considered to be @@ -88,14 +95,16 @@ When VERIFY-CERTIFICATE? is true, verify HTTPS server certificates. TIMEOUT specifies the timeout in seconds for connection establishment; when TIMEOUT is #f, connection establishment never times out. +Write information about redirects to LOG-PORT. + Raise an '&http-get-error' condition if downloading fails." (let loop ((uri (if (string? uri) (string->uri uri) uri))) - (let ((port (or port (guix:open-connection-for-uri uri - #:verify-certificate? - verify-certificate? - #:timeout timeout))) + (let ((port (or port (open-connection uri + #:verify-certificate? + verify-certificate? + #:timeout timeout))) (headers (match (uri-userinfo uri) ((? string? str) (cons (cons 'Authorization @@ -122,7 +131,7 @@ Raise an '&http-get-error' condition if downloading fails." 308) ; permanent redirection (let ((uri (resolve-uri-reference (response-location resp) uri))) (close-port port) - (format (current-error-port) (G_ "following redirection to `~a'...~%") + (format log-port (G_ "following redirection to `~a'...~%") (uri->string uri)) (loop uri))) (else @@ -138,6 +147,112 @@ Raise an '&http-get-error' condition if downloading fails." (uri->string uri) code (response-reason-phrase resp)))))))))))) +(define-syntax-rule (false-if-networking-error exp) + "Return #f if EXP triggers a network related exception as can occur when +reusing stale cached connections." + ;; FIXME: Duplicated from 'with-cached-connection'. + (catch #t + (lambda () + exp) + (lambda (key . args) + ;; If PORT was cached and the server closed the connection in the + ;; meantime, we get EPIPE. In that case, open a fresh connection and + ;; retry. We might also get 'bad-response or a similar exception from + ;; (web response) later on, once we've sent the request, or a + ;; ERROR/INVALID-SESSION from GnuTLS. + (if (or (and (eq? key 'system-error) + (= EPIPE (system-error-errno `(,key ,@args)))) + (and (eq? key 'gnutls-error) + (eq? (first args) error/invalid-session)) + (memq key + '(bad-response bad-header bad-header-component))) + #f + (apply throw key args))))) + +(define* (http-multiple-get base-uri proc seed requests + #:key port (verify-certificate? #t) + (open-connection guix:open-connection-for-uri) + (keep-alive? #t) + (batch-size 1000)) + "Send all of REQUESTS to the server at BASE-URI. Call PROC for each +response, passing it the request object, the response, a port from which to +read the response body, and the previous result, starting with SEED, à la +'fold'. Return the final result. + +When PORT is specified, use it as the initial connection on which HTTP +requests are sent; otherwise call OPEN-CONNECTION to open a new connection for +a URI. When KEEP-ALIVE? is false, close the connection port before +returning." + (let connect ((port port) + (requests requests) + (result seed)) + (define batch + (if (>= batch-size (length requests)) + requests + (take requests batch-size))) + + ;; (format (current-error-port) "connecting (~a requests left)..." + ;; (length requests)) + (let ((p (or port (open-connection base-uri + #:verify-certificate? + verify-certificate?)))) + ;; For HTTPS, P is not a file port and does not support 'setvbuf'. + (when (file-port? p) + (setvbuf p 'block (expt 2 16))) + + ;; Send BATCH in a row. + ;; XXX: Do our own caching to work around inefficiencies when + ;; communicating over TLS: <http://bugs.gnu.org/22966>. + (let-values (((buffer get) (open-bytevector-output-port))) + ;; Inherit the HTTP proxying property from P. + (set-http-proxy-port?! buffer (http-proxy-port? p)) + + (unless (false-if-networking-error + (begin + (for-each (cut write-request <> buffer) batch) + (put-bytevector p (get)) + (force-output p) + #t)) + ;; If PORT becomes unusable, open a fresh connection and retry. + (close-port p) ; close the broken port + (connect #f requests result))) + + ;; Now start processing responses. + (let loop ((sent batch) + (processed 0) + (result result)) + (match sent + (() + (match (drop requests processed) + (() + (unless keep-alive? + (close-port p)) + (reverse result)) + (remainder + (connect p remainder result)))) + ((head tail ...) + (match (false-if-networking-error (read-response p)) + ((? response? resp) + (let* ((body (response-body-port resp)) + (result (proc head resp body result))) + ;; The server can choose to stop responding at any time, + ;; in which case we have to try again. Check whether + ;; that is the case. Note that even upon "Connection: + ;; close", we can read from BODY. + (match (assq 'connection (response-headers resp)) + (('connection 'close) + (close-port p) + (connect #f ;try again + (drop requests (+ 1 processed)) + result)) + (_ + (loop tail (+ 1 processed) result))))) + (#f + (close-port p) + (connect #f ; try again + (drop requests processed) + result))))))))) + ;;; ;;; Caching. @@ -161,6 +276,7 @@ Raise an '&http-get-error' condition if downloading fails." (define* (http-fetch/cached uri #:key (ttl (%http-cache-ttl)) text? (write-cache dump-port) (cache-miss (const #t)) + (log-port (current-error-port)) (timeout 10)) "Like 'http-fetch', return an input port, but cache its contents in ~/.cache/guix. The cache remains valid for TTL seconds. @@ -169,7 +285,9 @@ Call WRITE-CACHE with the HTTP input port and the cache output port to write the data to cache. Call CACHE-MISS with URI just before fetching data from URI. -TIMEOUT specifies the timeout in seconds for connection establishment." +TIMEOUT specifies the timeout in seconds for connection establishment. + +Write information about redirects to LOG-PORT." (let ((file (cache-file-for-uri uri))) (define (update-cache cache-port) (define cache-time @@ -191,6 +309,7 @@ TIMEOUT specifies the timeout in seconds for connection establishment." cache-port) (raise c)))) (let ((port (http-fetch uri #:text? text? + #:log-port log-port #:headers headers #:timeout timeout))) (cache-miss uri) (mkdir-p (dirname file)) diff --git a/guix/import/cran.scm b/guix/import/cran.scm index e8caf080fd..dbc858cb84 100644 --- a/guix/import/cran.scm +++ b/guix/import/cran.scm @@ -361,7 +361,7 @@ empty list when the FIELD cannot be found." (define (directory-needs-fortran? dir) "Check if the directory DIR contains Fortran source files." - (match (find-files dir "\\.f(90|95)?") + (match (find-files dir "\\.f(90|95)$") (() #f) (_ #t))) diff --git a/guix/import/gnome.scm b/guix/import/gnome.scm index 436ec88ef9..43966c1028 100644 --- a/guix/import/gnome.scm +++ b/guix/import/gnome.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2017, 2019 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2017, 2019, 2021 Ludovic Courtès <ludo@gnu.org> ;;; ;;; This file is part of GNU Guix. ;;; @@ -64,8 +64,9 @@ not be determined." (match (string-tokenize version %not-dot) (((= string->number major) (= string->number minor) . rest) (and minor (even? minor))) - (_ - #t))) ;cross fingers + (((= string->number major) . _) + ;; It should at last start with a digit. + major))) (define upstream-name ;; Some packages like "NetworkManager" have camel-case names. @@ -82,7 +83,10 @@ not be determined." ;; ftp.gnome.org supports 'if-Modified-Since', so the local ;; cache can expire early. - #:ttl (* 60 10))) + #:ttl (* 60 10) + + ;; Hide messages about URL redirects. + #:log-port (%make-void-port "w"))) (json (json->scm port))) (close-port port) (match json diff --git a/guix/import/go.scm b/guix/import/go.scm new file mode 100644 index 0000000000..6c0231e113 --- /dev/null +++ b/guix/import/go.scm @@ -0,0 +1,550 @@ +;;; GNU Guix --- Functional package management for GNU +;;; Copyright © 2020 Katherine Cox-Buday <cox.katherine.e@gmail.com> +;;; Copyright © 2020 Helio Machado <0x2b3bfa0+guix@googlemail.com> +;;; Copyright © 2021 François Joulaud <francois.joulaud@radiofrance.com> +;;; Copyright © 2021 Maxim Cournoyer <maxim.cournoyer@gmail.com> +;;; Copyright © 2021 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2021 Xinglu Chen <public@yoctocell.xyz> +;;; +;;; This file is part of GNU Guix. +;;; +;;; GNU Guix is free software; you can redistribute it and/or modify it +;;; under the terms of the GNU General Public License as published by +;;; the Free Software Foundation; either version 3 of the License, or (at +;;; your option) any later version. +;;; +;;; GNU Guix is distributed in the hope that it will be useful, but +;;; WITHOUT ANY WARRANTY; without even the implied warranty of +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;;; GNU General Public License for more details. +;;; +;;; You should have received a copy of the GNU General Public License +;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>. + +(define-module (guix import go) + #:use-module (guix build-system go) + #:use-module (guix git) + #:use-module (guix i18n) + #:use-module (guix diagnostics) + #:use-module (guix import utils) + #:use-module (guix import json) + #:use-module (guix packages) + #:use-module ((guix utils) #:select (string-replace-substring)) + #:use-module (guix http-client) + #:use-module ((guix licenses) #:prefix license:) + #:use-module (guix memoization) + #:autoload (htmlprag) (html->sxml) ;from Guile-Lib + #:autoload (guix git) (update-cached-checkout) + #:autoload (gcrypt hash) (open-hash-port hash-algorithm sha256) + #:autoload (guix serialization) (write-file) + #:autoload (guix base32) (bytevector->nix-base32-string) + #:autoload (guix build utils) (mkdir-p) + #:use-module (ice-9 match) + #:use-module (ice-9 rdelim) + #:use-module (ice-9 receive) + #:use-module (ice-9 regex) + #:use-module ((rnrs io ports) #:select (call-with-port)) + #:use-module (srfi srfi-1) + #:use-module (srfi srfi-9) + #:use-module (srfi srfi-11) + #:use-module (srfi srfi-26) + #:use-module (sxml xpath) + #:use-module (web client) + #:use-module (web response) + #:use-module (web uri) + + #:export (go-path-escape + go-module->guix-package + go-module-recursive-import)) + +;;; Commentary: +;;; +;;; (guix import go) attempts to make it easier to create Guix package +;;; declarations for Go modules. +;;; +;;; Modules in Go are a "collection of related Go packages" which are "the +;;; unit of source code interchange and versioning". Modules are generally +;;; hosted in a repository. +;;; +;;; At this point it should handle correctly modules which have only Go +;;; dependencies and are accessible from proxy.golang.org (or configured via +;;; GOPROXY). +;;; +;;; We want it to work more or less this way: +;;; - get latest version for the module from GOPROXY +;;; - infer VCS root repo from which we will check-out source by +;;; + recognising known patterns (like github.com) +;;; + or recognizing .vcs suffix +;;; + or parsing meta tag in HTML served at the URL +;;; + or (TODO) if nothing else works by using zip file served by GOPROXY +;;; - get go.mod from GOPROXY (which is able to synthetize one if needed) +;;; - extract list of dependencies from this go.mod +;;; +;;; The Go module paths are translated to a Guix package name under the +;;; assumption that there will be no collision. + +;;; TODO list +;;; - get correct hash in vcs->origin +;;; - print partial result during recursive imports (need to catch +;;; exceptions) + +;;; Code: + +(define (go-path-escape path) + "Escape a module path by replacing every uppercase letter with an +exclamation mark followed with its lowercase equivalent, as per the module +Escaped Paths specification (see: +https://godoc.org/golang.org/x/mod/module#hdr-Escaped_Paths)." + (define (escape occurrence) + (string-append "!" (string-downcase (match:substring occurrence)))) + (regexp-substitute/global #f "[A-Z]" path 'pre escape 'post)) + +(define (go-module-latest-version goproxy-url module-path) + "Fetch the version number of the latest version for MODULE-PATH from the +given GOPROXY-URL server." + (assoc-ref (json-fetch (format #f "~a/~a/@latest" goproxy-url + (go-path-escape module-path))) + "Version")) + + +(define (go-package-licenses name) + "Retrieve the list of licenses that apply to NAME, a Go package or module +name (e.g. \"github.com/golang/protobuf/proto\"). The data is scraped from +the https://pkg.go.dev/ web site." + (let*-values (((url) (string-append "https://pkg.go.dev/" name + "?tab=licenses")) + ((response body) (http-get url)) + ;; Extract the text contained in a h2 child node of any + ;; element marked with a "License" class attribute. + ((select) (sxpath `(// (* (@ (equal? (class "License")))) + h2 // *text*)))) + (and (eq? (response-code response) 200) + (match (select (html->sxml body)) + (() #f) ;nothing selected + (licenses licenses))))) + +(define (go.pkg.dev-info name) + (http-get (string-append "https://pkg.go.dev/" name))) +(define go.pkg.dev-info* + (memoize go.pkg.dev-info)) + +(define (go-package-description name) + "Retrieve a short description for NAME, a Go package name, +e.g. \"google.golang.org/protobuf/proto\". The data is scraped from the +https://pkg.go.dev/ web site." + (let*-values (((response body) (go.pkg.dev-info* name)) + ;; Extract the text contained in a h2 child node of any + ;; element marked with a "License" class attribute. + ((select) (sxpath + `(// (section + (@ (equal? (class "Documentation-overview")))) + (p 1))))) + (and (eq? (response-code response) 200) + (match (select (html->sxml body)) + (() #f) ;nothing selected + (((p . strings)) + ;; The paragraph text is returned as a list of strings embedding + ;; newline characters. Join them and strip the newline + ;; characters. + (string-delete #\newline (string-join strings))))))) + +(define (go-package-synopsis module-name) + "Retrieve a short synopsis for a Go module named MODULE-NAME, +e.g. \"google.golang.org/protobuf\". The data is scraped from +the https://pkg.go.dev/ web site." + ;; Note: Only the *module* (rather than package) page has the README title + ;; used as a synopsis on the https://pkg.go.dev web site. + (let*-values (((response body) (go.pkg.dev-info* module-name)) + ;; Extract the text contained in a h2 child node of any + ;; element marked with a "License" class attribute. + ((select) (sxpath + `(// (div (@ (equal? (class "UnitReadme-content")))) + // h3 *text*)))) + (and (eq? (response-code response) 200) + (match (select (html->sxml body)) + (() #f) ;nothing selected + ((title more ...) ;title is the first string of the list + (string-trim-both title)))))) + +(define (list->licenses licenses) + "Given a list of LICENSES mostly following the SPDX conventions, return the +corresponding Guix license or 'unknown-license!" + (filter-map (lambda (license) + (and (not (string-null? license)) + (not (any (cut string=? <> license) + '("AND" "OR" "WITH"))) + ;; Adjust the license names scraped from + ;; https://pkg.go.dev to an equivalent SPDX identifier, + ;; if they differ (see: https://github.com/golang/pkgsite + ;; /internal/licenses/licenses.go#L174). + (or (spdx-string->license + (match license + ("BlueOak-1.0" "BlueOak-1.0.0") + ("BSD-0-Clause" "0BSD") + ("BSD-2-Clause" "BSD-2-Clause-FreeBSD") + ("GPL2" "GPL-2.0") + ("GPL3" "GPL-3.0") + ("NIST" "NIST-PD") + (_ license))) + 'unknown-license!))) + licenses)) + +(define (fetch-go.mod goproxy-url module-path version) + "Fetches go.mod from the given GOPROXY-URL server for the given MODULE-PATH +and VERSION." + (let ((url (format #f "~a/~a/@v/~a.mod" goproxy-url + (go-path-escape module-path) + (go-path-escape version)))) + (http-fetch url))) + +(define %go.mod-require-directive-rx + ;; A line in a require directive is composed of a module path and + ;; a version separated by whitespace and an optionnal '//' comment at + ;; the end. + (make-regexp + (string-append + "^[[:blank:]]*" + "([^[:blank:]]+)[[:blank:]]+([^[:blank:]]+)" + "([[:blank:]]+//.*)?"))) + +(define %go.mod-replace-directive-rx + ;; ReplaceSpec = ModulePath [ Version ] "=>" FilePath newline + ;; | ModulePath [ Version ] "=>" ModulePath Version newline . + (make-regexp + (string-append + "([^[:blank:]]+)([[:blank:]]+([^[:blank:]]+))?" + "[[:blank:]]+" "=>" "[[:blank:]]+" + "([^[:blank:]]+)([[:blank:]]+([^[:blank:]]+))?"))) + +(define (parse-go.mod port) + "Parse the go.mod file accessible via the input PORT, returning a list of +requirements." + (define-record-type <results> + (make-results requirements replacements) + results? + (requirements results-requirements) + (replacements results-replacements)) + ;; We parse only a subset of https://golang.org/ref/mod#go-mod-file-grammar + ;; which we think necessary for our use case. + (define (toplevel results) + "Main parser, RESULTS is a pair of alist serving as accumulator for + all encountered requirements and replacements." + (let ((line (read-line port))) + (cond + ((eof-object? line) + ;; parsing ended, give back the result + results) + ((string=? line "require (") + ;; a require block begins, delegate parsing to IN-REQUIRE + (in-require results)) + ((string=? line "replace (") + ;; a replace block begins, delegate parsing to IN-REPLACE + (in-replace results)) + ((string-prefix? "require " line) + ;; a standalone require directive + (let* ((stripped-line (string-drop line 8)) + (new-results (require-directive results stripped-line))) + (toplevel new-results))) + ((string-prefix? "replace " line) + ;; a standalone replace directive + (let* ((stripped-line (string-drop line 8)) + (new-results (replace-directive results stripped-line))) + (toplevel new-results))) + (#t + ;; unrecognised line, ignore silently + (toplevel results))))) + + (define (in-require results) + (let ((line (read-line port))) + (cond + ((eof-object? line) + ;; this should never happen here but we ignore silently + results) + ((string=? line ")") + ;; end of block, coming back to toplevel + (toplevel results)) + (#t + (in-require (require-directive results line)))))) + + (define (in-replace results) + (let ((line (read-line port))) + (cond + ((eof-object? line) + ;; this should never happen here but we ignore silently + results) + ((string=? line ")") + ;; end of block, coming back to toplevel + (toplevel results)) + (#t + (in-replace (replace-directive results line)))))) + + (define (replace-directive results line) + "Extract replaced modules and new requirements from replace directive + in LINE and add to RESULTS." + (match results + (($ <results> requirements replaced) + (let* ((rx-match (regexp-exec %go.mod-replace-directive-rx line)) + (module-path (match:substring rx-match 1)) + (version (match:substring rx-match 3)) + (new-module-path (match:substring rx-match 4)) + (new-version (match:substring rx-match 6)) + (new-replaced (alist-cons module-path version replaced)) + (new-requirements + (if (string-match "^\\.?\\./" new-module-path) + requirements + (alist-cons new-module-path new-version requirements)))) + (make-results new-requirements new-replaced))))) + (define (require-directive results line) + "Extract requirement from LINE and add it to RESULTS." + (let* ((rx-match (regexp-exec %go.mod-require-directive-rx line)) + (module-path (match:substring rx-match 1)) + ;; we saw double-quoted string in the wild without escape + ;; sequences so we just trim the quotes + (module-path (string-trim-both module-path #\")) + (version (match:substring rx-match 2))) + (match results + (($ <results> requirements replaced) + (make-results (alist-cons module-path version requirements) replaced))))) + + (let ((results (toplevel (make-results '() '())))) + (match results + (($ <results> requirements replaced) + ;; At last we remove replaced modules from the requirements list + (fold + (lambda (replacedelem requirements) + (alist-delete! (car replacedelem) requirements)) + requirements + replaced))))) + +;; Prevent inlining of this procedure, which is accessed by unit tests. +(set! parse-go.mod parse-go.mod) + +(define-record-type <vcs> + (%make-vcs url-prefix root-regex type) + vcs? + (url-prefix vcs-url-prefix) + (root-regex vcs-root-regex) + (type vcs-type)) +(define (make-vcs prefix regexp type) + (%make-vcs prefix (make-regexp regexp) type)) +(define known-vcs + ;; See the following URL for the official Go equivalent: + ;; https://github.com/golang/go/blob/846dce9d05f19a1f53465e62a304dea21b99f910/src/cmd/go/internal/vcs/vcs.go#L1026-L1087 + (list + (make-vcs + "github.com" + "^(github\\.com/[A-Za-z0-9_.\\-]+/[A-Za-z0-9_.\\-]+)(/[A-Za-z0-9_.\\-]+)*$" + 'git) + (make-vcs + "bitbucket.org" + "^(bitbucket\\.org/([A-Za-z0-9_.\\-]+/[A-Za-z0-9_.\\-]+))(/[A-Za-z0-9_.\\-]+)*$" + 'unknown) + (make-vcs + "hub.jazz.net/git/" + "^(hub\\.jazz\\.net/git/[a-z0-9]+/[A-Za-z0-9_.\\-]+)(/[A-Za-z0-9_.\\-]+)*$" + 'git) + (make-vcs + "git.apache.org" + "^(git\\.apache\\.org/[a-z0-9_.\\-]+\\.git)(/[A-Za-z0-9_.\\-]+)*$" + 'git) + (make-vcs + "git.openstack.org" + "^(git\\.openstack\\.org/[A-Za-z0-9_.\\-]+/[A-Za-z0-9_.\\-]+)(\\.git)?\ +(/[A-Za-z0-9_.\\-]+)*$" + 'git))) + +(define (module-path->repository-root module-path) + "Infer the repository root from a module path. Go modules can be +defined at any level of a repository tree, but querying for the meta tag +usually can only be done from the web page at the root of the repository, +hence the need to derive this information." + + ;; For reference, see: https://golang.org/ref/mod#vcs-find. + (define vcs-qualifiers '(".bzr" ".fossil" ".git" ".hg" ".svn")) + + (define (vcs-qualified-module-path->root-repo-url module-path) + (let* ((vcs-qualifiers-group (string-join vcs-qualifiers "|")) + (pattern (format #f "^(.*(~a))(/|$)" vcs-qualifiers-group)) + (m (string-match pattern module-path))) + (and=> m (cut match:substring <> 1)))) + + (or (and=> (find (lambda (vcs) + (string-prefix? (vcs-url-prefix vcs) module-path)) + known-vcs) + (lambda (vcs) + (match:substring (regexp-exec (vcs-root-regex vcs) + module-path) 1))) + (vcs-qualified-module-path->root-repo-url module-path) + module-path)) + +(define (go-module->guix-package-name module-path) + "Converts a module's path to the canonical Guix format for Go packages." + (string-downcase (string-append "go-" (string-replace-substring + (string-replace-substring + (string-replace-substring + module-path + "." "-") + "/" "-") + "_" "-")))) + +(define-record-type <module-meta> + (make-module-meta import-prefix vcs repo-root) + module-meta? + (import-prefix module-meta-import-prefix) + (vcs module-meta-vcs) ;a symbol + (repo-root module-meta-repo-root)) + +(define (fetch-module-meta-data module-path) + "Retrieve the module meta-data from its landing page. This is necessary +because goproxy servers don't currently provide all the information needed to +build a package." + ;; <meta name="go-import" content="import-prefix vcs repo-root"> + (let* ((port (http-fetch (format #f "https://~a?go-get=1" module-path))) + (select (sxpath `(// head (meta (@ (equal? (name "go-import")))) + // content)))) + (match (select (call-with-port port html->sxml)) + (() #f) ;nothing selected + (((content content-text)) + (match (string-split content-text #\space) + ((root-path vcs repo-url) + (make-module-meta root-path (string->symbol vcs) repo-url))))))) + +(define (module-meta-data-repo-url meta-data goproxy-url) + "Return the URL where the fetcher which will be used can download the +source." + (if (member (module-meta-vcs meta-data) '(fossil mod)) + goproxy-url + (module-meta-repo-root meta-data))) + +;; XXX: Copied from (guix scripts hash). +(define (vcs-file? file stat) + (case (stat:type stat) + ((directory) + (member (basename file) '(".bzr" ".git" ".hg" ".svn" "CVS"))) + ((regular) + ;; Git sub-modules have a '.git' file that is a regular text file. + (string=? (basename file) ".git")) + (else + #f))) + +;; XXX: Adapted from 'file-hash' in (guix scripts hash). +(define* (file-hash file #:optional (algorithm (hash-algorithm sha256))) + ;; Compute the hash of FILE. + (let-values (((port get-hash) (open-hash-port algorithm))) + (write-file file port #:select? (negate vcs-file?)) + (force-output port) + (get-hash))) + +(define* (git-checkout-hash url reference algorithm) + "Return the ALGORITHM hash of the checkout of URL at REFERENCE, a commit or +tag." + (define cache + (string-append (or (getenv "TMPDIR") "/tmp") + "/guix-import-go-" + (passwd:name (getpwuid (getuid))))) + + ;; Use a custom cache to avoid cluttering the default one under + ;; ~/.cache/guix, but choose one under /tmp so that it's persistent across + ;; subsequent "guix import" invocations. + (mkdir-p cache) + (chmod cache #o700) + (let-values (((checkout commit _) + (parameterize ((%repository-cache-directory cache)) + (update-cached-checkout url + #:ref + `(tag-or-commit . ,reference))))) + (file-hash checkout algorithm))) + +(define (vcs->origin vcs-type vcs-repo-url version) + "Generate the `origin' block of a package depending on what type of source +control system is being used." + (case vcs-type + ((git) + (let ((plain-version? (string=? version (go-version->git-ref version))) + (v-prefixed? (string-prefix? "v" version))) + `(origin + (method git-fetch) + (uri (git-reference + (url ,vcs-repo-url) + (commit ,(if (and plain-version? v-prefixed?) + '(string-append "v" version) + '(go-version->git-ref version))))) + (file-name (git-file-name name version)) + (sha256 + (base32 + ,(bytevector->nix-base32-string + (git-checkout-hash vcs-repo-url (go-version->git-ref version) + (hash-algorithm sha256)))))))) + ((hg) + `(origin + (method hg-fetch) + (uri (hg-reference + (url ,vcs-repo-url) + (changeset ,version))) + (file-name (string-append name "-" version "-checkout")) + (sha256 + (base32 + ;; FIXME: populate hash for hg repo checkout + "0000000000000000000000000000000000000000000000000000")))) + ((svn) + `(origin + (method svn-fetch) + (uri (svn-reference + (url ,vcs-repo-url) + (revision (string->number version)))) + (file-name (string-append name "-" version "-checkout")) + (sha256 + (base32 + ;; FIXME: populate hash for svn repo checkout + "0000000000000000000000000000000000000000000000000000")))) + (else + (raise + (formatted-message (G_ "unsupported vcs type '~a' for package '~a'") + vcs-type vcs-repo-url))))) + +(define* (go-module->guix-package module-path #:key + (goproxy-url "https://proxy.golang.org")) + (let* ((latest-version (go-module-latest-version goproxy-url module-path)) + (port (fetch-go.mod goproxy-url module-path latest-version)) + (dependencies (map car (call-with-port port parse-go.mod))) + (guix-name (go-module->guix-package-name module-path)) + (root-module-path (module-path->repository-root module-path)) + ;; The VCS type and URL are not included in goproxy information. For + ;; this we need to fetch it from the official module page. + (meta-data (fetch-module-meta-data root-module-path)) + (vcs-type (module-meta-vcs meta-data)) + (vcs-repo-url (module-meta-data-repo-url meta-data goproxy-url)) + (synopsis (go-package-synopsis root-module-path)) + (description (go-package-description module-path)) + (licenses (go-package-licenses module-path))) + (values + `(package + (name ,guix-name) + ;; Elide the "v" prefix Go uses + (version ,(string-trim latest-version #\v)) + (source + ,(vcs->origin vcs-type vcs-repo-url latest-version)) + (build-system go-build-system) + (arguments + '(#:import-path ,root-module-path)) + ,@(maybe-inputs (map go-module->guix-package-name dependencies)) + (home-page ,(format #f "https://~a" root-module-path)) + (synopsis ,synopsis) + (description ,description) + (license ,(match (and=> licenses list->licenses) + ((license) license) + ((licenses ...) `(list ,@licenses)) + (x x)))) + dependencies))) + +(define go-module->guix-package* (memoize go-module->guix-package)) + +(define* (go-module-recursive-import package-name + #:key (goproxy-url "https://proxy.golang.org")) + (recursive-import + package-name + #:repo->guix-package (lambda* (name . _) + (go-module->guix-package* + name + #:goproxy-url goproxy-url)) + #:guix-name go-module->guix-package-name)) diff --git a/guix/import/hackage.scm b/guix/import/hackage.scm index 6ca4f65cb0..9f992ffe8e 100644 --- a/guix/import/hackage.scm +++ b/guix/import/hackage.scm @@ -344,8 +344,8 @@ respectively." (memoize hackage->guix-package)) (define* (hackage-recursive-import package-name . args) - (recursive-import package-name #f - #:repo->guix-package (lambda (name repo) + (recursive-import package-name + #:repo->guix-package (lambda* (name #:key repo version) (apply hackage->guix-package/m (cons name args))) #:guix-name hackage-name->package-name)) diff --git a/guix/import/print.scm b/guix/import/print.scm index a2ab810a5c..dcc38abc70 100644 --- a/guix/import/print.scm +++ b/guix/import/print.scm @@ -79,7 +79,16 @@ when evaluated." (file-name (origin-file-name source)) (patches (origin-patches source))) `(origin - (method ,(procedure-name method)) + ;; Since 'procedure-name' returns the procedure name within the + ;; module where it's defined, not its public name. Thus, try hard to + ;; find its public name and use 'procedure-name' as a last resort. + (method ,(or (any (lambda (module) + (variable-name method module)) + '((guix download) + (guix git-download) + (guix hg-download) + (guix svn-download))) + (procedure-name method))) (uri (string-append ,@(match (factorize-uri uri version) ((? string? uri) (list uri)) (factorized factorized)))) diff --git a/guix/import/utils.scm b/guix/import/utils.scm index 2f5ccf7cea..6b85b3aa1d 100644 --- a/guix/import/utils.scm +++ b/guix/import/utils.scm @@ -136,6 +136,7 @@ of the string VERSION is replaced by the symbol 'version." ("Apache-1.1" 'license:asl1.1) ("Apache-2.0" 'license:asl2.0) ("BSL-1.0" 'license:boost1.0) + ("0BSD" 'license:bsd-0) ("BSD-2-Clause-FreeBSD" 'license:bsd-2) ("BSD-3-Clause" 'license:bsd-3) ("BSD-4-Clause" 'license:bsd-4) @@ -168,6 +169,7 @@ of the string VERSION is replaced by the symbol 'version." ("Imlib2" 'license:imlib2) ("IPA" 'license:ipa) ("IPL-1.0" 'license:ibmpl1.0) + ("LAL-1.3" 'license:lal1.3) ("LGPL-2.0" 'license:lgpl2.0) ("LGPL-2.0+" 'license:lgpl2.0+) ("LGPL-2.1" 'license:lgpl2.1) diff --git a/guix/inferior.scm b/guix/inferior.scm index 0990696e6c..eb457f81f9 100644 --- a/guix/inferior.scm +++ b/guix/inferior.scm @@ -120,6 +120,15 @@ (packages inferior-package-promise) ;promise of inferior packages (table inferior-package-table)) ;promise of vhash +(define (write-inferior inferior port) + (match inferior + (($ <inferior> pid _ _ version) + (format port "#<inferior ~a ~a ~a>" + pid version + (number->string (object-address inferior) 16))))) + +(set-record-type-printer! <inferior> write-inferior) + (define* (inferior-pipe directory command error-port) "Return an input/output pipe on the Guix instance in DIRECTORY. This runs 'DIRECTORY/COMMAND repl' if it exists, or falls back to some other method if @@ -740,8 +749,16 @@ determines whether CHANNELS are authenticated." (string-append directory "/" file)) (scandir directory base32-encoded-sha256?))) + (define (symlink/safe old new) + (catch 'system-error + (lambda () + (symlink old new)) + (lambda args + (unless (= EEXIST (system-error-errno args)) + (apply throw args))))) + (define symlink* - (lift2 symlink %store-monad)) + (lift2 symlink/safe %store-monad)) (define add-indirect-root* (store-lift add-indirect-root)) diff --git a/guix/licenses.scm b/guix/licenses.scm index 1091eee67c..4718ccf83f 100644 --- a/guix/licenses.scm +++ b/guix/licenses.scm @@ -16,6 +16,7 @@ ;;; Copyright © 2017 Rutger Helling <rhelling@mykolab.com> ;;; Copyright © 2020 André Batista <nandre@riseup.net> ;;; Copyright © 2020 Helio Machado <0x2b3bfa0+guix@googlemail.com> +;;; Copyright © 2021 Felix Gruber <felgru@posteo.net> ;;; ;;; This file is part of GNU Guix. ;;; @@ -39,7 +40,7 @@ apsl2 asl1.1 asl2.0 boost1.0 - bsd-2 bsd-3 bsd-4 + bsd-0 bsd-2 bsd-3 bsd-4 non-copyleft cc0 cc-by2.0 cc-by3.0 cc-by4.0 @@ -68,6 +69,7 @@ imlib2 ipa knuth + lal1.3 lgpl2.0 lgpl2.0+ lgpl2.1 lgpl2.1+ lgpl3 lgpl3+ llgpl lppl lppl1.0+ lppl1.1+ lppl1.2 lppl1.2+ lppl1.3 lppl1.3+ @@ -159,6 +161,11 @@ "http://directory.fsf.org/wiki/License:Boost1.0" "https://www.gnu.org/licenses/license-list#boost")) +(define bsd-0 + (license "Zero-Clause BSD" + "https://spdx.org/licenses/0BSD.html" + "https://opensource.org/licenses/0BSD")) + (define bsd-2 (license "FreeBSD" "http://directory.fsf.org/wiki/License:FreeBSD" @@ -406,6 +413,11 @@ at URI, which may be a file:// URI pointing the package's tree." "http://www.ctan.org/license/knuth" "Modification are only permitted under a different name.")) +(define lal1.3 + (license "Free Art License 1.3" + "http://artlibre.org/licence/lal/en/" + "https://www.gnu.org/licenses/license-list#FreeArt")) + (define lgpl2.0 (license "LGPL 2.0" "https://www.gnu.org/licenses/old-licenses/lgpl-2.0.html" diff --git a/guix/lint.scm b/guix/lint.scm index 311bc94cc3..a7d6bbba4f 100644 --- a/guix/lint.scm +++ b/guix/lint.scm @@ -6,7 +6,7 @@ ;;; Copyright © 2016 Danny Milosavljevic <dannym+a@scratchpost.org> ;;; Copyright © 2016 Hartmut Goebel <h.goebel@crazy-compilers.com> ;;; Copyright © 2017 Alex Kost <alezost@gmail.com> -;;; Copyright © 2017 Tobias Geerinckx-Rice <me@tobias.gr> +;;; Copyright © 2017, 2021 Tobias Geerinckx-Rice <me@tobias.gr> ;;; Copyright © 2017, 2018, 2020 Efraim Flashner <efraim@flashner.co.il> ;;; Copyright © 2018, 2019 Arun Isaac <arunisaac@systemreboot.net> ;;; Copyright © 2020 Chris Marusich <cmmarusich@gmail.com> @@ -170,6 +170,18 @@ (requires-store? lint-checker-requires-store? (default #f))) +(define (check-name package) + "Check whether PACKAGE's name matches our guidelines." + (let ((name (package-name package))) + ;; Currently checks only whether the name is too short. + (if (and (<= (string-length name) 1) + (not (string=? name "r"))) ; common-sense exception + (list + (make-warning package + (G_ "name should be longer than a single character") + #:field 'name)) + '()))) + (define (properly-starts-sentence? s) (string-match "^[(\"'`[:upper:][:digit:]]" s)) @@ -1179,21 +1191,32 @@ vulnerability records for PACKAGE by calling PACKAGE-VULNERABILITIES." (define (check-for-updates package) "Check if there is an update available for PACKAGE." - (match (with-networking-fail-safe - (format #f (G_ "while retrieving upstream info for '~a'") - (package-name package)) - #f - (package-latest-release* package)) - ((? upstream-source? source) - (if (version>? (upstream-source-version source) - (package-version package)) - (list - (make-warning package - (G_ "can be upgraded to ~a") - (list (upstream-source-version source)) - #:field 'version)) - '())) - (#f '()))) ; cannot find newer upstream release + (match (lookup-updater package) + (#f + (list (make-warning package (G_ "no updater for ~a") + (list (package-name package)) + #:field 'source))) + ((? upstream-updater? updater) + (match (with-networking-fail-safe + (format #f (G_ "while retrieving upstream info for '~a'") + (package-name package)) + #f + (package-latest-release package)) + ((? upstream-source? source) + (if (version>? (upstream-source-version source) + (package-version package)) + (list + (make-warning package + (G_ "can be upgraded to ~a") + (list (upstream-source-version source)) + #:field 'version)) + '())) + (#f ;cannot find upstream release + (list (make-warning package + (G_ "updater '~a' failed to find \ +upstream releases") + (list (upstream-updater-name updater)) + #:field 'source))))))) (define (check-archival package) @@ -1264,7 +1287,8 @@ try again later") ((? origin? origin) ;; Since "save" origins are not supported for non-VCS source, all ;; we can do is tell whether a given tarball is available or not. - (if (origin-hash origin) ;XXX: for ungoogled-chromium + (if (and=> (origin-hash origin) ;XXX: for ungoogled-chromium + content-hash-value) ;& icecat (let ((hash (origin-hash origin))) (match (lookup-content (content-hash-value hash) (symbol->string @@ -1446,6 +1470,10 @@ them for PACKAGE." (define %local-checkers (list (lint-checker + (name 'name) + (description "Validate package names") + (check check-name)) + (lint-checker (name 'description) (description "Validate package descriptions") (check check-description-style)) diff --git a/guix/narinfo.scm b/guix/narinfo.scm index d3deba28bd..72e0f75fda 100644 --- a/guix/narinfo.scm +++ b/guix/narinfo.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021 Ludovic Courtès <ludo@gnu.org> ;;; Copyright © 2014 Nikita Karetnikov <nikita@karetnikov.org> ;;; Copyright © 2018 Kyle Meyer <kyle@kyleam.com> ;;; @@ -25,7 +25,6 @@ #:use-module (guix base64) #:use-module (guix records) #:use-module (guix diagnostics) - #:use-module (guix scripts substitute) #:use-module (gcrypt hash) #:use-module (gcrypt pk-crypto) #:use-module (rnrs bytevectors) @@ -298,9 +297,21 @@ this is a rough approximation." (_ (or (string=? compression2 "none") (string=? compression2 "gzip"))))) -(define (narinfo-best-uri narinfo) +(define (decompresses-faster? compression1 compression2) + "Return true if COMPRESSION1 generally has a higher decompression throughput +than COMPRESSION2." + (match compression1 + ("none" #t) + ("zstd" #t) + ("gzip" (string=? compression2 "lzip")) + (_ #f))) + +(define* (narinfo-best-uri narinfo #:key fast-decompression?) "Select the \"best\" URI to download NARINFO's nar, and return three values: -the URI, its compression method (a string), and the compressed file size." +the URI, its compression method (a string), and the compressed file size. +When FAST-DECOMPRESSION? is true, prefer substitutes with faster +decompression (typically zstd) rather than substitutes with a higher +compression ratio (typically lzip)." (define choices (filter (match-lambda ((uri compression file-size) @@ -322,6 +333,13 @@ the URI, its compression method (a string), and the compressed file size." (compresses-better? compression1 compression2)))) (_ #f))) ;we can't tell - (match (sort choices file-size<?) + (define (speed<? c1 c2) + (match c1 + ((uri1 compression1 . _) + (match c2 + ((uri2 compression2 . _) + (decompresses-faster? compression2 compression1)))))) + + (match (sort choices (if fast-decompression? (negate speed<?) file-size<?)) (((uri compression file-size) _ ...) (values uri compression file-size)))) diff --git a/guix/openpgp.scm b/guix/openpgp.scm index 648c359621..9de7feb644 100644 --- a/guix/openpgp.scm +++ b/guix/openpgp.scm @@ -538,17 +538,6 @@ signature." (raise (condition (&openpgp-invalid-signature-error (port port)))))))) -(define (hash-algorithm-name algorithm) ;XXX: should be in Guile-Gcrypt - "Return the name of ALGORITHM, a 'hash-algorithm' integer, as a symbol." - (letrec-syntax ((->name (syntax-rules () - ((_) #f) - ((_ name rest ...) - (if (= algorithm (hash-algorithm name)) - 'name - (->name rest ...)))))) - (->name sha1 sha256 sha384 sha512 sha224 - sha3-224 sha3-256 sha3-384 sha3-512))) - (define (verify-openpgp-signature sig keyring dataport) "Verify that the data read from DATAPORT matches SIG, an <openpgp-signature>. Fetch the public key of the issuer of SIG from KEYRING, diff --git a/guix/packages.scm b/guix/packages.scm index 1b76afd045..55e5e70b8c 100644 --- a/guix/packages.scm +++ b/guix/packages.scm @@ -5,6 +5,7 @@ ;;; Copyright © 2016 Alex Kost <alezost@gmail.com> ;;; Copyright © 2017, 2019, 2020 Efraim Flashner <efraim@flashner.co.il> ;;; Copyright © 2019 Marius Bakke <mbakke@fastmail.com> +;;; Copyright © 2021 Chris Marusich <cmmarusich@gmail.com> ;;; ;;; This file is part of GNU Guix. ;;; @@ -345,7 +346,8 @@ name of its URI." (define %supported-systems ;; This is the list of system types that are supported. By default, we ;; expect all packages to build successfully here. - '("x86_64-linux" "i686-linux" "armhf-linux" "aarch64-linux" "mips64el-linux" "i586-gnu")) + '("x86_64-linux" "i686-linux" "armhf-linux" "aarch64-linux" "mips64el-linux" "i586-gnu" + "powerpc64le-linux")) (define %hurd-systems ;; The GNU/Hurd systems for which support is being developed. @@ -425,7 +427,7 @@ name of its URI." (define-syntax-rule (package/inherit p overrides ...) "Like (package (inherit P) OVERRIDES ...), except that the same -transformation is done to the package replacement, if any. P must be a bare +transformation is done to the package P's replacement, if any. P must be a bare identifier, and will be bound to either P or its replacement when evaluating OVERRIDES." (let loop ((p p)) @@ -475,29 +477,34 @@ object." (match (package-location package) (($ <location> file line column) - (catch 'system-error - (lambda () - ;; In general we want to keep relative file names for modules. - (call-with-input-file (search-path %load-path file) - (lambda (port) - (goto port line column) - (match (read port) - (('package inits ...) - (let ((field (assoc field inits))) - (match field - ((_ value) - (let ((loc (and=> (source-properties value) - source-properties->location))) - (and loc - ;; Preserve the original file name, which may be a - ;; relative file name. - (set-field loc (location-file) file)))) - (_ - #f)))) - (_ - #f))))) - (lambda _ - #f))) + (match (search-path %load-path file) + ((? string? file-found) + (catch 'system-error + (lambda () + ;; In general we want to keep relative file names for modules. + (call-with-input-file file-found + (lambda (port) + (goto port line column) + (match (read port) + (('package inits ...) + (let ((field (assoc field inits))) + (match field + ((_ value) + (let ((loc (and=> (source-properties value) + source-properties->location))) + (and loc + ;; Preserve the original file name, which may be a + ;; relative file name. + (set-field loc (location-file) file)))) + (_ + #f)))) + (_ + #f))))) + (lambda _ + #f))) + (#f + ;; FILE could not be found in %LOAD-PATH. + #f))) (_ #f))) diff --git a/guix/profiles.scm b/guix/profiles.scm index ea8bc6e593..67d90532c1 100644 --- a/guix/profiles.scm +++ b/guix/profiles.scm @@ -362,9 +362,16 @@ file name." #t lst))) +(define (default-properties package) + "Return the default properties of a manifest entry for PACKAGE." + ;; Preserve transformation options by default. + (match (assq-ref (package-properties package) 'transformations) + (#f '()) + (transformations `((transformations . ,transformations))))) + (define* (package->manifest-entry package #:optional (output "out") #:key (parent (delay #f)) - (properties '())) + (properties (default-properties package))) "Return a manifest entry for the OUTPUT of package PACKAGE." ;; For each dependency, keep a promise pointing to its "parent" entry. (letrec* ((deps (map (match-lambda diff --git a/guix/scripts.scm b/guix/scripts.scm index c9ea9f2e29..3aabaf5c9c 100644 --- a/guix/scripts.scm +++ b/guix/scripts.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2013, 2014, 2015, 2017, 2018, 2019, 2020 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2013, 2014, 2015, 2017, 2018, 2019, 2020, 2021, 2021 Ludovic Courtès <ludo@gnu.org> ;;; Copyright © 2014 Deck Pickard <deck.r.pickard@gmail.com> ;;; Copyright © 2015, 2016 Alex Kost <alezost@gmail.com> ;;; Copyright © 2020 Jan (janneke) Nieuwenhuizen <janneke@gnu.org> @@ -118,7 +118,12 @@ procedure, but both the category and synopsis are meant to be read (parsed) by according to'string-distance'." (define (options->long-names options) (filter string? (append-map option-names options))) - (string-closest guess (options->long-names options) #:threshold 3)) + (match guess + ((? string?) + (match (string-split guess #\=) + ((name rest ...) + (string-closest name (options->long-names options) #:threshold 3)))) + (_ #f))) (define (args-fold* args options unrecognized-option-proc operand-proc . seeds) "A wrapper on top of `args-fold' that does proper user-facing error diff --git a/guix/scripts/archive.scm b/guix/scripts/archive.scm index 91be1b02e1..ceac640432 100644 --- a/guix/scripts/archive.scm +++ b/guix/scripts/archive.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2013, 2014, 2015, 2016, 2017, 2019, 2020 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2013, 2014, 2015, 2016, 2017, 2019, 2020, 2021 Ludovic Courtès <ludo@gnu.org> ;;; Copyright © 2020 Tobias Geerinckx-Rice <me@tobias.gr> ;;; ;;; This file is part of GNU Guix. @@ -63,7 +63,7 @@ (print-build-trace? . #t) (print-extended-build-trace? . #t) (multiplexed-build-output? . #t) - (verbosity . 2) + (verbosity . 3) (debug . 0))) (define (show-help) diff --git a/guix/scripts/build.scm b/guix/scripts/build.scm index a959cb827d..2decdb45ed 100644 --- a/guix/scripts/build.scm +++ b/guix/scripts/build.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2021 Ludovic Courtès <ludo@gnu.org> ;;; Copyright © 2013 Mark H Weaver <mhw@netris.org> ;;; Copyright © 2020 Marius Bakke <mbakke@fastmail.com> ;;; Copyright © 2020 Ricardo Wurmus <rekado@elephly.net> @@ -22,7 +22,7 @@ (define-module (guix scripts build) #:use-module (guix ui) #:use-module (guix scripts) - #:use-module (guix import json) + #:autoload (guix import json) (json->scheme-file) #:use-module (guix store) #:use-module (guix derivations) #:use-module (guix packages) @@ -333,7 +333,7 @@ use '--no-offload' instead~%"))) (print-build-trace? . #t) (print-extended-build-trace? . #t) (multiplexed-build-output? . #t) - (verbosity . 2) + (verbosity . 3) (debug . 0))) (define (show-help) diff --git a/guix/scripts/challenge.scm b/guix/scripts/challenge.scm index cc9cbe6f27..4ec3be99ca 100644 --- a/guix/scripts/challenge.scm +++ b/guix/scripts/challenge.scm @@ -27,7 +27,7 @@ #:use-module (guix packages) #:use-module ((guix progress) #:hide (dump-port*)) #:use-module (guix serialization) - #:use-module (guix scripts substitute) + #:use-module (guix substitutes) #:use-module (guix narinfo) #:use-module (rnrs bytevectors) #:autoload (guix http-client) (http-fetch) diff --git a/guix/scripts/copy.scm b/guix/scripts/copy.scm index 2780d4fbe9..52b476db54 100644 --- a/guix/scripts/copy.scm +++ b/guix/scripts/copy.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2016, 2017, 2019, 2020 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2016, 2017, 2019, 2020, 2021 Ludovic Courtès <ludo@gnu.org> ;;; ;;; This file is part of GNU Guix. ;;; @@ -163,7 +163,7 @@ Copy ITEMS to or from the specified host over SSH.\n")) (print-extended-build-trace? . #t) (multiplexed-build-output? . #t) (debug . 0) - (verbosity . 2))) + (verbosity . 3))) ;;; diff --git a/guix/scripts/describe.scm b/guix/scripts/describe.scm index 6f8d9aceec..b5f6249176 100644 --- a/guix/scripts/describe.scm +++ b/guix/scripts/describe.scm @@ -2,6 +2,7 @@ ;;; Copyright © 2018, 2019, 2020, 2021 Ludovic Courtès <ludo@gnu.org> ;;; Copyright © 2018 Oleg Pykhalov <go.wigust@gmail.com> ;;; Copyright © 2020 Ekaitz Zarraga <ekaitz@elenq.tech> +;;; Copyright © 2021 Simon Tournier <zimon.toutoune@gmail.com> ;;; ;;; This file is part of GNU Guix. ;;; @@ -28,7 +29,7 @@ #:use-module (guix profiles) #:autoload (guix openpgp) (openpgp-format-fingerprint) #:use-module (git) - #:use-module (json) + #:autoload (json builder) (scm->json-string) #:use-module (srfi srfi-1) #:use-module (srfi srfi-26) #:use-module (srfi srfi-37) @@ -286,12 +287,9 @@ text. The hyperlink links to a web view of COMMIT, when available." (define-command (guix-describe . args) (synopsis "describe the channel revisions currently used") - (let* ((opts (args-fold* args %options - (lambda (opt name arg result) - (leave (G_ "~A: unrecognized option~%") - name)) - cons - %default-options)) + (let* ((opts (parse-command-line args %options (list %default-options) + #:build-options? #f + #:argument-handler cons)) (format (assq-ref opts 'format)) (profile (or (assq-ref opts 'profile) (current-profile)))) (with-error-handling diff --git a/guix/scripts/discover.scm b/guix/scripts/discover.scm index 6aade81ed1..be1eaa6e95 100644 --- a/guix/scripts/discover.scm +++ b/guix/scripts/discover.scm @@ -1,5 +1,6 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2020 Mathieu Othacehe <othacehe@gnu.org> +;;; Copyright © 2021 Simon Tournier <zimon.toutoune@gmail.com> ;;; ;;; This file is part of GNU Guix. ;;; @@ -127,12 +128,11 @@ to synchronize with the writer." (synopsis "discover Guix related services using Avahi") (with-error-handling - (let* ((opts (args-fold* args %options - (lambda (opt name arg result) - (leave (G_ "~A: unrecognized option~%") name)) - (lambda (arg result) - (leave (G_ "~A: extraneous argument~%") arg)) - %default-options)) + (let* ((opts (parse-command-line args %options (list %default-options) + #:build-options? #f + #:argument-handler + (lambda (arg result) + (leave (G_ "~A: extraneous argument~%") arg)))) (cache (assoc-ref opts 'cache)) (publish-file (publish-file cache))) (parameterize ((%publish-file publish-file)) diff --git a/guix/scripts/download.scm b/guix/scripts/download.scm index ce8dd8b02c..5a91390358 100644 --- a/guix/scripts/download.scm +++ b/guix/scripts/download.scm @@ -1,5 +1,6 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2012, 2013, 2015, 2016, 2017, 2020 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2021 Simon Tournier <zimon.toutoune@gmail.com> ;;; ;;; This file is part of GNU Guix. ;;; @@ -162,15 +163,13 @@ and 'base16' ('hex' and 'hexadecimal' can be used as well).\n")) (define (parse-options) ;; Return the alist of option values. - (args-fold* args %options - (lambda (opt name arg result) - (leave (G_ "~A: unrecognized option~%") name)) - (lambda (arg result) - (when (assq 'argument result) - (leave (G_ "~A: extraneous argument~%") arg)) - - (alist-cons 'argument arg result)) - %default-options)) + (parse-command-line args %options (list %default-options) + #:build-options? #f + #:argument-handler + (lambda (arg result) + (when (assq 'argument result) + (leave (G_ "~A: extraneous argument~%") arg)) + (alist-cons 'argument arg result)))) (with-error-handling (let* ((opts (parse-options)) diff --git a/guix/scripts/edit.scm b/guix/scripts/edit.scm index 49c9d945b6..b4c0507591 100644 --- a/guix/scripts/edit.scm +++ b/guix/scripts/edit.scm @@ -1,7 +1,7 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2015, 2016, 2019, 2020 Ludovic Courtès <ludo@gnu.org> ;;; Copyright © 2015 Mathieu Lirzin <mthl@gnu.org> -;;; Copyright © 2020 Simon Tournier <zimon.toutoune@gmail.com> +;;; Copyright © 2020, 2021 Simon Tournier <zimon.toutoune@gmail.com> ;;; ;;; This file is part of GNU Guix. ;;; @@ -84,11 +84,9 @@ line." (define (parse-arguments) ;; Return the list of package names. - (args-fold* args %options - (lambda (opt name arg result) - (leave (G_ "~A: unrecognized option~%") name)) - cons - '())) + (parse-command-line args %options (list (list)) + #:build-options? #f + #:argument-handler cons)) (with-error-handling (let* ((specs (reverse (parse-arguments))) diff --git a/guix/scripts/environment.scm b/guix/scripts/environment.scm index a39347743e..0360761683 100644 --- a/guix/scripts/environment.scm +++ b/guix/scripts/environment.scm @@ -1,6 +1,6 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2014, 2015, 2018 David Thompson <davet@gnu.org> -;;; Copyright © 2015, 2016, 2017, 2018, 2019, 2020 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2015, 2016, 2017, 2018, 2019, 2020, 2021 Ludovic Courtès <ludo@gnu.org> ;;; Copyright © 2018 Mike Gerwitz <mtg@gnu.org> ;;; ;;; This file is part of GNU Guix. @@ -745,14 +745,15 @@ message if any test fails." (with-status-verbosity (assoc-ref opts 'verbosity) (define manifest-from-opts (options/resolve-packages store opts)) - (when (and profile - (> (length (manifest-entries manifest-from-opts)) 0)) - (leave (G_ "'--profile' cannot be used with package options~%"))) (define manifest (if profile - (profile-manifest profile) - manifest-from-opts)) + (profile-manifest profile) + manifest-from-opts)) + + (when (and profile + (> (length (manifest-entries manifest-from-opts)) 0)) + (leave (G_ "'--profile' cannot be used with package options~%"))) (set-build-options-from-command-line store opts) diff --git a/guix/scripts/import.scm b/guix/scripts/import.scm index 0a3863f965..1d2b45d942 100644 --- a/guix/scripts/import.scm +++ b/guix/scripts/import.scm @@ -77,7 +77,7 @@ rather than \\n." ;;; (define importers '("gnu" "nix" "pypi" "cpan" "hackage" "stackage" "elpa" "gem" - "cran" "crate" "texlive" "json" "opam")) + "go" "cran" "crate" "texlive" "json" "opam")) (define (resolve-importer name) (let ((module (resolve-interface diff --git a/guix/scripts/import/cran.scm b/guix/scripts/import/cran.scm index 4767bc082d..aa3ef324e0 100644 --- a/guix/scripts/import/cran.scm +++ b/guix/scripts/import/cran.scm @@ -50,6 +50,8 @@ Import and convert the CRAN package for PACKAGE-NAME.\n")) (display (G_ " -r, --recursive import packages recursively")) (display (G_ " + -s, --style=STYLE choose output style, either specification or variable")) + (display (G_ " -V, --version display version information and exit")) (newline) (show-bug-report-information)) diff --git a/guix/scripts/import/go.scm b/guix/scripts/import/go.scm new file mode 100644 index 0000000000..afdba4e8f1 --- /dev/null +++ b/guix/scripts/import/go.scm @@ -0,0 +1,118 @@ +;;; GNU Guix --- Functional package management for GNU +;;; Copyright © 2020 Katherine Cox-Buday <cox.katherine.e@gmail.com> +;;; +;;; This file is part of GNU Guix. +;;; +;;; GNU Guix is free software; you can redistribute it and/or modify it +;;; under the terms of the GNU General Public License as published by +;;; the Free Software Foundation; either version 3 of the License, or (at +;;; your option) any later version. +;;; +;;; GNU Guix is distributed in the hope that it will be useful, but +;;; WITHOUT ANY WARRANTY; without even the implied warranty of +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;;; GNU General Public License for more details. +;;; +;;; You should have received a copy of the GNU General Public License +;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>. + +(define-module (guix scripts import go) + #:use-module (guix ui) + #:use-module (guix utils) + #:use-module (guix scripts) + #:use-module (guix import go) + #:use-module (guix scripts import) + #:use-module (srfi srfi-1) + #:use-module (srfi srfi-11) + #:use-module (srfi srfi-37) + #:use-module (ice-9 match) + #:use-module (ice-9 format) + #:export (guix-import-go)) + + +;;; +;;; Command-line options. +;;; + +(define %default-options + '()) + +(define (show-help) + (display (G_ "Usage: guix import go PACKAGE-PATH +Import and convert the Go module for PACKAGE-PATH.\n")) + (display (G_ " + -h, --help display this help and exit")) + (display (G_ " + -V, --version display version information and exit")) + (display (G_ " + -r, --recursive generate package expressions for all Go modules\ + that are not yet in Guix")) + (display (G_ " + -p, --goproxy=GOPROXY specify which goproxy server to use")) + (newline) + (show-bug-report-information)) + +(define %options + ;; Specification of the command-line options. + (cons* (option '(#\h "help") #f #f + (lambda args + (show-help) + (exit 0))) + (option '(#\V "version") #f #f + (lambda args + (show-version-and-exit "guix import go"))) + (option '(#\r "recursive") #f #f + (lambda (opt name arg result) + (alist-cons 'recursive #t result))) + (option '(#\p "goproxy") #t #f + (lambda (opt name arg result) + (alist-cons 'goproxy + (string->symbol arg) + (alist-delete 'goproxy result)))) + %standard-import-options)) + + +;;; +;;; Entry point. +;;; + +(define (guix-import-go . args) + (define (parse-options) + ;; Return the alist of option values. + (args-fold* args %options + (lambda (opt name arg result) + (leave (G_ "~A: unrecognized option~%") name)) + (lambda (arg result) + (alist-cons 'argument arg result)) + %default-options)) + + (let* ((opts (parse-options)) + (args (filter-map (match-lambda + (('argument . value) + value) + (_ #f)) + (reverse opts)))) + (match args + ((module-name) + (if (assoc-ref opts 'recursive) + (map (match-lambda + ((and ('package ('name name) . rest) pkg) + `(define-public ,(string->symbol name) + ,pkg)) + (_ #f)) + (go-module-recursive-import module-name + #:goproxy-url + (or (assoc-ref opts 'goproxy) + "https://proxy.golang.org"))) + (let ((sexp (go-module->guix-package module-name + #:goproxy-url + (or (assoc-ref opts 'goproxy) + "https://proxy.golang.org")))) + (unless sexp + (leave (G_ "failed to download meta-data for module '~a'~%") + module-name)) + sexp))) + (() + (leave (G_ "too few arguments~%"))) + ((many ...) + (leave (G_ "too many arguments~%")))))) diff --git a/guix/scripts/pack.scm b/guix/scripts/pack.scm index 8ecdcb823f..b653138f2c 100644 --- a/guix/scripts/pack.scm +++ b/guix/scripts/pack.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2015, 2017, 2018, 2019, 2020 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2015, 2017, 2018, 2019, 2020, 2021 Ludovic Courtès <ludo@gnu.org> ;;; Copyright © 2017, 2018 Ricardo Wurmus <rekado@elephly.net> ;;; Copyright © 2018 Konrad Hinsen <konrad.hinsen@fastmail.net> ;;; Copyright © 2018 Chris Marusich <cmmarusich@gmail.com> @@ -1170,24 +1170,19 @@ Create a bundle of PACKAGE.\n")) manifest)) identity)) - (define (with-transformations manifest) - (map-manifest-entries manifest-entry-with-transformations - manifest)) - (with-provenance - (with-transformations - (cond - ((and (not (null? manifests)) (not (null? packages))) - (leave (G_ "both a manifest and a package list were given~%"))) - ((not (null? manifests)) - (concatenate-manifests - (map (lambda (file) - (let ((user-module (make-user-module - '((guix profiles) (gnu))))) - (load* file user-module))) - manifests))) - (else - (packages->manifest packages))))))) + (cond + ((and (not (null? manifests)) (not (null? packages))) + (leave (G_ "both a manifest and a package list were given~%"))) + ((not (null? manifests)) + (concatenate-manifests + (map (lambda (file) + (let ((user-module (make-user-module + '((guix profiles) (gnu))))) + (load* file user-module))) + manifests))) + (else + (packages->manifest packages)))))) (with-error-handling (with-store store diff --git a/guix/scripts/package.scm b/guix/scripts/package.scm index 8234a1703d..e3d40d5142 100644 --- a/guix/scripts/package.scm +++ b/guix/scripts/package.scm @@ -35,14 +35,15 @@ #:use-module (guix packages) #:use-module (guix profiles) #:use-module (guix search-paths) - #:use-module (guix import json) + #:autoload (guix import json) (json->scheme-file) #:use-module (guix monads) #:use-module (guix utils) #:use-module (guix config) #:use-module (guix scripts) #:use-module (guix scripts build) #:use-module (guix transformations) - #:use-module (guix describe) + #:autoload (guix describe) (manifest-entry-provenance + manifest-entry-with-provenance) #:autoload (guix channels) (channel-name channel-commit channel->code) #:autoload (guix store roots) (gc-roots user-owned?) #:use-module ((guix build utils) @@ -235,14 +236,12 @@ non-zero relevance score." (case (version-compare candidate-version version) ((>) (manifest-transaction-install-entry - (manifest-entry-with-transformations - (package->manifest-entry* pkg output)) + (package->manifest-entry* pkg output) transaction)) ((<) transaction) ((=) - (let* ((new (manifest-entry-with-transformations - (package->manifest-entry* pkg output)))) + (let* ((new (package->manifest-entry* pkg output))) ;; Here we want to determine whether the NEW actually ;; differs from ENTRY, but we need to intercept ;; 'build-things' calls because they would prevent us from diff --git a/guix/scripts/publish.scm b/guix/scripts/publish.scm index fa85088ed0..39bb224cad 100644 --- a/guix/scripts/publish.scm +++ b/guix/scripts/publish.scm @@ -3,6 +3,7 @@ ;;; Copyright © 2020 by Amar M. Singh <nly@disroot.org> ;;; Copyright © 2015, 2016, 2017, 2018, 2019, 2020 Ludovic Courtès <ludo@gnu.org> ;;; Copyright © 2020 Maxim Cournoyer <maxim.cournoyer@gmail.com> +;;; Copyright © 2021 Simon Tournier <zimon.toutoune@gmail.com> ;;; ;;; This file is part of GNU Guix. ;;; @@ -1117,12 +1118,11 @@ methods, return the applicable compression." (synopsis "publish build results over HTTP") (with-error-handling - (let* ((opts (args-fold* args %options - (lambda (opt name arg result) - (leave (G_ "~A: unrecognized option~%") name)) - (lambda (arg result) - (leave (G_ "~A: extraneous argument~%") arg)) - %default-options)) + (let* ((opts (parse-command-line args %options (list %default-options) + #:build-options? #f + #:argument-handler + (lambda (arg result) + (leave (G_ "~A: extraneous argument~%") arg)))) (advertise? (assoc-ref opts 'advertise?)) (user (assoc-ref opts 'user)) (port (assoc-ref opts 'port)) diff --git a/guix/scripts/pull.scm b/guix/scripts/pull.scm index 13d5eceada..07613240a8 100644 --- a/guix/scripts/pull.scm +++ b/guix/scripts/pull.scm @@ -1,7 +1,7 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2013, 2014, 2015, 2017, 2018, 2019, 2020 Ludovic Courtès <ludo@gnu.org> ;;; Copyright © 2017 Marius Bakke <mbakke@fastmail.com> -;;; Copyright © 2020 Tobias Geerinckx-Rice <me@tobias.gr> +;;; Copyright © 2020, 2021 Tobias Geerinckx-Rice <me@tobias.gr> ;;; ;;; This file is part of GNU Guix. ;;; @@ -91,11 +91,11 @@ Download and deploy the latest version of Guix.\n")) (display (G_ " -C, --channels=FILE deploy the channels defined in FILE")) (display (G_ " - --url=URL download from the Git repository at URL")) + --url=URL download \"guix\" channel from the Git repository at URL")) (display (G_ " - --commit=COMMIT download the specified COMMIT")) + --commit=COMMIT download the specified \"guix\" channel COMMIT")) (display (G_ " - --branch=BRANCH download the tip of the specified BRANCH")) + --branch=BRANCH download the tip of the specified \"guix\" channel BRANCH")) (display (G_ " --allow-downgrades allow downgrades to earlier channel revisions")) (display (G_ " diff --git a/guix/scripts/repl.scm b/guix/scripts/repl.scm index 9f20803efc..50d18c7760 100644 --- a/guix/scripts/repl.scm +++ b/guix/scripts/repl.scm @@ -1,6 +1,6 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2018, 2019, 2020 Ludovic Courtès <ludo@gnu.org> -;;; Copyright © 2020 Simon Tournier <zimon.toutoune@gmail.com> +;;; Copyright © 2020, 2021 Simon Tournier <zimon.toutoune@gmail.com> ;;; Copyright © 2020 Konrad Hinsen <konrad.hinsen@fastmail.net> ;;; ;;; This file is part of GNU Guix. @@ -143,14 +143,13 @@ call THUNK." (synopsis "read-eval-print loop (REPL) for interactive programming") (define opts - (args-fold* args %options - (lambda (opt name arg result) - (leave (G_ "~A: unrecognized option~%") name)) + (parse-command-line args %options (list %default-options) + #:build-options? #f + #:argument-handler (lambda (arg result) (append `((script . ,arg) (ignore-dot-guile? . #t)) - result)) - %default-options)) + result)))) (define user-config (and=> (getenv "HOME") diff --git a/guix/scripts/search.scm b/guix/scripts/search.scm index 0c9e6af07b..27b9da5278 100644 --- a/guix/scripts/search.scm +++ b/guix/scripts/search.scm @@ -1,5 +1,6 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2019, 2020 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2021 Simon Tournier <zimon.toutoune@gmail.com> ;;; ;;; This file is part of GNU Guix. ;;; @@ -66,11 +67,9 @@ This is an alias for 'guix package -s'.\n")) result)) (define opts - (args-fold* args %options - (lambda (opt name arg . rest) - (leave (G_ "~A: unrecognized option~%") name)) - handle-argument - '())) + (parse-command-line args %options (list (list)) + #:build-options? #f + #:argument-handler handle-argument)) (unless (assoc-ref opts 'query) (leave (G_ "missing arguments: no regular expressions to search for~%"))) diff --git a/guix/scripts/show.scm b/guix/scripts/show.scm index 535d03c1a6..c747eedd21 100644 --- a/guix/scripts/show.scm +++ b/guix/scripts/show.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2019 Simon Tournier <zimon.toutoune@gmail.com> +;;; Copyright © 2019, 2021 Simon Tournier <zimon.toutoune@gmail.com> ;;; ;;; This file is part of GNU Guix. ;;; @@ -66,11 +66,9 @@ This is an alias for 'guix package --show='.\n")) result)) (define opts - (args-fold* args %options - (lambda (opt name arg . rest) - (leave (G_ "~A: unrecognized option~%") name)) - handle-argument - '())) + (parse-command-line args %options (list (list)) + #:build-options? #f + #:argument-handler handle-argument)) (unless (assoc-ref opts 'query) (leave (G_ "missing arguments: no package to show~%"))) diff --git a/guix/scripts/substitute.scm b/guix/scripts/substitute.scm index f9bcead045..79eaabd8fd 100755 --- a/guix/scripts/substitute.scm +++ b/guix/scripts/substitute.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021 Ludovic Courtès <ludo@gnu.org> ;;; Copyright © 2014 Nikita Karetnikov <nikita@karetnikov.org> ;;; Copyright © 2018 Kyle Meyer <kyle@kyleam.com> ;;; Copyright © 2020 Christopher Baines <mail@cbaines.net> @@ -24,6 +24,7 @@ #:use-module (guix scripts) #:use-module (guix narinfo) #:use-module (guix store) + #:use-module (guix substitutes) #:use-module (guix utils) #:use-module (guix combinators) #:use-module (guix config) @@ -39,40 +40,29 @@ #:use-module (guix cache) #:use-module (gcrypt pk-crypto) #:use-module (guix pki) - #:use-module ((guix build utils) #:select (mkdir-p dump-port)) + #:use-module ((guix build utils) #:select (mkdir-p)) #:use-module ((guix build download) #:select (uri-abbreviation nar-uri-abbreviation (open-connection-for-uri - . guix:open-connection-for-uri) - store-path-abbreviation byte-count->string)) + . guix:open-connection-for-uri))) #:autoload (gnutls) (error/invalid-session) #:use-module (guix progress) #:use-module ((guix build syscalls) #:select (set-thread-name)) #:use-module (ice-9 rdelim) - #:use-module (ice-9 regex) #:use-module (ice-9 match) #:use-module (ice-9 format) #:use-module (ice-9 ftw) - #:use-module (ice-9 binary-ports) - #:use-module (ice-9 vlist) #:use-module (rnrs bytevectors) #:use-module (srfi srfi-1) - #:use-module (srfi srfi-9) #:use-module (srfi srfi-11) #:use-module (srfi srfi-19) #:use-module (srfi srfi-26) #:use-module (srfi srfi-34) #:use-module (srfi srfi-35) #:use-module (web uri) - #:use-module (web http) - #:use-module (web request) - #:use-module (web response) #:use-module (guix http-client) - #:export (lookup-narinfos - lookup-narinfos/diverse - - %allow-unauthenticated-substitutes? + #:export (%allow-unauthenticated-substitutes? %error-to-file-descriptor-4? substitute-urls @@ -89,16 +79,9 @@ ;;; ;;; Code: -(define %narinfo-cache-directory - ;; A local cache of narinfos, to avoid going to the network. Most of the - ;; time, 'guix substitute' is called by guix-daemon as root and stores its - ;; cached data in /var/guix/…. However, when invoked from 'guix challenge' - ;; as a user, it stores its cache in ~/.cache. - (if (zero? (getuid)) - (or (and=> (getenv "XDG_CACHE_HOME") - (cut string-append <> "/guix/substitute")) - (string-append %state-directory "/substitute/cache")) - (string-append (cache-directory #:ensure? #f) "/substitute"))) +(define %narinfo-expired-cache-entry-removal-delay + ;; How often we want to remove files corresponding to expired cache entries. + (* 7 24 3600)) (define (warn-about-missing-authentication) (warning (G_ "authentication and authorization of substitutes \ @@ -112,24 +95,6 @@ disabled!~%")) (and=> (getenv "GUIX_ALLOW_UNAUTHENTICATED_SUBSTITUTES") (cut string-ci=? <> "yes")))) -(define %narinfo-ttl - ;; Number of seconds during which cached narinfo lookups are considered - ;; valid for substitute servers that do not advertise a TTL via the - ;; 'Cache-Control' response header. - (* 36 3600)) - -(define %narinfo-negative-ttl - ;; Likewise, but for negative lookups---i.e., cached lookup failures (404). - (* 1 3600)) - -(define %narinfo-transient-error-ttl - ;; Likewise, but for transient errors such as 504 ("Gateway timeout"). - (* 10 60)) - -(define %narinfo-expired-cache-entry-removal-delay - ;; How often we want to remove files corresponding to expired cache entries. - (* 7 24 3600)) - (define %fetch-timeout ;; Number of seconds after which networking is considered "slow". 5) @@ -169,128 +134,6 @@ again." (sigaction SIGALRM SIG_DFL) (apply values result))))) -(define* (fetch uri #:key (buffered? #t) (timeout? #t) - (keep-alive? #f) (port #f)) - "Return a binary input port to URI and the number of bytes it's expected to -provide. - -When PORT is true, use it as the underlying I/O port for HTTP transfers; when -PORT is false, open a new connection for URI. When KEEP-ALIVE? is true, the -connection (typically PORT) is kept open once data has been fetched from URI." - (case (uri-scheme uri) - ((file) - (let ((port (open-file (uri-path uri) - (if buffered? "rb" "r0b")))) - (values port (stat:size (stat port))))) - ((http https) - (guard (c ((http-get-error? c) - (leave (G_ "download from '~a' failed: ~a, ~s~%") - (uri->string (http-get-error-uri c)) - (http-get-error-code c) - (http-get-error-reason c)))) - ;; Test this with: - ;; sudo tc qdisc add dev eth0 root netem delay 1500ms - ;; and then cancel with: - ;; sudo tc qdisc del dev eth0 root - (let ((port port)) - (with-timeout (if timeout? - %fetch-timeout - 0) - (begin - (warning (G_ "while fetching ~a: server is somewhat slow~%") - (uri->string uri)) - (warning (G_ "try `--no-substitutes' if the problem persists~%"))) - (begin - (when (or (not port) (port-closed? port)) - (set! port (guix:open-connection-for-uri - uri #:verify-certificate? #f))) - (unless (or buffered? (not (file-port? port))) - (setvbuf port 'none)) - (http-fetch uri #:text? #f #:port port - #:keep-alive? keep-alive? - #:verify-certificate? #f)))))) - (else - (leave (G_ "unsupported substitute URI scheme: ~a~%") - (uri->string uri))))) - -(define (narinfo-cache-file cache-url path) - "Return the name of the local file that contains an entry for PATH. The -entry is stored in a sub-directory specific to CACHE-URL." - ;; The daemon does not sanitize its input, so PATH could be something like - ;; "/gnu/store/foo". Gracefully handle that. - (match (store-path-hash-part path) - (#f - (leave (G_ "'~a' does not name a store item~%") path)) - ((? string? hash-part) - (string-append %narinfo-cache-directory "/" - (bytevector->base32-string (sha256 (string->utf8 cache-url))) - "/" hash-part)))) - -(define (cached-narinfo cache-url path) - "Check locally if we have valid info about PATH coming from CACHE-URL. -Return two values: a Boolean indicating whether we have valid cached info, and -that info, which may be either #f (when PATH is unavailable) or the narinfo -for PATH." - (define now - (current-time time-monotonic)) - - (define cache-file - (narinfo-cache-file cache-url path)) - - (catch 'system-error - (lambda () - (call-with-input-file cache-file - (lambda (p) - (match (read p) - (('narinfo ('version 2) - ('cache-uri cache-uri) - ('date date) ('ttl ttl) ('value #f)) - ;; A cached negative lookup. - (if (obsolete? date now ttl) - (values #f #f) - (values #t #f))) - (('narinfo ('version 2) - ('cache-uri cache-uri) - ('date date) ('ttl ttl) ('value value)) - ;; A cached positive lookup - (if (obsolete? date now ttl) - (values #f #f) - (values #t (string->narinfo value cache-uri)))) - (('narinfo ('version v) _ ...) - (values #f #f)))))) - (lambda _ - (values #f #f)))) - -(define (cache-narinfo! cache-url path narinfo ttl) - "Cache locally NARNIFO for PATH, which originates from CACHE-URL, with the -given TTL (a number of seconds or #f). NARINFO may be #f, in which case it -indicates that PATH is unavailable at CACHE-URL." - (define now - (current-time time-monotonic)) - - (define (cache-entry cache-uri narinfo) - `(narinfo (version 2) - (cache-uri ,cache-uri) - (date ,(time-second now)) - (ttl ,(or ttl - (if narinfo %narinfo-ttl %narinfo-negative-ttl))) - (value ,(and=> narinfo narinfo->string)))) - - (let ((file (narinfo-cache-file cache-url path))) - (mkdir-p (dirname file)) - (with-atomic-file-output file - (lambda (out) - (write (cache-entry cache-url narinfo) out)))) - - narinfo) - -(define (narinfo-request cache-url path) - "Return an HTTP request for the narinfo of PATH at CACHE-URL." - (let ((url (string-append cache-url "/" (store-path-hash-part path) - ".narinfo")) - (headers '((User-Agent . "GNU Guile")))) - (build-request (string->uri url) #:method 'GET #:headers headers))) - (define (at-most max-length lst) "If LST is shorter than MAX-LENGTH, return it and the empty list; otherwise return its MAX-LENGTH first elements and its tail." @@ -305,80 +148,6 @@ return its MAX-LENGTH first elements and its tail." (values (reverse result) lst) (loop (+ 1 len) tail (cons head result))))))) -(define* (http-multiple-get base-uri proc seed requests - #:key port (verify-certificate? #t) - (open-connection guix:open-connection-for-uri) - (keep-alive? #t) - (batch-size 1000)) - "Send all of REQUESTS to the server at BASE-URI. Call PROC for each -response, passing it the request object, the response, a port from which to -read the response body, and the previous result, starting with SEED, à la -'fold'. Return the final result. - -When PORT is specified, use it as the initial connection on which HTTP -requests are sent; otherwise call OPEN-CONNECTION to open a new connection for -a URI. When KEEP-ALIVE? is false, close the connection port before -returning." - (let connect ((port port) - (requests requests) - (result seed)) - (define batch - (at-most batch-size requests)) - - ;; (format (current-error-port) "connecting (~a requests left)..." - ;; (length requests)) - (let ((p (or port (open-connection base-uri - #:verify-certificate? - verify-certificate?)))) - ;; For HTTPS, P is not a file port and does not support 'setvbuf'. - (when (file-port? p) - (setvbuf p 'block (expt 2 16))) - - ;; Send BATCH in a row. - ;; XXX: Do our own caching to work around inefficiencies when - ;; communicating over TLS: <http://bugs.gnu.org/22966>. - (let-values (((buffer get) (open-bytevector-output-port))) - ;; Inherit the HTTP proxying property from P. - (set-http-proxy-port?! buffer (http-proxy-port? p)) - - (for-each (cut write-request <> buffer) - batch) - (put-bytevector p (get)) - (force-output p)) - - ;; Now start processing responses. - (let loop ((sent batch) - (processed 0) - (result result)) - (match sent - (() - (match (drop requests processed) - (() - (unless keep-alive? - (close-port p)) - (reverse result)) - (remainder - (connect p remainder result)))) - ((head tail ...) - (let* ((resp (read-response p)) - (body (response-body-port resp)) - (result (proc head resp body result))) - ;; The server can choose to stop responding at any time, in which - ;; case we have to try again. Check whether that is the case. - ;; Note that even upon "Connection: close", we can read from BODY. - (match (assq 'connection (response-headers resp)) - (('connection 'close) - (close-port p) - (connect #f ;try again - (drop requests (+ 1 processed)) - result)) - (_ - (loop tail (+ 1 processed) result)))))))))) ;keep going - -(define (read-to-eof port) - "Read from PORT until EOF is reached. The data are discarded." - (dump-port port (%make-void-port "w"))) - (define (narinfo-from-file file url) "Attempt to read a narinfo from FILE, using URL as the cache URL. Return #f if file doesn't exist, and the narinfo otherwise." @@ -391,191 +160,6 @@ if file doesn't exist, and the narinfo otherwise." #f (apply throw args))))) -(define %unreachable-hosts - ;; Set of names of unreachable hosts. - (make-hash-table)) - -(define* (open-connection-for-uri/maybe uri - #:key - fresh? - (time %fetch-timeout)) - "Open a connection to URI via 'open-connection-for-uri/cached' and return a -port to it, or, if connection failed, print a warning and return #f. Pass -#:fresh? to 'open-connection-for-uri/cached'." - (define host - (uri-host uri)) - - (catch #t - (lambda () - (open-connection-for-uri/cached uri #:timeout time - #:fresh? fresh?)) - (match-lambda* - (('getaddrinfo-error error) - (unless (hash-ref %unreachable-hosts host) - (hash-set! %unreachable-hosts host #t) ;warn only once - (warning (G_ "~a: host not found: ~a~%") - host (gai-strerror error))) - #f) - (('system-error . args) - (unless (hash-ref %unreachable-hosts host) - (hash-set! %unreachable-hosts host #t) - (warning (G_ "~a: connection failed: ~a~%") host - (strerror - (system-error-errno `(system-error ,@args))))) - #f) - (args - (apply throw args))))) - -(define (fetch-narinfos url paths) - "Retrieve all the narinfos for PATHS from the cache at URL and return them." - (define update-progress! - (let ((done 0) - (total (length paths))) - (lambda () - (display "\r\x1b[K" (current-error-port)) ;erase current line - (force-output (current-error-port)) - (format (current-error-port) - (G_ "updating substitutes from '~a'... ~5,1f%") - url (* 100. (/ done total))) - (set! done (+ 1 done))))) - - (define hash-part->path - (let ((mapping (fold (lambda (path result) - (vhash-cons (store-path-hash-part path) path - result)) - vlist-null - paths))) - (lambda (hash) - (match (vhash-assoc hash mapping) - (#f #f) - ((_ . path) path))))) - - (define (handle-narinfo-response request response port result) - (let* ((code (response-code response)) - (len (response-content-length response)) - (cache (response-cache-control response)) - (ttl (and cache (assoc-ref cache 'max-age)))) - (update-progress!) - - ;; Make sure to read no more than LEN bytes since subsequent bytes may - ;; belong to the next response. - (if (= code 200) ; hit - (let ((narinfo (read-narinfo port url #:size len))) - (if (string=? (dirname (narinfo-path narinfo)) - (%store-prefix)) - (begin - (cache-narinfo! url (narinfo-path narinfo) narinfo ttl) - (cons narinfo result)) - result)) - (let* ((path (uri-path (request-uri request))) - (hash-part (basename - (string-drop-right path 8)))) ;drop ".narinfo" - (if len - (get-bytevector-n port len) - (read-to-eof port)) - (cache-narinfo! url (hash-part->path hash-part) #f - (if (or (= 404 code) (= 202 code)) - ttl - %narinfo-transient-error-ttl)) - result)))) - - (define (do-fetch uri) - (case (and=> uri uri-scheme) - ((http https) - ;; Note: Do not check HTTPS server certificates to avoid depending - ;; on the X.509 PKI. We can do it because we authenticate - ;; narinfos, which provides a much stronger guarantee. - (let* ((requests (map (cut narinfo-request url <>) paths)) - (result (call-with-cached-connection uri - (lambda (port) - (if port - (begin - (update-progress!) - (http-multiple-get uri - handle-narinfo-response '() - requests - #:open-connection - open-connection-for-uri/cached - #:verify-certificate? #f - #:port port)) - '())) - open-connection-for-uri/maybe))) - (newline (current-error-port)) - result)) - ((file #f) - (let* ((base (string-append (uri-path uri) "/")) - (files (map (compose (cut string-append base <> ".narinfo") - store-path-hash-part) - paths))) - (filter-map (cut narinfo-from-file <> url) files))) - (else - (leave (G_ "~s: unsupported server URI scheme~%") - (if uri (uri-scheme uri) url))))) - - (do-fetch (string->uri url))) - -(define (lookup-narinfos cache paths) - "Return the narinfos for PATHS, invoking the server at CACHE when no -information is available locally." - (let-values (((cached missing) - (fold2 (lambda (path cached missing) - (let-values (((valid? value) - (cached-narinfo cache path))) - (if valid? - (if value - (values (cons value cached) missing) - (values cached missing)) - (values cached (cons path missing))))) - '() - '() - paths))) - (if (null? missing) - cached - (let ((missing (fetch-narinfos cache missing))) - (append cached (or missing '())))))) - -(define (lookup-narinfos/diverse caches paths authorized?) - "Look up narinfos for PATHS on all of CACHES, a list of URLS, in that order. -That is, when a cache lacks an AUTHORIZED? narinfo, look it up in the next -cache, and so on. - -Return a list of narinfos for PATHS or a subset thereof. The returned -narinfos are either AUTHORIZED?, or they claim a hash that matches an -AUTHORIZED? narinfo." - (define (select-hit result) - (lambda (path) - (match (vhash-fold* cons '() path result) - ((one) - one) - ((several ..1) - (let ((authorized (find authorized? (reverse several)))) - (and authorized - (find (cut equivalent-narinfo? <> authorized) - several))))))) - - (let loop ((caches caches) - (paths paths) - (result vlist-null) ;path->narinfo vhash - (hits '())) ;paths - (match paths - (() ;we're done - ;; Now iterate on all the HITS, and return exactly one match for each - ;; hit: the first narinfo that is authorized, or that has the same hash - ;; as an authorized narinfo, in the order of CACHES. - (filter-map (select-hit result) hits)) - (_ - (match caches - ((cache rest ...) - (let* ((narinfos (lookup-narinfos cache paths)) - (definite (map narinfo-path (filter authorized? narinfos))) - (missing (lset-difference string=? paths definite))) ;XXX: perf - (loop rest missing - (fold vhash-cons result - (map narinfo-path narinfos) narinfos) - (append definite hits)))) - (() ;that's it - (filter-map (select-hit result) hits))))))) - (define (lookup-narinfo caches path authorized?) "Return the narinfo for PATH in CACHES, or #f when no substitute for PATH was found." @@ -629,7 +213,9 @@ was found." ;; lookup errors are typically the first one, and because other errors are ;; a subset of `system-error', which is harder to filter. ((_ exp ...) - (catch #t + ;; Use a pre-unwind handler so that re-throwing preserves useful + ;; backtraces. 'with-throw-handler' works for Guile 2.2 and 3.0. + (with-throw-handler #t (lambda () exp ...) (match-lambda* (('getaddrinfo-error error) @@ -672,6 +258,27 @@ Internal tool to substitute a pre-built binary to a local build.\n")) ;;; Daemon/substituter protocol. ;;; +(define %prefer-fast-decompression? + ;; Whether to prefer fast decompression over good compression ratios. This + ;; serves in particular to choose between lzip (high compression ratio but + ;; low decompression throughput) and zstd (lower compression ratio but high + ;; decompression throughput). + #f) + +(define (call-with-cpu-usage-monitoring proc) + (let ((before (times))) + (proc) + (let ((after (times))) + (if (= (tms:clock after) (tms:clock before)) + 0 + (/ (- (tms:utime after) (tms:utime before)) + (- (tms:clock after) (tms:clock before)) + 1.))))) + +(define-syntax-rule (with-cpu-usage-monitoring exp ...) + "Evaluate EXP... Return its CPU usage as a fraction between 0 and 1." + (call-with-cpu-usage-monitoring (lambda () exp ...))) + (define (display-narinfo-data narinfo) "Write to the current output port the contents of NARINFO in the format expected by the daemon." @@ -684,7 +291,10 @@ expected by the daemon." (for-each (cute format #t "~a/~a~%" (%store-prefix) <>) (narinfo-references narinfo)) - (let-values (((uri compression file-size) (narinfo-best-uri narinfo))) + (let-values (((uri compression file-size) + (narinfo-best-uri narinfo + #:fast-decompression? + %prefer-fast-decompression?))) (format #t "~a\n~a\n" (or file-size 0) (or (narinfo-size narinfo) 0)))) @@ -703,17 +313,40 @@ authorized substitutes." (lambda (obj) (valid-narinfo? obj acl)))) + (define* (make-progress-reporter total #:key url) + (define done 0) + + (define (report-progress) + (erase-current-line (current-error-port)) ;erase current line + (force-output (current-error-port)) + (format (current-error-port) + (G_ "updating substitutes from '~a'... ~5,1f%") + url (* 100. (/ done total))) + (set! done (+ 1 done))) + + (progress-reporter + (start report-progress) + (report report-progress) + (stop (lambda () + (newline (current-error-port)))))) + (match (string-tokenize command) (("have" paths ..1) ;; Return the subset of PATHS available in CACHE-URLS. - (let ((substitutable (lookup-narinfos/diverse cache-urls paths valid?))) + (let ((substitutable (lookup-narinfos/diverse + cache-urls paths valid? + #:open-connection open-connection-for-uri/cached + #:make-progress-reporter make-progress-reporter))) (for-each (lambda (narinfo) (format #t "~a~%" (narinfo-path narinfo))) substitutable) (newline))) (("info" paths ..1) ;; Reply info about PATHS if it's in CACHE-URLS. - (let ((substitutable (lookup-narinfos/diverse cache-urls paths valid?))) + (let ((substitutable (lookup-narinfos/diverse + cache-urls paths valid? + #:open-connection open-connection-for-uri/cached + #:make-progress-reporter make-progress-reporter))) (for-each display-narinfo-data substitutable) (newline))) (wtf @@ -726,7 +359,7 @@ authorized substitutes." (define open-connection-for-uri/cached (let ((cache '())) - (lambda* (uri #:key fresh? timeout verify-certificate?) + (lambda* (uri #:key fresh? (timeout %fetch-timeout) verify-certificate?) "Return a connection for URI, possibly reusing a cached connection. When FRESH? is true, delete any cached connections for URI and open a new one. Return #f if URI's scheme is 'file' or #f. @@ -769,26 +402,26 @@ server certificates." (drain-input socket) socket)))))))) -(define* (call-with-cached-connection uri proc - #:optional - (open-connection - open-connection-for-uri/cached)) - (let ((port (open-connection uri))) +(define (call-with-cached-connection uri proc) + (let ((port (open-connection-for-uri/cached uri + #:verify-certificate? #f))) (catch #t (lambda () (proc port)) (lambda (key . args) ;; If PORT was cached and the server closed the connection in the - ;; meantime, we get EPIPE. In that case, open a fresh connection and - ;; retry. We might also get 'bad-response or a similar exception from - ;; (web response) later on, once we've sent the request, or a - ;; ERROR/INVALID-SESSION from GnuTLS. + ;; meantime, we get EPIPE. In that case, open a fresh connection + ;; and retry. We might also get 'bad-response or a similar + ;; exception from (web response) later on, once we've sent the + ;; request, or a ERROR/INVALID-SESSION from GnuTLS. (if (or (and (eq? key 'system-error) (= EPIPE (system-error-errno `(,key ,@args)))) (and (eq? key 'gnutls-error) (eq? (first args) error/invalid-session)) (memq key '(bad-response bad-header bad-header-component))) - (proc (open-connection uri #:fresh? #t)) + (proc (open-connection-for-uri/cached uri + #:verify-certificate? #f + #:fresh? #t)) (apply throw key args)))))) (define-syntax-rule (with-cached-connection uri port exp ...) @@ -819,12 +452,43 @@ the current output port." (apply dump-file/deduplicate (append args (list #:store (%store-prefix))))) + (define (fetch uri) + (case (uri-scheme uri) + ((file) + (let ((port (open-file (uri-path uri) "r0b"))) + (values port (stat:size (stat port))))) + ((http https) + (guard (c ((http-get-error? c) + (leave (G_ "download from '~a' failed: ~a, ~s~%") + (uri->string (http-get-error-uri c)) + (http-get-error-code c) + (http-get-error-reason c)))) + ;; Test this with: + ;; sudo tc qdisc add dev eth0 root netem delay 1500ms + ;; and then cancel with: + ;; sudo tc qdisc del dev eth0 root + (with-timeout %fetch-timeout + (begin + (warning (G_ "while fetching ~a: server is somewhat slow~%") + (uri->string uri)) + (warning (G_ "try `--no-substitutes' if the problem persists~%"))) + (with-cached-connection uri port + (http-fetch uri #:text? #f + #:port port + #:keep-alive? #t + #:buffered? #f))))) + (else + (leave (G_ "unsupported substitute URI scheme: ~a~%") + (uri->string uri))))) + (unless narinfo (leave (G_ "no valid substitute for '~a'~%") store-item)) (let-values (((uri compression file-size) - (narinfo-best-uri narinfo))) + (narinfo-best-uri narinfo + #:fast-decompression? + %prefer-fast-decompression?))) (unless print-build-trace? (format (current-error-port) (G_ "Downloading ~a...~%") (uri->string uri))) @@ -832,10 +496,7 @@ the current output port." (let*-values (((raw download-size) ;; 'guix publish' without '--cache' doesn't specify a ;; Content-Length, so DOWNLOAD-SIZE is #f in this case. - (with-cached-connection uri port - (fetch uri #:buffered? #f #:timeout? #f - #:port port - #:keep-alive? #t))) + (fetch uri)) ((progress) (let* ((dl-size (or download-size (and (equal? compression "none") @@ -865,11 +526,28 @@ the current output port." ((hashed get-hash) (open-hash-input-port algorithm input))) ;; Unpack the Nar at INPUT into DESTINATION. - (restore-file hashed destination - #:dump-file (if (and destination-in-store? - deduplicate?) - dump-file/deduplicate* - dump-file)) + (define cpu-usage + (with-cpu-usage-monitoring + (restore-file hashed destination + #:dump-file (if (and destination-in-store? + deduplicate?) + dump-file/deduplicate* + dump-file)))) + + ;; Create a hysteresis: depending on CPU usage, favor compression + ;; methods with faster decompression (like ztsd) or methods with better + ;; compression ratios (like lzip). This stems from the observation that + ;; substitution can be CPU-bound when high-speed networks are used: + ;; <https://lists.gnu.org/archive/html/guix-devel/2020-12/msg00177.html>. + ;; To simulate "slow" networking or changing conditions, run: + ;; sudo tc qdisc add dev eno1 root tbf rate 512kbit latency 50ms burst 1540 + ;; and then cancel with: + ;; sudo tc qdisc del dev eno1 root + (when (> cpu-usage .8) + (set! %prefer-fast-decompression? #t)) + (when (< cpu-usage .2) + (set! %prefer-fast-decompression? #f)) + (close-port hashed) (close-port input) @@ -877,8 +555,11 @@ the current output port." (every (compose zero? cdr waitpid) pids) ;; Skip a line after what 'progress-reporter/file' printed, and another - ;; one to visually separate substitutions. - (display "\n\n" (current-error-port)) + ;; one to visually separate substitutions. When PRINT-BUILD-TRACE? is + ;; true, leave it up to (guix status) to prettify things. + (newline (current-error-port)) + (unless print-build-trace? + (newline (current-error-port))) ;; Check whether we got the data announced in NARINFO. (let ((actual (get-hash))) @@ -977,7 +658,7 @@ is shorter than MAX elements, then it is directly returned." ;; If the following option is passed to the daemon, use the substitutes list ;; provided by "guix discover" process. (let* ((option (find-daemon-option "discover")) - (discover? (and option (string=? option "yes")))) + (discover? (and option (string=? option "true")))) (if discover? (randomize-substitute-urls (read-substitute-urls)) '()))) @@ -1006,6 +687,24 @@ default value." ;; 'guix-daemon' expects. (make-parameter #t)) +;; The daemon's agent code opens file descriptor 4 for us and this is where +;; stderr should go. +(define-syntax-rule (with-redirected-error-port exp ...) + "Evaluate EXP... with the current error port redirected to file descriptor 4 +if needed, as expected by the daemon's agent." + (let ((thunk (lambda () exp ...))) + (if (%error-to-file-descriptor-4?) + (parameterize ((current-error-port (fdopen 4 "wl"))) + ;; Redirect diagnostics to file descriptor 4 as well. + (guix-warning-port (current-error-port)) + + ;; 'with-continuation-barrier' captures the initial value of + ;; 'current-error-port' to report backtraces in case of uncaught + ;; exceptions. Without it, backtraces would be printed to FD 2, + ;; thereby confusing the daemon. + (with-continuation-barrier thunk)) + (thunk)))) + (define-command (guix-substitute . args) (category internal) (synopsis "implement the build daemon's substituter protocol") @@ -1020,14 +719,7 @@ default value." (define deduplicate? (find-daemon-option "deduplicate")) - ;; The daemon's agent code opens file descriptor 4 for us and this is where - ;; stderr should go. - (parameterize ((current-error-port (if (%error-to-file-descriptor-4?) - (fdopen 4 "wl") - (current-error-port)))) - ;; Redirect diagnostics to file descriptor 4 as well. - (guix-warning-port (current-error-port)) - + (with-redirected-error-port (mkdir-p %narinfo-cache-directory) (maybe-remove-expired-cache-entries %narinfo-cache-directory cached-narinfo-files @@ -1092,6 +784,7 @@ default value." ;;; Local Variables: ;;; eval: (put 'with-timeout 'scheme-indent-function 1) +;;; eval: (put 'with-redirected-error-port 'scheme-indent-function 0) ;;; eval: (put 'with-cached-connection 'scheme-indent-function 2) ;;; eval: (put 'call-with-cached-connection 'scheme-indent-function 1) ;;; End: diff --git a/guix/scripts/system.scm b/guix/scripts/system.scm index 19b8c5163c..c226f08371 100644 --- a/guix/scripts/system.scm +++ b/guix/scripts/system.scm @@ -8,6 +8,7 @@ ;;; Copyright © 2020 Jan (janneke) Nieuwenhuizen <janneke@gnu.org> ;;; Copyright © 2020 Julien Lepiller <julien@lepiller.eu> ;;; Copyright © 2020 Efraim Flashner <efraim@flashner.co.il> +;;; Copyright © 2021 Brice Waegeneire <brice@waegenei.re> ;;; ;;; This file is part of GNU Guix. ;;; @@ -363,11 +364,14 @@ connection to the store." "Switch the system profile to the generation specified by SPEC, and re-install bootloader with a configuration file that uses the specified system generation as its default entry. STORE is an open connection to the store." - (let ((number (relative-generation-spec->number %system-profile spec))) + (let* ((number (relative-generation-spec->number %system-profile spec)) + (generation (generation-file-name %system-profile number)) + (activate (string-append generation "/activate"))) (if number (begin (reinstall-bootloader store number) - (switch-to-generation* %system-profile number)) + (switch-to-generation* %system-profile number) + (unless-file-not-found (primitive-load activate))) (leave (G_ "cannot switch to system generation '~a'~%") spec)))) (define* (system-bootloader-name #:optional (system %system-profile)) @@ -680,13 +684,15 @@ checking this by themselves in their 'check' procedure." ;;; Action. ;;; -(define* (system-derivation-for-action os action - #:key image-size image-type - full-boot? container-shared-network? - mappings label - volatile-root?) - "Return as a monadic value the derivation for OS according to ACTION." - (mlet %store-monad ((target (current-target-system))) +(define* (system-derivation-for-action image action + #:key + full-boot? + container-shared-network? + mappings) + "Return as a monadic value the derivation for IMAGE according to ACTION." + (mlet %store-monad ((target (current-target-system)) + (os -> (image-operating-system image)) + (image-size -> (image-size image))) (case action ((build init reconfigure) (operating-system-derivation os)) @@ -695,8 +701,6 @@ checking this by themselves in their 'check' procedure." os #:mappings mappings #:shared-network? container-shared-network?)) - ((vm-image) - (system-qemu-image os #:disk-image-size image-size)) ((vm) (system-qemu-image/shared-store-script os #:full-boot? full-boot? @@ -705,21 +709,12 @@ checking this by themselves in their 'check' procedure." image-size (* 70 (expt 2 20))) #:mappings mappings)) - ((image disk-image) - (let* ((base-image (os->image os #:type image-type)) - (base-target (image-target base-image))) - (when (eq? action 'disk-image) - (warning (G_ "'disk-image' is deprecated: use 'image' instead~%"))) - (lower-object - (system-image - (image - (inherit (if label - (image-with-label base-image label) - base-image)) - (target (or base-target target)) - (size image-size) - (operating-system os) - (volatile-root? volatile-root?)))))) + ((image disk-image vm-image) + (when (eq? action 'disk-image) + (warning (G_ "'disk-image' is deprecated: use 'image' instead~%"))) + (when (eq? action 'vm-image) + (warning (G_ "'vm-image' is deprecated: use 'image' instead~%"))) + (lower-object (system-image image))) ((docker-image) (system-docker-image os #:shared-network? container-shared-network?))))) @@ -765,7 +760,7 @@ and TARGET arguments." (set! %load-compiled-path (lowered-gexp-load-compiled-path lowered)) (return (primitive-eval (lowered-gexp-sexp lowered)))))) -(define* (perform-action action os +(define* (perform-action action image #:key (validate-reconfigure ensure-forward-reconfigure) save-provenance? @@ -773,17 +768,13 @@ and TARGET arguments." install-bootloader? dry-run? derivations-only? use-substitutes? bootloader-target target - image-size image-type - volatile-root? - full-boot? label container-shared-network? + full-boot? + container-shared-network? (mappings '()) (gc-root #f)) - "Perform ACTION for OS. INSTALL-BOOTLOADER? specifies whether to install + "Perform ACTION for IMAGE. INSTALL-BOOTLOADER? specifies whether to install bootloader; BOOTLOADER-TAGET is the target for the bootloader; TARGET is the -target root directory; IMAGE-SIZE is the size of the image to be built, for -the 'vm-image' and 'image' actions. IMAGE-TYPE is the type of image to -be built. When VOLATILE-ROOT? is #t, the root file system is mounted -volatile. +target root directory. FULL-BOOT? is used for the 'vm' action; it determines whether to boot directly to the kernel or to the bootloader. CONTAINER-SHARED-NETWORK? @@ -805,6 +796,9 @@ static checks." '() (map boot-parameters->menu-entry (profile-boot-parameters)))) + (define os + (image-operating-system image)) + (define bootloader (operating-system-bootloader os)) @@ -827,11 +821,7 @@ static checks." (check-initrd-modules os))) (mlet* %store-monad - ((sys (system-derivation-for-action os action - #:label label - #:image-type image-type - #:image-size image-size - #:volatile-root? volatile-root? + ((sys (system-derivation-for-action image action #:full-boot? full-boot? #:container-shared-network? container-shared-network? #:mappings mappings)) @@ -969,8 +959,6 @@ Some ACTIONS support additional ARGS.\n")) (display (G_ "\ vm build a virtual machine image that shares the host's store\n")) (display (G_ "\ - vm-image build a freestanding virtual machine image\n")) - (display (G_ "\ image build a Guix System image\n")) (display (G_ "\ docker-image build a Docker image\n")) @@ -999,7 +987,7 @@ Some ACTIONS support additional ARGS.\n")) (display (G_ " -t, --image-type=TYPE for 'image', produce an image of TYPE")) (display (G_ " - --image-size=SIZE for 'vm-image', produce an image of SIZE")) + --image-size=SIZE for 'image', produce an image of SIZE")) (display (G_ " --no-bootloader for 'init', do not install a bootloader")) (display (G_ " @@ -1017,8 +1005,8 @@ Some ACTIONS support additional ARGS.\n")) (display (G_ " -N, --network for 'container', allow containers to access the network")) (display (G_ " - -r, --root=FILE for 'vm', 'vm-image', 'image', 'container', - and 'build', make FILE a symlink to the result, and + -r, --root=FILE for 'vm', 'image', 'container' and 'build', + make FILE a symlink to the result, and register it as a garbage collector root")) (display (G_ " --full-boot for 'vm', make a full boot sequence")) @@ -1169,9 +1157,9 @@ Some ACTIONS support additional ARGS.\n")) ACTION must be one of the sub-commands that takes an operating system declaration as an argument (a file name.) OPTS is the raw alist of options resulting from command-line parsing." - (define (ensure-operating-system file-or-exp obj) - (unless (operating-system? obj) - (leave (G_ "'~a' does not return an operating system~%") + (define (ensure-operating-system-or-image file-or-exp obj) + (unless (or (operating-system? obj) (image? obj)) + (leave (G_ "'~a' does not return an operating system or an image~%") file-or-exp)) obj) @@ -1185,27 +1173,47 @@ resulting from command-line parsing." (expr (assoc-ref opts 'expression)) (system (assoc-ref opts 'system)) (target (assoc-ref opts 'target)) - (transform (if save-provenance? - (cut operating-system-with-provenance <> file) - identity)) - (os (transform - (ensure-operating-system - (or file expr) - (cond - ((and expr file) - (leave - (G_ "both file and expression cannot be specified~%"))) - (expr - (read/eval expr)) - (file - (load* file %user-module - #:on-error (assoc-ref opts 'on-error))) - (else - (leave (G_ "no configuration specified~%"))))))) - + (transform (lambda (obj) + (if (and save-provenance? (operating-system? obj)) + (operating-system-with-provenance obj file) + obj))) + (obj (transform + (ensure-operating-system-or-image + (or file expr) + (cond + ((and expr file) + (leave + (G_ "both file and expression cannot be specified~%"))) + (expr + (read/eval expr)) + (file + (load* file %user-module + #:on-error (assoc-ref opts 'on-error))) + (else + (leave (G_ "no configuration specified~%"))))))) (dry? (assoc-ref opts 'dry-run?)) (bootloader? (assoc-ref opts 'install-bootloader?)) (label (assoc-ref opts 'label)) + (image-type (lookup-image-type-by-name + (assoc-ref opts 'image-type))) + (image (let* ((image-type (if (eq? action 'vm-image) + qcow2-image-type + image-type)) + (image-size (assoc-ref opts 'image-size)) + (volatile? (assoc-ref opts 'volatile-root?)) + (base-image (if (operating-system? obj) + (os->image obj + #:type image-type) + obj)) + (base-target (image-target base-image))) + (image + (inherit (if label + (image-with-label base-image label) + base-image)) + (target (or base-target target)) + (size image-size) + (volatile-root? volatile?)))) + (os (image-operating-system image)) (target-file (match args ((first second) second) (_ #f))) @@ -1241,7 +1249,7 @@ resulting from command-line parsing." (warn-about-old-distro #:suggested-command "guix system reconfigure")) - (perform-action action os + (perform-action action image #:dry-run? dry? #:derivations-only? (assoc-ref opts 'derivations-only?) @@ -1250,11 +1258,6 @@ resulting from command-line parsing." (assoc-ref opts 'skip-safety-checks?) #:validate-reconfigure (assoc-ref opts 'validate-reconfigure) - #:image-type (lookup-image-type-by-name - (assoc-ref opts 'image-type)) - #:image-size (assoc-ref opts 'image-size) - #:volatile-root? - (assoc-ref opts 'volatile-root?) #:full-boot? (assoc-ref opts 'full-boot?) #:container-shared-network? (assoc-ref opts 'container-shared-network?) @@ -1264,7 +1267,6 @@ resulting from command-line parsing." (_ #f)) opts) #:install-bootloader? bootloader? - #:label label #:target target-file #:bootloader-target bootloader-target #:gc-root (assoc-ref opts 'gc-root))))) diff --git a/guix/scripts/time-machine.scm b/guix/scripts/time-machine.scm index 0d27414702..4aafd432e8 100644 --- a/guix/scripts/time-machine.scm +++ b/guix/scripts/time-machine.scm @@ -1,6 +1,7 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2019 Konrad Hinsen <konrad.hinsen@fastmail.net> ;;; Copyright © 2019, 2020 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2021 Simon Tournier <zimon.toutoune@gmail.com> ;;; ;;; This file is part of GNU Guix. ;;; @@ -118,6 +119,10 @@ Execute COMMAND ARGS... in an older version of Guix.\n")) (let-values (((args command) (break (cut string=? "--" <>) args))) (let ((opts (parse-command-line args %options (list %default-options)))) + (when (assoc-ref opts 'argument) + (leave (G_ "~A: extraneous argument~%") + (assoc-ref opts 'argument))) + (match command (() opts) (("--") opts) diff --git a/guix/scripts/weather.scm b/guix/scripts/weather.scm index 97e4a73802..5164fe0494 100644 --- a/guix/scripts/weather.scm +++ b/guix/scripts/weather.scm @@ -32,7 +32,7 @@ #:use-module (guix gexp) #:use-module ((guix build syscalls) #:select (terminal-columns)) #:use-module ((guix build utils) #:select (every*)) - #:use-module (guix scripts substitute) + #:use-module (guix substitutes) #:use-module (guix narinfo) #:use-module (guix http-client) #:use-module (guix ci) @@ -117,8 +117,8 @@ values." (end (current-time time-monotonic))) (apply kont (time-difference end start) result))) -(define-syntax-rule (let/time ((time result exp)) body ...) - (call-with-time (lambda () exp) (lambda (time result) body ...))) +(define-syntax-rule (let/time ((time result ... exp)) body ...) + (call-with-time (lambda () exp) (lambda (time result ...) body ...))) (define (histogram field proc seed lst) "Return an alist giving a histogram of all the values of FIELD for elements @@ -181,7 +181,12 @@ Return the coverage ratio, an exact number between 0 and 1." (format #t (G_ "looking for ~h store items on ~a...~%") (length items) server) - (let/time ((time narinfos (lookup-narinfos server items))) + (let/time ((time narinfos requests-made + (lookup-narinfos + server items + #:make-progress-reporter + (lambda* (total #:key url #:allow-other-keys) + (progress-reporter/bar total))))) (format #t "~a~%" server) (let ((obtained (length narinfos)) (requested (length items)) @@ -207,10 +212,11 @@ Return the coverage ratio, an exact number between 0 and 1." total))))) (format #t (G_ " ~,1h MiB on disk (uncompressed)~%") (/ (reduce + 0 (map narinfo-size narinfos)) MiB)) - (format #t (G_ " ~,3h seconds per request (~,1h seconds in total)~%") - (/ time requested 1.) time) - (format #t (G_ " ~,1h requests per second~%") - (/ requested time 1.)) + (when (> requests-made 0) + (format #t (G_ " ~,3h seconds per request (~,1h seconds in total)~%") + (/ time requests-made 1.) time) + (format #t (G_ " ~,1h requests per second~%") + (/ requests-made time 1.))) (guard (c ((http-get-error? c) (if (= 404 (http-get-error-code c)) diff --git a/guix/self.scm b/guix/self.scm index 35fba1152d..3154d180ac 100644 --- a/guix/self.scm +++ b/guix/self.scm @@ -56,6 +56,7 @@ ("guile-ssh" (ref '(gnu packages ssh) 'guile-ssh)) ("guile-git" (ref '(gnu packages guile) 'guile-git)) ("guile-semver" (ref '(gnu packages guile-xyz) 'guile-semver)) + ("guile-lib" (ref '(gnu packages guile-xyz) 'guile-lib)) ("guile-sqlite3" (ref '(gnu packages guile) 'guile-sqlite3)) ("guile-zlib" (ref '(gnu packages guile) 'guile-zlib)) ("guile-lzlib" (ref '(gnu packages guile) 'guile-lzlib)) @@ -814,6 +815,9 @@ itself." (define guile-ssh (specification->package "guile-ssh")) + (define guile-lib + (specification->package "guile-lib")) + (define guile-git (specification->package "guile-git")) @@ -842,7 +846,7 @@ itself." (append-map transitive-package-dependencies (list guile-gcrypt gnutls guile-git guile-avahi guile-json guile-semver guile-ssh guile-sqlite3 - guile-zlib guile-lzlib guile-zstd))) + guile-lib guile-zlib guile-lzlib guile-zstd))) (define *core-modules* (scheme-node "guix-core" diff --git a/guix/serialization.scm b/guix/serialization.scm index 9d0739f6c5..9b888a7d25 100644 --- a/guix/serialization.scm +++ b/guix/serialization.scm @@ -199,6 +199,37 @@ substitute invalid byte sequences with question marks. This is a (define write-store-path-list write-string-list) (define read-store-path-list read-string-list) +(define-syntax write-literal-strings + (lambda (s) + "Write the given literal strings to PORT in an optimized fashion, without +any run-time allocations or computations." + (define (padding len) + (let ((m (modulo len 8))) + (if (zero? m) + 0 + (- 8 m)))) + + (syntax-case s () + ((_ port strings ...) + (let* ((bytes (map string->utf8 (syntax->datum #'(strings ...)))) + (len (fold (lambda (bv size) + (+ size 8 (bytevector-length bv) + (padding (bytevector-length bv)))) + 0 + bytes)) + (bv (make-bytevector len)) + (zeros (make-bytevector 8 0))) + (fold (lambda (str offset) + (let ((len (bytevector-length str))) + (bytevector-u32-set! bv offset len (endianness little)) + (bytevector-copy! str 0 bv (+ 8 offset) len) + (bytevector-copy! zeros 0 bv (+ 8 offset len) + (padding len)) + (+ offset 8 len (padding len)))) + 0 + bytes) + #`(put-bytevector port #,bv)))))) + (define-condition-type &nar-read-error &nar-error nar-read-error? @@ -332,14 +363,12 @@ which case you can use 'identity'." (define-values (type size) (file-type+size f)) - (write-string "(" p) + (write-literal-strings p "(") (case type ((regular executable) - (write-string "type" p) - (write-string "regular" p) + (write-literal-strings p "type" "regular") (when (eq? 'executable type) - (write-string "executable" p) - (write-string "" p)) + (write-literal-strings p "executable" "")) (let ((input (file-port f))) (dynamic-wind (const #t) @@ -348,28 +377,23 @@ which case you can use 'identity'." (lambda () (close-port input))))) ((directory) - (write-string "type" p) - (write-string "directory" p) + (write-literal-strings p "type" "directory") (let ((entries (postprocess-entries (directory-entries f)))) (for-each (lambda (e) (let* ((f (string-append f "/" e))) - (write-string "entry" p) - (write-string "(" p) - (write-string "name" p) + (write-literal-strings p "entry" "(" "name") (write-string e p) - (write-string "node" p) + (write-literal-strings p "node") (dump f) - (write-string ")" p))) + (write-literal-strings p ")"))) entries))) ((symlink) - (write-string "type" p) - (write-string "symlink" p) - (write-string "target" p) + (write-literal-strings p "type" "symlink" "target") (write-string (symlink-target f) p)) (else (raise (condition (&message (message "unsupported file type")) (&nar-error (file f) (port port)))))) - (write-string ")" p))) + (write-literal-strings p ")"))) (define port-conversion-strategy (fluid->parameter %default-port-conversion-strategy)) diff --git a/guix/status.scm b/guix/status.scm index 9ca6d92470..362ae2882c 100644 --- a/guix/status.scm +++ b/guix/status.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2017, 2018, 2019, 2020 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2017, 2018, 2019, 2020, 2021 Ludovic Courtès <ludo@gnu.org> ;;; Copyright © 2018, 2019 Ricardo Wurmus <rekado@elephly.net> ;;; ;;; This file is part of GNU Guix. @@ -23,8 +23,7 @@ #:use-module (guix colors) #:use-module (guix progress) #:autoload (guix build syscalls) (terminal-columns) - #:use-module ((guix build download) - #:select (nar-uri-abbreviation)) + #:autoload (guix build download) (nar-uri-abbreviation) #:use-module (guix store) #:use-module (guix derivations) #:use-module (guix memoization) @@ -404,10 +403,12 @@ the current build phase." #:optional (port (current-error-port)) #:key (colorize? (color-output? port)) + (print-urls? #t) (print-log? #t)) "Print information about EVENT and STATUS to PORT. When COLORIZE? is true, produce colorful output. When PRINT-LOG? is true, display the build log in -addition to build events." +addition to build events. When PRINT-URLS? is true, display the URL of +substitutes being downloaded." (define info (if colorize? (cute colorize-string <> (color BOLD)) @@ -527,9 +528,10 @@ addition to build events." (format port (info (G_ "substituting ~a...")) item) (newline port))) (('download-started item uri _ ...) - (erase-current-line*) - (format port (info (G_ "downloading from ~a ...")) uri) - (newline port)) + (when print-urls? + (erase-current-line*) + (format port (info (G_ "downloading from ~a ...")) uri) + (newline port))) (('download-progress item uri (= string->number size) (= string->number transferred)) @@ -603,6 +605,17 @@ addition to build events." (colorize? (color-output? port))) (print-build-event event old-status status port #:colorize? colorize? + #:print-urls? #f + #:print-log? #f)) + +(define* (print-build-event/quiet-with-urls event old-status status + #:optional + (port (current-error-port)) + #:key + (colorize? (color-output? port))) + (print-build-event event old-status status port + #:colorize? colorize? + #:print-urls? #t ;show download URLs #:print-log? #f)) (define* (build-status-updater #:optional (on-change (const #t))) @@ -788,6 +801,7 @@ evaluate EXP... in that context." "Return the logging procedure that corresponds to LEVEL." (cond ((<= level 0) (const #t)) ((= level 1) print-build-event/quiet) + ((= level 2) print-build-event/quiet-with-urls) (else print-build-event))) (define (call-with-status-verbosity level thunk) diff --git a/guix/store.scm b/guix/store.scm index 81bb9eb847..37ae6cfedd 100644 --- a/guix/store.scm +++ b/guix/store.scm @@ -1835,18 +1835,21 @@ and RESULT is typically its derivation." (if (profiled? "object-cache") (let ((fresh 0) (lookups 0) - (hits 0)) + (hits 0) + (size 0)) (register-profiling-hook! "object-cache" (lambda () (format (current-error-port) "Store object cache: fresh caches: ~5@a lookups: ~5@a - hits: ~5@a (~,1f%)~%" + hits: ~5@a (~,1f%) + cache size: ~5@a entries~%" fresh lookups hits (if (zero? lookups) 100. - (* 100. (/ hits lookups)))))) + (* 100. (/ hits lookups))) + size))) (lambda (hit? cache) (set! fresh @@ -1854,12 +1857,13 @@ and RESULT is typically its derivation." (+ 1 fresh) fresh)) (set! lookups (+ 1 lookups)) - (set! hits (if hit? (+ hits 1) hits)))) + (set! hits (if hit? (+ hits 1) hits)) + (set! size (+ (if hit? 0 1) + (vlist-length cache))))) (lambda (x y) #t))) -(define* (lookup-cached-object object #:optional (keys '()) - #:key (vhash-fold* vhash-foldq*)) +(define-inlinable (lookup-cached-object object keys vhash-fold*) "Return the cached object in the store connection corresponding to OBJECT and KEYS; use VHASH-FOLD* to look for OBJECT in the cache. KEYS is a list of additional keys to match against, and which are compared with 'equal?'. @@ -1890,7 +1894,7 @@ Return #f on failure and the cached result otherwise." OBJECT/KEYS, or return its cached value. Use VHASH-CONS to insert OBJECT into the cache, and VHASH-FOLD* to look it up." (mlet %store-monad ((cached (lookup-cached-object object keys - #:vhash-fold* vhash-fold*))) + vhash-fold*))) (if cached (return cached) (>>= (mthunk) diff --git a/guix/substitutes.scm b/guix/substitutes.scm new file mode 100644 index 0000000000..08f8c24efd --- /dev/null +++ b/guix/substitutes.scm @@ -0,0 +1,371 @@ +;;; GNU Guix --- Functional package management for GNU +;;; Copyright © 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2014 Nikita Karetnikov <nikita@karetnikov.org> +;;; Copyright © 2018 Kyle Meyer <kyle@kyleam.com> +;;; Copyright © 2020 Christopher Baines <mail@cbaines.net> +;;; +;;; This file is part of GNU Guix. +;;; +;;; GNU Guix is free software; you can redistribute it and/or modify it +;;; under the terms of the GNU General Public License as published by +;;; the Free Software Foundation; either version 3 of the License, or (at +;;; your option) any later version. +;;; +;;; GNU Guix is distributed in the hope that it will be useful, but +;;; WITHOUT ANY WARRANTY; without even the implied warranty of +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;;; GNU General Public License for more details. +;;; +;;; You should have received a copy of the GNU General Public License +;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>. + +(define-module (guix substitutes) + #:use-module (guix narinfo) + #:use-module (guix store) + #:use-module (guix utils) + #:use-module (guix combinators) + #:use-module (guix config) + #:use-module (guix records) + #:use-module (guix diagnostics) + #:use-module (guix i18n) + #:use-module (gcrypt hash) + #:use-module (guix base32) + #:use-module (guix base64) + #:use-module (guix cache) + #:use-module (gcrypt pk-crypto) + #:use-module (guix pki) + #:use-module ((guix build utils) #:select (mkdir-p dump-port)) + #:use-module ((guix build download) + #:select ((open-connection-for-uri + . guix:open-connection-for-uri))) + #:use-module (guix progress) + #:use-module (ice-9 rdelim) + #:use-module (ice-9 regex) + #:use-module (ice-9 match) + #:use-module (ice-9 format) + #:use-module (ice-9 ftw) + #:use-module (ice-9 binary-ports) + #:use-module (ice-9 vlist) + #:use-module (rnrs bytevectors) + #:use-module (srfi srfi-1) + #:use-module (srfi srfi-11) + #:use-module (srfi srfi-19) + #:use-module (srfi srfi-26) + #:use-module (srfi srfi-34) + #:use-module (srfi srfi-35) + #:use-module (web uri) + #:use-module (web request) + #:use-module (web response) + #:use-module (guix http-client) + #:export (%narinfo-cache-directory + + call-with-connection-error-handling + + lookup-narinfos + lookup-narinfos/diverse)) + +(define %narinfo-ttl + ;; Number of seconds during which cached narinfo lookups are considered + ;; valid for substitute servers that do not advertise a TTL via the + ;; 'Cache-Control' response header. + (* 36 3600)) + +(define %narinfo-negative-ttl + ;; Likewise, but for negative lookups---i.e., cached lookup failures (404). + (* 1 3600)) + +(define %narinfo-transient-error-ttl + ;; Likewise, but for transient errors such as 504 ("Gateway timeout"). + (* 10 60)) + +(define %narinfo-cache-directory + ;; A local cache of narinfos, to avoid going to the network. Most of the + ;; time, 'guix substitute' is called by guix-daemon as root and stores its + ;; cached data in /var/guix/…. However, when invoked from 'guix challenge' + ;; as a user, it stores its cache in ~/.cache. + (if (zero? (getuid)) + (or (and=> (getenv "XDG_CACHE_HOME") + (cut string-append <> "/guix/substitute")) + (string-append %state-directory "/substitute/cache")) + (string-append (cache-directory #:ensure? #f) "/substitute"))) + +(define (narinfo-cache-file cache-url path) + "Return the name of the local file that contains an entry for PATH. The +entry is stored in a sub-directory specific to CACHE-URL." + ;; The daemon does not sanitize its input, so PATH could be something like + ;; "/gnu/store/foo". Gracefully handle that. + (match (store-path-hash-part path) + (#f + (leave (G_ "'~a' does not name a store item~%") path)) + ((? string? hash-part) + (string-append %narinfo-cache-directory "/" + (bytevector->base32-string (sha256 (string->utf8 cache-url))) + "/" hash-part)))) + +(define (cache-narinfo! cache-url path narinfo ttl) + "Cache locally NARNIFO for PATH, which originates from CACHE-URL, with the +given TTL (a number of seconds or #f). NARINFO may be #f, in which case it +indicates that PATH is unavailable at CACHE-URL." + (define now + (current-time time-monotonic)) + + (define (cache-entry cache-uri narinfo) + `(narinfo (version 2) + (cache-uri ,cache-uri) + (date ,(time-second now)) + (ttl ,(or ttl + (if narinfo %narinfo-ttl %narinfo-negative-ttl))) + (value ,(and=> narinfo narinfo->string)))) + + (let ((file (narinfo-cache-file cache-url path))) + (mkdir-p (dirname file)) + (with-atomic-file-output file + (lambda (out) + (write (cache-entry cache-url narinfo) out)))) + + narinfo) + +(define %unreachable-hosts + ;; Set of names of unreachable hosts. + (make-hash-table)) + +(define* (call-with-connection-error-handling uri proc) + "Call PROC, and catch if a connection fails, print a warning and return #f." + (define host + (uri-host uri)) + + (catch #t + proc + (match-lambda* + (('getaddrinfo-error error) + (unless (hash-ref %unreachable-hosts host) + (hash-set! %unreachable-hosts host #t) ;warn only once + (warning (G_ "~a: host not found: ~a~%") + host (gai-strerror error))) + #f) + (('system-error . args) + (unless (hash-ref %unreachable-hosts host) + (hash-set! %unreachable-hosts host #t) + (warning (G_ "~a: connection failed: ~a~%") host + (strerror + (system-error-errno `(system-error ,@args))))) + #f) + (args + (apply throw args))))) + +(define (narinfo-request cache-url path) + "Return an HTTP request for the narinfo of PATH at CACHE-URL." + (let ((url (string-append cache-url "/" (store-path-hash-part path) + ".narinfo")) + (headers '((User-Agent . "GNU Guile")))) + (build-request (string->uri url) #:method 'GET #:headers headers))) + +(define (narinfo-from-file file url) + "Attempt to read a narinfo from FILE, using URL as the cache URL. Return #f +if file doesn't exist, and the narinfo otherwise." + (catch 'system-error + (lambda () + (call-with-input-file file + (cut read-narinfo <> url))) + (lambda args + (if (= ENOENT (system-error-errno args)) + #f + (apply throw args))))) + +(define* (fetch-narinfos url paths + #:key + (open-connection guix:open-connection-for-uri) + (make-progress-reporter + (const progress-reporter/silent))) + "Retrieve all the narinfos for PATHS from the cache at URL and return them." + (define progress-reporter + (make-progress-reporter (length paths) + #:url url)) + + (define hash-part->path + (let ((mapping (fold (lambda (path result) + (vhash-cons (store-path-hash-part path) path + result)) + vlist-null + paths))) + (lambda (hash) + (match (vhash-assoc hash mapping) + (#f #f) + ((_ . path) path))))) + + (define (read-to-eof port) + "Read from PORT until EOF is reached. The data are discarded." + (dump-port port (%make-void-port "w"))) + + (define (handle-narinfo-response request response port result) + (let* ((code (response-code response)) + (len (response-content-length response)) + (cache (response-cache-control response)) + (ttl (and cache (assoc-ref cache 'max-age)))) + (progress-reporter-report! progress-reporter) + + ;; Make sure to read no more than LEN bytes since subsequent bytes may + ;; belong to the next response. + (if (= code 200) ; hit + (let ((narinfo (read-narinfo port url #:size len))) + (if (string=? (dirname (narinfo-path narinfo)) + (%store-prefix)) + (begin + (cache-narinfo! url (narinfo-path narinfo) narinfo ttl) + (cons narinfo result)) + result)) + (let* ((path (uri-path (request-uri request))) + (hash-part (basename + (string-drop-right path 8)))) ;drop ".narinfo" + (if len + (get-bytevector-n port len) + (read-to-eof port)) + (cache-narinfo! url (hash-part->path hash-part) #f + (if (or (= 404 code) (= 202 code)) + ttl + %narinfo-transient-error-ttl)) + result)))) + + (define (do-fetch uri) + (case (and=> uri uri-scheme) + ((http https) + ;; Note: Do not check HTTPS server certificates to avoid depending + ;; on the X.509 PKI. We can do it because we authenticate + ;; narinfos, which provides a much stronger guarantee. + (let* ((requests (map (cut narinfo-request url <>) paths)) + (result (begin + (start-progress-reporter! progress-reporter) + (call-with-connection-error-handling + uri + (lambda () + (http-multiple-get uri + handle-narinfo-response '() + requests + #:open-connection open-connection + #:verify-certificate? #f)))))) + (stop-progress-reporter! progress-reporter) + result)) + ((file #f) + (let* ((base (string-append (uri-path uri) "/")) + (files (map (compose (cut string-append base <> ".narinfo") + store-path-hash-part) + paths))) + (filter-map (cut narinfo-from-file <> url) files))) + (else + (leave (G_ "~s: unsupported server URI scheme~%") + (if uri (uri-scheme uri) url))))) + + (do-fetch (string->uri url))) + +(define (cached-narinfo cache-url path) + "Check locally if we have valid info about PATH coming from CACHE-URL. +Return two values: a Boolean indicating whether we have valid cached info, and +that info, which may be either #f (when PATH is unavailable) or the narinfo +for PATH." + (define now + (current-time time-monotonic)) + + (define cache-file + (narinfo-cache-file cache-url path)) + + (catch 'system-error + (lambda () + (call-with-input-file cache-file + (lambda (p) + (match (read p) + (('narinfo ('version 2) + ('cache-uri cache-uri) + ('date date) ('ttl ttl) ('value #f)) + ;; A cached negative lookup. + (if (obsolete? date now ttl) + (values #f #f) + (values #t #f))) + (('narinfo ('version 2) + ('cache-uri cache-uri) + ('date date) ('ttl ttl) ('value value)) + ;; A cached positive lookup + (if (obsolete? date now ttl) + (values #f #f) + (values #t (string->narinfo value cache-uri)))) + (('narinfo ('version v) _ ...) + (values #f #f)))))) + (lambda _ + (values #f #f)))) + +(define* (lookup-narinfos cache paths + #:key (open-connection guix:open-connection-for-uri) + (make-progress-reporter + (const progress-reporter/silent))) + "Return the narinfos for PATHS, invoking the server at CACHE when no +information is available locally." + (let-values (((cached missing) + (fold2 (lambda (path cached missing) + (let-values (((valid? value) + (cached-narinfo cache path))) + (if valid? + (if value + (values (cons value cached) missing) + (values cached missing)) + (values cached (cons path missing))))) + '() + '() + paths))) + (values (if (null? missing) + cached + (let ((missing (fetch-narinfos cache missing + #:open-connection open-connection + #:make-progress-reporter + make-progress-reporter))) + (append cached (or missing '())))) + (length missing)))) + +(define* (lookup-narinfos/diverse caches paths authorized? + #:key (open-connection + guix:open-connection-for-uri) + (make-progress-reporter + (const progress-reporter/silent))) + "Look up narinfos for PATHS on all of CACHES, a list of URLS, in that order. +That is, when a cache lacks an AUTHORIZED? narinfo, look it up in the next +cache, and so on. + +Return a list of narinfos for PATHS or a subset thereof. The returned +narinfos are either AUTHORIZED?, or they claim a hash that matches an +AUTHORIZED? narinfo." + (define (select-hit result) + (lambda (path) + (match (vhash-fold* cons '() path result) + ((one) + one) + ((several ..1) + (let ((authorized (find authorized? (reverse several)))) + (and authorized + (find (cut equivalent-narinfo? <> authorized) + several))))))) + + (let loop ((caches caches) + (paths paths) + (result vlist-null) ;path->narinfo vhash + (hits '())) ;paths + (match paths + (() ;we're done + ;; Now iterate on all the HITS, and return exactly one match for each + ;; hit: the first narinfo that is authorized, or that has the same hash + ;; as an authorized narinfo, in the order of CACHES. + (filter-map (select-hit result) hits)) + (_ + (match caches + ((cache rest ...) + (let* ((narinfos (lookup-narinfos cache paths + #:open-connection open-connection + #:make-progress-reporter + make-progress-reporter)) + (definite (map narinfo-path (filter authorized? narinfos))) + (missing (lset-difference string=? paths definite))) ;XXX: perf + (loop rest missing + (fold vhash-cons result + (map narinfo-path narinfos) narinfos) + (append definite hits)))) + (() ;that's it + (filter-map (select-hit result) hits))))))) + +;;; substitutes.scm ends here diff --git a/guix/tests/http.scm b/guix/tests/http.scm index 4119e9ce01..8f50eaefca 100644 --- a/guix/tests/http.scm +++ b/guix/tests/http.scm @@ -1,5 +1,6 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2014, 2015, 2016, 2017, 2019 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2021 Maxime Devos <maximedevos@telenet.be> ;;; ;;; This file is part of GNU Guix. ;;; @@ -21,12 +22,12 @@ #:use-module (web server) #:use-module (web server http) #:use-module (web response) + #:use-module (srfi srfi-11) #:use-module (srfi srfi-39) #:use-module (ice-9 match) #:export (with-http-server call-with-http-server %http-server-port - http-server-can-listen? %local-url)) ;;; Commentary: @@ -37,12 +38,13 @@ (define %http-server-port ;; TCP port to use for the stub HTTP server. - (make-parameter 9999)) + ;; If 0, the OS will automatically choose + ;; a port. + (make-parameter 0)) (define (open-http-server-socket) - "Return a listening socket for the web server. It is useful to export it so -that tests can check whether we succeeded opening the socket and tests skip if -needed." + "Return a listening socket for the web server and the port +actually listened at (in case %http-server-port was 0)." (catch 'system-error (lambda () (let ((sock (socket PF_INET SOCK_STREAM 0))) @@ -50,22 +52,18 @@ needed." (bind sock (make-socket-address AF_INET INADDR_LOOPBACK (%http-server-port))) - sock)) + (values sock + (sockaddr:port (getsockname sock))))) (lambda args (let ((err (system-error-errno args))) (format (current-error-port) "warning: cannot run Web server for tests: ~a~%" (strerror err)) - #f)))) - -(define (http-server-can-listen?) - "Return #t if we managed to open a listening socket." - (and=> (open-http-server-socket) - (lambda (socket) - (close-port socket) - #t))) + (values #f #f))))) (define* (%local-url #:optional (port (%http-server-port))) + (when (= port 0) + (error "no web server is running!")) ;; URL to use for 'home-page' tests. (string-append "http://localhost:" (number->string port) "/foo/bar")) @@ -73,7 +71,10 @@ needed." (define* (call-with-http-server responses+data thunk) "Call THUNK with an HTTP server running and returning RESPONSES+DATA on HTTP requests. Each element of RESPONSES+DATA must be a tuple containing a -response and a string, or an HTTP response code and a string." +response and a string, or an HTTP response code and a string. + +%http-server-port will be set to the port listened at +The port listened at will be set for the dynamic extent of THUNK." (define responses (map (match-lambda (((? response? response) data) @@ -100,6 +101,7 @@ response and a string, or an HTTP response code and a string." ;; Mutex and condition variable to synchronize with the HTTP server. (define %http-server-lock (make-mutex)) (define %http-server-ready (make-condition-variable)) + (define %http-real-server-port #f) (define (http-open . args) "Start listening for HTTP requests and signal %HTTP-SERVER-READY." @@ -122,7 +124,8 @@ response and a string, or an HTTP response code and a string." (set! responses rest) (values response data)))) - (let ((socket (open-http-server-socket))) + (let-values (((socket port) (open-http-server-socket))) + (set! %http-real-server-port port) (catch 'quit (lambda () (run-server handle stub-http-server @@ -134,7 +137,8 @@ response and a string, or an HTTP response code and a string." (let ((server (make-thread server-body))) (wait-condition-variable %http-server-ready %http-server-lock) ;; Normally SERVER exits automatically once it has received a request. - (thunk)))) + (parameterize ((%http-server-port %http-real-server-port)) + (thunk))))) (define-syntax with-http-server (syntax-rules () diff --git a/guix/ui.scm b/guix/ui.scm index 9cea405456..7fbd4c63a2 100644 --- a/guix/ui.scm +++ b/guix/ui.scm @@ -552,7 +552,9 @@ Report bugs to: ~a.") %guix-bug-report-address) ~a home page: <~a>") %guix-package-name %guix-home-page-url) (format #t (G_ " General help using Guix and GNU software: <~a>") - "https://guix.gnu.org/help/") + ;; TRANSLATORS: Change the "/en" bit of this URL appropriately if + ;; the web site is translated in your language. + (G_ "https://guix.gnu.org/en/help/")) (newline)) (define (augmented-system-error-handler file) @@ -1968,7 +1970,7 @@ way." display-generation-change)) (define (switch-to-generation* profile number) - "Like 'switch-generation', but display what is happening." + "Like 'switch-to-generation', but display what is happening." (let ((previous (switch-to-generation profile number))) (display-generation-change previous number))) diff --git a/guix/upstream.scm b/guix/upstream.scm index accd8967d8..632e9ebc4f 100644 --- a/guix/upstream.scm +++ b/guix/upstream.scm @@ -264,12 +264,15 @@ them matches." #:optional (updaters (force %updaters))) "Return an upstream source to update PACKAGE, a <package> object, or #f if -none of UPDATERS matches PACKAGE. It is the caller's responsibility to ensure -that the returned source is newer than the current one." - (match (lookup-updater package updaters) - ((? upstream-updater? updater) - ((upstream-updater-latest updater) package)) - (_ #f))) +none of UPDATERS matches PACKAGE. When several updaters match PACKAGE, try +them until one of them returns an upstream source. It is the caller's +responsibility to ensure that the returned source is newer than the current +one." + (any (match-lambda + (($ <upstream-updater> name description pred latest) + (and (pred package) + (latest package)))) + updaters)) (define* (package-latest-release* package #:optional diff --git a/guix/utils.scm b/guix/utils.scm index 1625cab19b..05af86fc37 100644 --- a/guix/utils.scm +++ b/guix/utils.scm @@ -9,6 +9,7 @@ ;;; Copyright © 2018, 2020 Marius Bakke <marius@gnu.org> ;;; Copyright © 2020 Efraim Flashner <efraim@flashner.co.il> ;;; Copyright © 2021 Simon Tournier <zimon.toutoune@gmail.com> +;;; Copyright © 2021 Chris Marusich <cmmarusich@gmail.com> ;;; ;;; This file is part of GNU Guix. ;;; @@ -45,6 +46,7 @@ #:use-module (ice-9 match) #:use-module (ice-9 format) #:use-module ((ice-9 iconv) #:prefix iconv:) + #:autoload (zlib) (make-zlib-input-port make-zlib-output-port) #:use-module (system foreign) #:re-export (<location> ;for backwards compatibility location @@ -78,6 +80,7 @@ target-arm32? target-aarch64? target-arm? + target-powerpc? target-64bit? cc-for-target cxx-for-target @@ -231,7 +234,8 @@ a symbol such as 'xz." ((or #f 'none) (values input '())) ('bzip2 (filtered-port `(,%bzip2 "-dc") input)) ('xz (filtered-port `(,%xz "-dc") input)) - ('gzip (filtered-port `(,%gzip "-dc") input)) + ('gzip (values (make-zlib-input-port input #:format 'gzip) + '())) ('lzip (values (lzip-port 'make-lzip-input-port input) '())) ('zstd (values (zstd-port 'make-zstd-input-port input) @@ -292,7 +296,8 @@ program--e.g., '(\"--fast\")." ((or #f 'none) (values output '())) ('bzip2 (filtered-output-port `(,%bzip2 "-c" ,@options) output)) ('xz (filtered-output-port `(,%xz "-c" ,@options) output)) - ('gzip (filtered-output-port `(,%gzip "-c" ,@options) output)) + ('gzip (values (make-zlib-output-port output #:format 'gzip) + '())) ('lzip (values (lzip-port 'make-lzip-output-port output) '())) ('zstd (values (zstd-port 'make-zstd-output-port output) @@ -538,9 +543,13 @@ a character other than '@'." (%current-system)))) (or (target-arm32? target) (target-aarch64? target))) +(define* (target-powerpc? #:optional (target (or (%current-target-system) + (%current-system)))) + (string-prefix? "powerpc" target)) + (define* (target-64bit? #:optional (system (or (%current-target-system) (%current-system)))) - (any (cut string-prefix? <> system) '("x86_64" "aarch64" "mips64" "ppc64"))) + (any (cut string-prefix? <> system) '("x86_64" "aarch64" "mips64" "powerpc64"))) (define* (cc-for-target #:optional (target (%current-target-system))) (if target @@ -687,6 +696,7 @@ VERSIONS. For example: (define (tarball-sans-extension tarball) "Return TARBALL without its .tar.* or .zip extension." (let ((end (or (string-contains tarball ".tar") + (string-contains tarball ".tgz") (string-contains tarball ".zip")))) (substring tarball 0 end))) |