diff options
Diffstat (limited to 'guix')
-rw-r--r-- | guix/android-repo-download.scm | 5 | ||||
-rw-r--r-- | guix/build-system/asdf.scm | 18 | ||||
-rw-r--r-- | guix/build/asdf-build-system.scm | 15 | ||||
-rw-r--r-- | guix/build/download.scm | 84 | ||||
-rw-r--r-- | guix/build/emacs-build-system.scm | 79 | ||||
-rw-r--r-- | guix/ci.scm | 4 | ||||
-rw-r--r-- | guix/cve.scm | 31 | ||||
-rw-r--r-- | guix/cvs-download.scm | 12 | ||||
-rw-r--r-- | guix/diagnostics.scm | 6 | ||||
-rw-r--r-- | guix/download.scm | 19 | ||||
-rw-r--r-- | guix/git-download.scm | 2 | ||||
-rw-r--r-- | guix/hg-download.scm | 9 | ||||
-rw-r--r-- | guix/http-client.scm | 28 | ||||
-rw-r--r-- | guix/import/go.scm | 13 | ||||
-rw-r--r-- | guix/import/pypi.scm | 4 | ||||
-rw-r--r-- | guix/import/snix.scm | 467 | ||||
-rw-r--r-- | guix/lint.scm | 24 | ||||
-rw-r--r-- | guix/packages.scm | 3 | ||||
-rw-r--r-- | guix/profiles.scm | 43 | ||||
-rw-r--r-- | guix/scripts/import.scm | 4 | ||||
-rw-r--r-- | guix/scripts/import/nix.scm | 90 | ||||
-rw-r--r-- | guix/scripts/perform-download.scm | 7 | ||||
-rwxr-xr-x | guix/scripts/substitute.scm | 11 | ||||
-rw-r--r-- | guix/scripts/weather.scm | 29 | ||||
-rw-r--r-- | guix/self.scm | 3 | ||||
-rw-r--r-- | guix/ssh.scm | 30 | ||||
-rw-r--r-- | guix/status.scm | 18 | ||||
-rw-r--r-- | guix/store.scm | 72 | ||||
-rw-r--r-- | guix/swh.scm | 85 | ||||
-rw-r--r-- | guix/ui.scm | 17 |
30 files changed, 473 insertions, 759 deletions
diff --git a/guix/android-repo-download.scm b/guix/android-repo-download.scm index 5ff3e7edd4..1c3502e673 100644 --- a/guix/android-repo-download.scm +++ b/guix/android-repo-download.scm @@ -78,6 +78,9 @@ generic name if unset." (define zlib (module-ref (resolve-interface '(gnu packages compression)) 'zlib)) + (define guile-json + (module-ref (resolve-interface '(gnu packages guile)) 'guile-json-4)) + (define gnutls (module-ref (resolve-interface '(gnu packages tls)) 'gnutls)) @@ -99,7 +102,7 @@ generic name if unset." (define build (with-imported-modules modules - (with-extensions (list gnutls) + (with-extensions (list gnutls guile-json) ;for (guix swh) #~(begin (use-modules (guix build android-repo) (guix build utils) diff --git a/guix/build-system/asdf.scm b/guix/build-system/asdf.scm index 7bf2f97992..79de2ee5ba 100644 --- a/guix/build-system/asdf.scm +++ b/guix/build-system/asdf.scm @@ -1,6 +1,6 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2016, 2017 Andy Patterson <ajpatter@uwaterloo.ca> -;;; Copyright © 2019, 2020 Guillaume Le Vaillant <glv@posteo.net> +;;; Copyright © 2019, 2020, 2021 Guillaume Le Vaillant <glv@posteo.net> ;;; Copyright © 2021 Ludovic Courtès <ludo@gnu.org> ;;; ;;; This file is part of GNU Guix. @@ -279,16 +279,16 @@ set up using CL source package conventions." (imported-modules %asdf-build-system-modules) (modules %asdf-build-modules)) - ;; FIXME: The definition of 'systems' is pretty hacky. - ;; Is there a more elegant way to do it? (define systems (if (null? (cadr asd-systems)) - `(quote - ,(list - (string-drop - ;; NAME is the value returned from `package-full-name'. - (hyphen-separated-name->name+version name) - (1+ (string-length lisp-type))))) ; drop the "<lisp>-" prefix. + ;; FIXME: Find a more reliable way to get the main system name. + (let* ((lisp-prefix (string-append lisp-type "-")) + (package-name (hyphen-separated-name->name+version + (if (string-prefix? lisp-prefix name) + (string-drop name + (string-length lisp-prefix)) + name)))) + `(quote ,(list package-name))) asd-systems)) (define builder diff --git a/guix/build/asdf-build-system.scm b/guix/build/asdf-build-system.scm index 6ad855cab2..7f1037c4f9 100644 --- a/guix/build/asdf-build-system.scm +++ b/guix/build/asdf-build-system.scm @@ -1,6 +1,6 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2016, 2017 Andy Patterson <ajpatter@uwaterloo.ca> -;;; Copyright © 2020 Guillaume Le Vaillant <glv@posteo.net> +;;; Copyright © 2020, 2021 Guillaume Le Vaillant <glv@posteo.net> ;;; ;;; This file is part of GNU Guix. ;;; @@ -52,12 +52,13 @@ (string-append %source-install-prefix "/systems")) (define (main-system-name output) - (let ((package-name (package-name->name+version - (strip-store-file-name output))) - (lisp-prefix (string-append (%lisp-type) "-"))) - (if (string-prefix? lisp-prefix package-name) - (string-drop package-name (string-length lisp-prefix)) - package-name))) + ;; FIXME: Find a more reliable way to get the main system name. + (let* ((full-name (strip-store-file-name output)) + (lisp-prefix (string-append (%lisp-type) "-")) + (package-name (if (string-prefix? lisp-prefix full-name) + (string-drop full-name (string-length lisp-prefix)) + full-name))) + (package-name->name+version package-name))) (define (lisp-source-directory output name) (string-append output (%lisp-source-install-prefix) "/" name)) diff --git a/guix/build/download.scm b/guix/build/download.scm index a22d4064ca..b14db42352 100644 --- a/guix/build/download.scm +++ b/guix/build/download.scm @@ -2,6 +2,7 @@ ;;; Copyright © 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021 Ludovic Courtès <ludo@gnu.org> ;;; Copyright © 2015 Mark H Weaver <mhw@netris.org> ;;; Copyright © 2017 Tobias Geerinckx-Rice <me@tobias.gr> +;;; Copyright © 2021 Timothy Sample <samplet@ngyro.com> ;;; ;;; This file is part of GNU Guix. ;;; @@ -34,6 +35,8 @@ #:use-module (srfi srfi-19) #:use-module (srfi srfi-26) #:autoload (ice-9 ftw) (scandir) + #:autoload (guix base16) (bytevector->base16-string) + #:autoload (guix swh) (swh-download-directory) #:use-module (ice-9 match) #:use-module (ice-9 format) #:export (open-socket-for-uri @@ -626,10 +629,54 @@ Return a list of URIs." (else (list uri)))) +(define* (disarchive-fetch/any uris file + #:key (timeout 10) (verify-certificate? #t)) + "Fetch a Disarchive specification from any of URIS, assemble it, +and write the output to FILE." + (define (fetch-specification uris) + (any (lambda (uri) + (false-if-exception* + (let-values (((port size) (http-fetch uri + #:verify-certificate? + verify-certificate? + #:timeout timeout))) + (let ((specification (read port))) + (close-port port) + specification)))) + uris)) + + (define (resolve addresses output) + (any (match-lambda + (('swhid swhid) + (match (string-split swhid #\:) + (("swh" "1" "dir" id) + (format #t "Downloading ~a from Software Heritage...~%" file) + (false-if-exception* + (swh-download-directory id output))) + (_ #f))) + (_ #f)) + addresses)) + + (format #t "Trying to use Disarchive to assemble ~a...~%" file) + (match (and=> (resolve-module '(disarchive) #:ensure #f) + (lambda (disarchive) + (cons (module-ref disarchive '%disarchive-log-port) + (module-ref disarchive 'disarchive-assemble)))) + (#f (format #t "could not load Disarchive~%") + #f) + ((%disarchive-log-port . disarchive-assemble) + (match (fetch-specification uris) + (#f (format #t "could not find its Disarchive specification~%") + #f) + (spec (parameterize ((%disarchive-log-port (current-output-port))) + (false-if-exception* + (disarchive-assemble spec file #:resolver resolve)))))))) + (define* (url-fetch url file #:key (timeout 10) (verify-certificate? #t) (mirrors '()) (content-addressed-mirrors '()) + (disarchive-mirrors '()) (hashes '()) print-build-trace?) "Fetch FILE from URL; URL may be either a single string, or a list of @@ -693,6 +740,18 @@ otherwise simply ignore them." hashes)) content-addressed-mirrors)) + (define disarchive-uris + (append-map (match-lambda + ((? string? mirror) + (map (match-lambda + ((hash-algo . hash) + (string->uri + (string-append mirror + (symbol->string hash-algo) "/" + (bytevector->base16-string hash))))) + hashes))) + disarchive-mirrors)) + ;; Make this unbuffered so 'progress-report/file' works as expected. 'line ;; means '\n', not '\r', so it's not appropriate here. (setvbuf (current-output-port) 'none) @@ -705,15 +764,20 @@ otherwise simply ignore them." (or (fetch uri file) (try tail))) (() - (format (current-error-port) "failed to download ~s from ~s~%" - file url) - - ;; Remove FILE in case we made an incomplete download, for example due - ;; to ENOSPC. - (catch 'system-error - (lambda () - (delete-file file)) - (const #f)) - #f)))) + ;; If we are looking for a software archive, one last thing we + ;; can try is to use Disarchive to assemble it. + (or (disarchive-fetch/any disarchive-uris file + #:verify-certificate? verify-certificate? + #:timeout timeout) + (begin + (format (current-error-port) "failed to download ~s from ~s~%" + file url) + ;; Remove FILE in case we made an incomplete download, for + ;; example due to ENOSPC. + (catch 'system-error + (lambda () + (delete-file file)) + (const #f)) + #f)))))) ;;; download.scm ends here diff --git a/guix/build/emacs-build-system.scm b/guix/build/emacs-build-system.scm index 79a1a42c4a..ba2c1b4aad 100644 --- a/guix/build/emacs-build-system.scm +++ b/guix/build/emacs-build-system.scm @@ -26,13 +26,16 @@ #:use-module (srfi srfi-1) #:use-module (srfi srfi-11) #:use-module (srfi srfi-26) + #:use-module (ice-9 format) + #:use-module (ice-9 ftw) #:use-module (ice-9 rdelim) #:use-module (ice-9 regex) #:use-module (ice-9 match) #:export (%standard-phases %default-include %default-exclude - emacs-build)) + emacs-build + elpa-directory)) ;; Commentary: ;; @@ -40,9 +43,12 @@ ;; ;; Code: -;;; All the packages are installed directly under site-lisp, which means that -;;; having that directory in the EMACSLOADPATH is enough to have them found by -;;; Emacs. +;;; The location in which Emacs looks for packages. Emacs Lisp code that is +;;; installed there directly will be found when that directory is added to +;;; EMACSLOADPATH. To avoid clashes between packages (particularly considering +;;; auxiliary files), we install them one directory level below, however. +;;; This indirection is handled by ‘expand-load-path’ during build and a +;;; profile hook otherwise. (define %install-dir "/share/emacs/site-lisp") ;; These are the default inclusion/exclusion regexps for the install phase. @@ -73,40 +79,51 @@ archive, a directory, or an Emacs Lisp file." #t) (gnu:unpack #:source source))) -(define* (add-source-to-load-path #:key dummy #:allow-other-keys) - "Augment the EMACSLOADPATH environment variable with the source directory." +(define* (expand-load-path #:key (prepend-source? #t) #:allow-other-keys) + "Expand EMACSLOADPATH, so that inputs, whose code resides in subdirectories, +are properly found. +If @var{prepend-source?} is @code{#t} (the default), also add the current +directory to EMACSLOADPATH in front of any other directories." (let* ((source-directory (getcwd)) (emacs-load-path (string-split (getenv "EMACSLOADPATH") #\:)) - ;; XXX: Make sure the Emacs core libraries appear at the end of - ;; EMACSLOADPATH, to avoid shadowing any other libraries depended - ;; upon. - (emacs-load-path-non-core (filter (cut string-contains <> - "/share/emacs/site-lisp") - emacs-load-path)) + (emacs-load-path* + (map + (lambda (dir) + (match (scandir dir (negate (cute member <> '("." "..")))) + ((sub) (string-append dir "/" sub)) + (_ dir))) + emacs-load-path)) (emacs-load-path-value (string-append - (string-join (cons source-directory - emacs-load-path-non-core) - ":") + (string-join + (if prepend-source? + (cons source-directory emacs-load-path*) + emacs-load-path*) + ":") ":"))) (setenv "EMACSLOADPATH" emacs-load-path-value) - (format #t "source directory ~s prepended to the `EMACSLOADPATH' \ -environment variable\n" source-directory))) + (when prepend-source? + (format #t "source directory ~s prepended to the `EMACSLOADPATH' \ +environment variable\n" source-directory)) + (let ((diff (lset-difference string=? emacs-load-path* emacs-load-path))) + (unless (null? diff) + (format #t "expanded load paths for ~{~a~^, ~}\n" + (map basename diff)))))) (define* (build #:key outputs inputs #:allow-other-keys) "Compile .el files." (let* ((emacs (string-append (assoc-ref inputs "emacs") "/bin/emacs")) - (out (assoc-ref outputs "out")) - (site-lisp (string-append out %install-dir))) + (out (assoc-ref outputs "out"))) (setenv "SHELL" "sh") (parameterize ((%emacs emacs)) - (emacs-byte-compile-directory site-lisp)))) + (emacs-byte-compile-directory (elpa-directory out))))) (define* (patch-el-files #:key outputs #:allow-other-keys) "Substitute the absolute \"/bin/\" directory with the right location in the store in '.el' files." (let* ((out (assoc-ref outputs "out")) - (site-lisp (string-append out %install-dir)) + (elpa-name-ver (store-directory->elpa-name-version out)) + (el-dir (string-append out %install-dir "/" elpa-name-ver)) (el-files (find-files (getcwd) "\\.el$"))) (define (substitute-program-names) (substitute* el-files @@ -116,7 +133,7 @@ store in '.el' files." (error "patch-el-files: unable to locate " cmd-name)) (string-append "\"" cmd "\""))))) - (with-directory-excursion site-lisp + (with-directory-excursion el-dir ;; Some old '.el' files (e.g., tex-buf.el in AUCTeX) are still ;; ISO-8859-1-encoded. (unless (false-if-exception (substitute-program-names)) @@ -167,14 +184,14 @@ parallel. PARALLEL-TESTS? is ignored when using a non-make TEST-COMMAND." (not (any (cut match-stripped-file "excluded" <>) exclude))))) (let* ((out (assoc-ref outputs "out")) - (site-lisp (string-append out %install-dir)) + (el-dir (elpa-directory out)) (files-to-install (find-files source install-file?))) (cond ((not (null? files-to-install)) (for-each (lambda (file) (let* ((stripped-file (string-drop file (string-length source))) - (target-file (string-append site-lisp stripped-file))) + (target-file (string-append el-dir stripped-file))) (format #t "`~a' -> `~a'~%" file target-file) (install-file file (dirname target-file)))) files-to-install) @@ -205,11 +222,11 @@ parallel. PARALLEL-TESTS? is ignored when using a non-make TEST-COMMAND." "Generate the autoloads file." (let* ((emacs (string-append (assoc-ref inputs "emacs") "/bin/emacs")) (out (assoc-ref outputs "out")) - (site-lisp (string-append out %install-dir)) (elpa-name-ver (store-directory->elpa-name-version out)) - (elpa-name (package-name->name+version elpa-name-ver))) + (elpa-name (package-name->name+version elpa-name-ver)) + (el-dir (elpa-directory out))) (parameterize ((%emacs emacs)) - (emacs-generate-autoloads elpa-name site-lisp)))) + (emacs-generate-autoloads elpa-name el-dir)))) (define* (enable-autoloads-compilation #:key outputs #:allow-other-keys) "Remove the NO-BYTE-COMPILATION local variable embedded in the generated @@ -244,10 +261,16 @@ second hyphen. This corresponds to 'name-version' as used in ELPA packages." strip-store-file-name) store-dir)) +(define (elpa-directory store-dir) + "Given the store directory STORE-DIR return the absolute install directory +for libraries following the ELPA convention." + (string-append store-dir %install-dir "/" + (store-directory->elpa-name-version store-dir))) + (define %standard-phases (modify-phases gnu:%standard-phases (replace 'unpack unpack) - (add-after 'unpack 'add-source-to-load-path add-source-to-load-path) + (add-after 'unpack 'expand-load-path expand-load-path) (delete 'bootstrap) (delete 'configure) (delete 'build) diff --git a/guix/ci.scm b/guix/ci.scm index f04109112c..c70e5bb9e6 100644 --- a/guix/ci.scm +++ b/guix/ci.scm @@ -43,7 +43,7 @@ checkout? checkout-commit - checkout-input + checkout-channel evaluation? evaluation-id @@ -94,7 +94,7 @@ (define-json-mapping <checkout> make-checkout checkout? json->checkout (commit checkout-commit) ;string (SHA1) - (input checkout-input)) ;string (name) + (channel checkout-channel)) ;string (name) (define-json-mapping <evaluation> make-evaluation evaluation? json->evaluation diff --git a/guix/cve.scm b/guix/cve.scm index b3a8b13a06..9e1cf5b587 100644 --- a/guix/cve.scm +++ b/guix/cve.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2015, 2016, 2017, 2018, 2019, 2020 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2015, 2016, 2017, 2018, 2019, 2020, 2021 Ludovic Courtès <ludo@gnu.org> ;;; ;;; This file is part of GNU Guix. ;;; @@ -99,7 +99,9 @@ (define (reference-data->cve-references alist) (map json->cve-reference - (vector->list (assoc-ref alist "reference_data")))) + ;; Normally "reference_data" is always present but rejected CVEs such + ;; as CVE-2020-10020 can lack it. + (vector->list (or (assoc-ref alist "reference_data") '#())))) (define %cpe-package-rx ;; For applications: "cpe:2.3:a:VENDOR:PACKAGE:VERSION", or sometimes @@ -137,17 +139,20 @@ package." (starte (assoc-ref alist "versionStartExcluding")) (endi (assoc-ref alist "versionEndIncluding")) (ende (assoc-ref alist "versionEndExcluding"))) - (let-values (((package version) (cpe->package-name cpe))) - (and package - `(,package - ,(cond ((and (or starti starte) (or endi ende)) - `(and ,(if starti `(>= ,starti) `(> ,starte)) - ,(if endi `(<= ,endi) `(< ,ende)))) - (starti `(>= ,starti)) - (starte `(> ,starte)) - (endi `(<= ,endi)) - (ende `(< ,ende)) - (else version))))))) + ;; Normally "cpe23Uri" is here in each "cpe_match" item, but CVE-2020-0534 + ;; has a configuration that lacks it. + (and cpe + (let-values (((package version) (cpe->package-name cpe))) + (and package + `(,package + ,(cond ((and (or starti starte) (or endi ende)) + `(and ,(if starti `(>= ,starti) `(> ,starte)) + ,(if endi `(<= ,endi) `(< ,ende)))) + (starti `(>= ,starti)) + (starte `(> ,starte)) + (endi `(<= ,endi)) + (ende `(< ,ende)) + (else version)))))))) (define (configuration-data->cve-configurations alist) "Given ALIST, a JSON dictionary for the baroque \"configurations\" diff --git a/guix/cvs-download.scm b/guix/cvs-download.scm index 76b3eac739..943d971622 100644 --- a/guix/cvs-download.scm +++ b/guix/cvs-download.scm @@ -28,7 +28,8 @@ #:use-module (ice-9 match) #:export (cvs-reference cvs-reference? - cvs-reference-url + cvs-reference-root-directory + cvs-reference-module cvs-reference-revision cvs-fetch)) @@ -63,13 +64,20 @@ HASH-ALGO (a symbol). Use NAME as the file name, or a generic name if #f." (define guile-zlib (module-ref (resolve-interface '(gnu packages guile)) 'guile-zlib)) + (define guile-json + (module-ref (resolve-interface '(gnu packages guile)) 'guile-json-4)) + + (define gnutls + (module-ref (resolve-interface '(gnu packages tls)) 'gnutls)) + (define modules (delete '(guix config) (source-module-closure '((guix build cvs) (guix build download-nar))))) (define build (with-imported-modules modules - (with-extensions (list guile-zlib) + (with-extensions (list guile-json gnutls ;for (guix swh) + guile-zlib) #~(begin (use-modules (guix build cvs) (guix build download-nar)) diff --git a/guix/diagnostics.scm b/guix/diagnostics.scm index 7b9ffc61b5..6a792febd4 100644 --- a/guix/diagnostics.scm +++ b/guix/diagnostics.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021 Ludovic Courtès <ludo@gnu.org> ;;; ;;; This file is part of GNU Guix. ;;; @@ -233,6 +233,10 @@ etc." (make-location file (+ line 1) col))) (#f #f) + (#(file line column) + ;; Guile >= 3.0.6 uses vectors instead of alists internally, which can be + ;; seen in the arguments to 'syntax-error' exceptions. + (location file (+ 1 line) column)) (_ (let ((file (assq-ref loc 'filename)) (line (assq-ref loc 'line)) diff --git a/guix/download.scm b/guix/download.scm index 30f69c0325..72094e7318 100644 --- a/guix/download.scm +++ b/guix/download.scm @@ -406,12 +406,19 @@ (plain-file "content-addressed-mirrors" (object->string %content-addressed-mirrors))) +(define %disarchive-mirrors + '("https://disarchive.ngyro.com/")) + +(define %disarchive-mirror-file + (plain-file "disarchive-mirrors" (object->string %disarchive-mirrors))) + (define built-in-builders* (store-lift built-in-builders)) (define* (built-in-download file-name url #:key system hash-algo hash mirrors content-addressed-mirrors + disarchive-mirrors executable? (guile 'unused)) "Download FILE-NAME from URL using the built-in 'download' builder. When @@ -422,13 +429,16 @@ explicitly depend on Guile, GnuTLS, etc. Instead, the daemon performs the download by itself using its own dependencies." (mlet %store-monad ((mirrors (lower-object mirrors)) (content-addressed-mirrors - (lower-object content-addressed-mirrors))) + (lower-object content-addressed-mirrors)) + (disarchive-mirrors (lower-object disarchive-mirrors))) (raw-derivation file-name "builtin:download" '() #:system system #:hash-algo hash-algo #:hash hash #:recursive? executable? - #:sources (list mirrors content-addressed-mirrors) + #:sources (list mirrors + content-addressed-mirrors + disarchive-mirrors) ;; Honor the user's proxy and locale settings. #:leaked-env-vars '("http_proxy" "https_proxy" @@ -439,6 +449,7 @@ download by itself using its own dependencies." ("mirrors" . ,mirrors) ("content-addressed-mirrors" . ,content-addressed-mirrors) + ("disarchive-mirrors" . ,disarchive-mirrors) ,@(if executable? '(("executable" . "1")) '())) @@ -492,7 +503,9 @@ name in the store." #:executable? executable? #:mirrors %mirror-file #:content-addressed-mirrors - %content-addressed-mirror-file))))) + %content-addressed-mirror-file + #:disarchive-mirrors + %disarchive-mirror-file))))) (define* (url-fetch/executable url hash-algo hash #:optional name diff --git a/guix/git-download.scm b/guix/git-download.scm index 425184717a..199effece5 100644 --- a/guix/git-download.scm +++ b/guix/git-download.scm @@ -27,6 +27,7 @@ #:use-module (guix packages) #:use-module (guix modules) #:autoload (guix build-system gnu) (standard-packages) + #:autoload (git bindings) (libgit2-init!) #:autoload (git repository) (repository-open repository-close! repository-discover @@ -225,6 +226,7 @@ upon Git errors, return #f instead of a predicate. The returned predicate takes two arguments FILE and STAT where FILE is an absolute file name and STAT is the result of 'lstat'." + (libgit2-init!) (catch 'git-error (lambda () (let* ((files (git-file-list directory)) diff --git a/guix/hg-download.scm b/guix/hg-download.scm index bd55946523..c6cee2dbb8 100644 --- a/guix/hg-download.scm +++ b/guix/hg-download.scm @@ -65,6 +65,12 @@ HASH-ALGO (a symbol). Use NAME as the file name, or a generic name if #f." (define guile-zlib (module-ref (resolve-interface '(gnu packages guile)) 'guile-zlib)) + (define guile-json + (module-ref (resolve-interface '(gnu packages guile)) 'guile-json-4)) + + (define gnutls + (module-ref (resolve-interface '(gnu packages tls)) 'gnutls)) + (define modules (delete '(guix config) (source-module-closure '((guix build hg) @@ -72,7 +78,8 @@ HASH-ALGO (a symbol). Use NAME as the file name, or a generic name if #f." (define build (with-imported-modules modules - (with-extensions (list guile-zlib) + (with-extensions (list guile-json gnutls ;for (guix swh) + guile-zlib) #~(begin (use-modules (guix build hg) (guix build download-nar)) diff --git a/guix/http-client.scm b/guix/http-client.scm index a2e11a1b73..10bc278023 100644 --- a/guix/http-client.scm +++ b/guix/http-client.scm @@ -38,7 +38,7 @@ #:use-module (guix utils) #:use-module (guix base64) #:autoload (gcrypt hash) (sha256) - #:autoload (gnutls) (error/invalid-session) + #:autoload (gnutls) (error/invalid-session error/again error/interrupted) #:use-module ((guix build utils) #:select (mkdir-p dump-port)) #:use-module ((guix build download) @@ -163,7 +163,14 @@ reusing stale cached connections." (if (or (and (eq? key 'system-error) (= EPIPE (system-error-errno `(,key ,@args)))) (and (eq? key 'gnutls-error) - (eq? (first args) error/invalid-session)) + (memq (first args) + (list error/invalid-session + + ;; XXX: These two are not properly handled in + ;; GnuTLS < 3.7.2, in + ;; 'write_to_session_record_port'; see + ;; <https://bugs.gnu.org/47867>. + error/again error/interrupted))) (memq key '(bad-response bad-header bad-header-component))) #f @@ -207,15 +214,14 @@ returning." ;; Inherit the HTTP proxying property from P. (set-http-proxy-port?! buffer (http-proxy-port? p)) - (unless (false-if-networking-error - (begin - (for-each (cut write-request <> buffer) batch) - (put-bytevector p (get)) - (force-output p) - #t)) - ;; If PORT becomes unusable, open a fresh connection and retry. - (close-port p) ; close the broken port - (connect #f requests result))) + ;; Swallow networking errors that could occur due to connection reuse + ;; and the like; they will be handled down the road when trying to + ;; read responses. + (false-if-networking-error + (begin + (for-each (cut write-request <> buffer) batch) + (put-bytevector p (get)) + (force-output p)))) ;; Now start processing responses. (let loop ((sent batch) diff --git a/guix/import/go.scm b/guix/import/go.scm index bc53f8f558..d110954664 100644 --- a/guix/import/go.scm +++ b/guix/import/go.scm @@ -33,7 +33,7 @@ #:use-module (guix http-client) #:use-module ((guix licenses) #:prefix license:) #:use-module (guix memoization) - #:use-module (htmlprag) ;from Guile-Lib + #:autoload (htmlprag) (html->sxml) ;from Guile-Lib #:autoload (guix git) (update-cached-checkout) #:autoload (gcrypt hash) (open-hash-port hash-algorithm sha256) #:autoload (guix serialization) (write-file) @@ -63,9 +63,6 @@ #:export (go-module->guix-package go-module-recursive-import)) -;;; Parameterize htmlprag to parse valid HTML more reliably. -(%strict-tokenizer? #t) - ;;; Commentary: ;;; ;;; (guix import go) attempts to make it easier to create Guix package @@ -149,7 +146,7 @@ name (e.g. \"github.com/golang/protobuf/proto\")." ;; element marked with a "License" class attribute. (select (sxpath `(// (* (@ (equal? (class "License")))) h2 // *text*)))) - (select (html->sxml body)))) + (select (html->sxml body #:strict? #t)))) (define (sxml->texi sxml-node) "A very basic SXML to Texinfo converter which attempts to preserve HTML @@ -167,7 +164,7 @@ formatting and links as text." "Retrieve a short description for NAME, a Go package name, e.g. \"google.golang.org/protobuf/proto\"." (let* ((body (go.pkg.dev-info name)) - (sxml (html->sxml body)) + (sxml (html->sxml body #:strict? #t)) (overview ((sxpath `(// (* (@ (equal? (class "Documentation-overview")))) @@ -209,7 +206,7 @@ the https://pkg.go.dev/ web site." (select-title (sxpath `(// (div (@ (equal? (class "UnitReadme-content")))) // h3 *text*)))) - (match (select-title (html->sxml body)) + (match (select-title (html->sxml body #:strict? #t)) (() #f) ;nothing selected ((title more ...) ;title is the first string of the list (string-trim-both title))))) @@ -465,7 +462,7 @@ build a package." (let* ((meta-data (http-fetch* (format #f "https://~a?go-get=1" module-path))) (select (sxpath `(// head (meta (@ (equal? (name "go-import")))) // content)))) - (match (select (html->sxml meta-data)) + (match (select (html->sxml meta-data #:strict? #t)) (() #f) ;nothing selected (((content content-text)) (match (string-split content-text #\space) diff --git a/guix/import/pypi.scm b/guix/import/pypi.scm index bf4dc50138..6731d50891 100644 --- a/guix/import/pypi.scm +++ b/guix/import/pypi.scm @@ -229,8 +229,8 @@ the input field." '("test" "dev"))) (define (parse-requires.txt requires.txt) - "Given REQUIRES.TXT, a Setuptools requires.txt file, return a list of lists -of requirements. + "Given REQUIRES.TXT, a path to a Setuptools requires.txt file, return a list +of lists of requirements. The first list contains the required dependencies while the second the optional test dependencies. Note that currently, optional, non-test diff --git a/guix/import/snix.scm b/guix/import/snix.scm deleted file mode 100644 index 56934e8cf9..0000000000 --- a/guix/import/snix.scm +++ /dev/null @@ -1,467 +0,0 @@ -;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018 Ludovic Courtès <ludo@gnu.org> -;;; -;;; This file is part of GNU Guix. -;;; -;;; GNU Guix is free software; you can redistribute it and/or modify it -;;; under the terms of the GNU General Public License as published by -;;; the Free Software Foundation; either version 3 of the License, or (at -;;; your option) any later version. -;;; -;;; GNU Guix is distributed in the hope that it will be useful, but -;;; WITHOUT ANY WARRANTY; without even the implied warranty of -;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -;;; GNU General Public License for more details. -;;; -;;; You should have received a copy of the GNU General Public License -;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>. - -(define-module (guix import snix) - #:use-module (sxml ssax) - #:use-module (ice-9 popen) - #:use-module (ice-9 match) - #:use-module (ice-9 rdelim) - #:use-module (ice-9 format) - #:use-module (ice-9 regex) - #:use-module (ice-9 vlist) - #:use-module (srfi srfi-1) - #:use-module (srfi srfi-9) - #:use-module (srfi srfi-11) - #:use-module (srfi srfi-26) - #:use-module (srfi srfi-37) - #:use-module (system foreign) - #:use-module (rnrs bytevectors) - - ;; Use the 'package-name->name+version' procedure that works with - ;; hyphen-separate name/version, not the one that works with '@'-separated - ;; name/version. Subtle! - #:use-module ((guix utils) #:hide (package-name->name+version)) - #:use-module ((guix build utils) #:select (package-name->name+version)) - - #:use-module (guix import utils) - #:use-module (guix base16) - #:use-module (guix base32) - #:use-module (guix config) - #:use-module (guix gnu-maintenance) - #:export (open-nixpkgs - xml->snix - nixpkgs->guix-package)) - -;;; Commentary: -;;; -;;; Converting Nix code to s-expressions, and then to Guix `package' -;;; declarations, using the XML output of `nix-instantiate'. -;;; -;;; Code: - - -;;; -;;; SNix. -;;; - -;; Nix object types visible in the XML output of `nix-instantiate' and -;; mapping to S-expressions (we map to sexps, not records, so that we -;; can do pattern matching): -;; -;; at (at varpat attrspat) -;; attr (attribute loc name value) -;; attrs (attribute-set attributes) -;; attrspat (attribute-set-pattern patterns) -;; bool #f|#t -;; derivation (derivation drv-path out-path attributes) -;; ellipsis '... -;; expr (snix loc body ...) -;; function (function loc at|attrspat|varpat) -;; int int -;; list list -;; null 'null -;; path string -;; string string -;; unevaluated 'unevaluated -;; varpat (varpat name) -;; -;; Initially ATTRIBUTES in `derivation' and `attribute-set' was a promise; -;; however, handling `repeated' nodes makes it impossible to do anything -;; lazily because the whole SXML tree has to be traversed to maintain the -;; list of known derivations. - -(define (xml-element->snix elem attributes body derivations) - "Return an SNix element corresponding to XML element ELEM." - - (define (loc) - (location (assq-ref attributes 'path) - (assq-ref attributes 'line) - (assq-ref attributes 'column))) - - (case elem - ((at) - (values `(at ,(car body) ,(cadr body)) derivations)) - ((attr) - (let ((name (assq-ref attributes 'name))) - (cond ((null? body) - (values `(attribute-pattern ,name) derivations)) - ((and (pair? body) (null? (cdr body))) - (values `(attribute ,(loc) ,name ,(car body)) - derivations)) - (else - (error "invalid attribute body" name (loc) body))))) - ((attrs) - (values `(attribute-set ,(reverse body)) derivations)) - ((attrspat) - (values `(attribute-set-pattern ,body) derivations)) - ((bool) - (values (string-ci=? "true" (assq-ref attributes 'value)) - derivations)) - ((derivation) - (let ((drv-path (assq-ref attributes 'drvPath)) - (out-path (assq-ref attributes 'outPath))) - (if (equal? body '(repeated)) - (let ((body (vhash-assoc drv-path derivations))) - (if (pair? body) - (values `(derivation ,drv-path ,out-path ,(cdr body)) - derivations) - - ;; DRV-PATH hasn't been encountered yet but may be later - ;; (see <http://article.gmane.org/gmane.linux.distributions.nixos/5946>.) - ;; Return an `unresolved' node. - (values `(unresolved - ,(lambda (derivations) - (let ((body (vhash-assoc drv-path derivations))) - (if (pair? body) - `(derivation ,drv-path ,out-path - ,(cdr body)) - (error "no previous occurrence of derivation" - drv-path))))) - derivations))) - (values `(derivation ,drv-path ,out-path ,body) - (vhash-cons drv-path body derivations))))) - ((ellipsis) - (values '... derivations)) - ((expr) - (values `(snix ,(loc) ,@body) derivations)) - ((function) - (values `(function ,(loc) ,body) derivations)) - ((int) - (values (string->number (assq-ref attributes 'value)) - derivations)) - ((list) - (values body derivations)) - ((null) - (values 'null derivations)) - ((path) - (values (assq-ref attributes 'value) derivations)) - ((repeated) - (values 'repeated derivations)) - ((string) - (values (assq-ref attributes 'value) derivations)) - ((unevaluated) - (values 'unevaluated derivations)) - ((varpat) - (values `(varpat ,(assq-ref attributes 'name)) derivations)) - (else (error "unhandled Nix XML element" elem)))) - -(define (resolve snix derivations) - "Return a new SNix tree where `unresolved' nodes from SNIX have been -replaced by the result of their application to DERIVATIONS, a vhash." - (let loop ((node snix) - (seen vlist-null)) - (if (vhash-assq node seen) - (values node seen) - (match node - (('unresolved proc) - (let ((n (proc derivations))) - (values n seen))) - ((tag body ...) - (let ((body+seen (fold (lambda (n body+seen) - (call-with-values - (lambda () - (loop n (cdr body+seen))) - (lambda (n* seen) - (cons (cons n* (car body+seen)) - (vhash-consq n #t seen))))) - (cons '() (vhash-consq node #t seen)) - body))) - (values (cons tag (reverse (car body+seen))) - (vhash-consq node #t (cdr body+seen))))) - (anything - (values anything seen)))))) - -(define xml->snix - (let ((parse - (ssax:make-parser NEW-LEVEL-SEED - (lambda (elem-gi attributes namespaces expected-content - seed) - (cons '() (cdr seed))) - - FINISH-ELEMENT - (lambda (elem-gi attributes namespaces parent-seed - seed) - (let ((snix (car seed)) - (derivations (cdr seed))) - (let-values (((snix derivations) - (xml-element->snix elem-gi - attributes - snix - derivations))) - (cons (cons snix (car parent-seed)) - derivations)))) - - CHAR-DATA-HANDLER - (lambda (string1 string2 seed) - ;; Discard inter-node strings, which are blanks. - seed)))) - (lambda (port) - "Return the SNix represention of TREE, an SXML tree as returned by -parsing the XML output of `nix-instantiate' on Nixpkgs." - (match (parse port (cons '() vlist-null)) - (((snix) . derivations) - (resolve snix derivations)))))) - -(define (attribute-value attribute) - "Return the value of ATTRIBUTE." - (match attribute - (('attribute _ _ value) value))) - -(define (derivation-source derivation) - "Return the \"src\" attribute of DERIVATION or #f if not found." - (match derivation - (('derivation _ _ (attributes ...)) - (find-attribute-by-name "src" attributes)))) - -(define (derivation-output-path derivation) - "Return the output path of DERIVATION." - (match derivation - (('derivation _ out-path _) - out-path) - (_ #f))) - -(define (source-output-path src) - "Return the output path of SRC, the \"src\" attribute of a derivation." - (derivation-output-path (attribute-value src))) - -(define (source-urls src) - "Return the URLs of SRC, the \"src\" attribute of a derivation." - (match src - (('attribute _ _ ('derivation _ _ (attributes ...))) - (match (find-attribute-by-name "urls" attributes) - (('attribute _ _ value) - value))) - (_ #f))) - -(define (source-sha256 src) - "Return the sha256 of SRC, the \"src\" attribute of a derivation, as a -bytevector." - (match src - (('attribute _ _ ('derivation _ _ (attributes ...))) - (match (find-attribute-by-name "outputHash" attributes) - (('attribute _ _ value) - (match value - ((= string-length 52) - (nix-base32-string->bytevector value)) - ((= string-length 64) - (base16-string->bytevector value)) - (_ - (error "unsupported hash format" value)))))) - (_ #f))) - -(define (derivation-source-output-path derivation) - "Return the output path of the \"src\" attribute of DERIVATION or #f -if DERIVATION lacks an \"src\" attribute." - (and=> (derivation-source derivation) source-output-path)) - -(define* (open-nixpkgs nixpkgs #:optional attribute) - "Return an input pipe to the XML representation of Nixpkgs. When -ATTRIBUTE is true, only that attribute is considered." - (with-fluids ((%default-port-encoding "UTF-8")) - (let ((cross-system (format #f "{ - config = \"i686-guix-linux-gnu\"; - libc = \"glibc\"; - arch = \"guix\"; - withTLS = true; - float = \"hard\"; - openssl.system = \"linux-generic32\"; - platform = (import ~a/pkgs/top-level/platforms.nix).sheevaplug; -}" nixpkgs))) - (apply open-pipe* OPEN_READ - "nix-instantiate" "--strict" "--eval-only" "--xml" - - ;; Pass a dummy `crossSystem' argument so that `buildInputs' and - ;; `nativeBuildInputs' are not coalesced. - ;; XXX: This is hacky and has other problems. - ;"--arg" "crossSystem" cross-system - - `(,@(if attribute - `("-A" ,attribute) - '()) - ,nixpkgs))))) - -(define (pipe-failed? pipe) - "Close pipe and return its status if it failed." - (let ((status (close-pipe pipe))) - (if (or (status:term-sig status) - (not (= (status:exit-val status) 0))) - status - #f))) - -(define (find-attribute-by-name name attributes) - "Return attribute NAME in ATTRIBUTES, an attribute set or list of SNix -attributes, or #f if NAME cannot be found." - (find (lambda (a) - (match a - (('attribute _ (? (cut string=? <> name)) _) - a) - (_ #f))) - (match attributes - (('attribute-set (attributes ...)) - attributes) - (_ - attributes)))) - -(define (license-variable license) - "Return the name of the (guix licenses) variable for LICENSE." - (match license - ("GPLv2+" 'gpl2+) - ("GPLv3+" 'gpl3+) - ("LGPLv2+" 'lgpl2.1+) - ("LGPLv2.1+" 'lgpl2.1+) - ("LGPLv3+" 'lgpl3+) - (('attribute-set _ ...) - ;; At some point in 2013, Nixpkgs switched to attribute sets to represent - ;; licenses. These are listed in lib/licenses.nix. - (match (and=> (find-attribute-by-name "shortName" license) - attribute-value) - ("agpl3Plus" 'agpl3+) - ("gpl2Plus" 'gpl2+) - ("gpl3Plus" 'gpl3+) - ("lgpl2Plus" 'lgpl2.0+) - ("lgpl21Plus" 'lgpl2.1+) - ("lgpl3Plus" 'lgpl3+) - ((? string? x) x) - (_ license))) - (_ license))) - -(define (package-source-output-path package) - "Return the output path of the \"src\" derivation of PACKAGE." - (derivation-source-output-path (attribute-value package))) - - -;;; -;;; Conversion of "Nix expressions" to "Guix expressions". -;;; - -(define (snix-derivation->guix-package derivation) - "Return the `package' s-expression corresponding to SNix DERIVATION, a -Nixpkgs `stdenv.mkDerivation'-style derivation, and the original source -location of DERIVATION." - (match derivation - (('derivation _ _ (attributes ...)) - (let*-values (((full-name loc) - (match (find-attribute-by-name "name" attributes) - (('attribute loc _ value) - (values value loc)) - (_ - (values #f #f)))) - ((name version) - (package-name->name+version full-name))) - (define (convert-inputs type) - ;; Convert the derivation's input from a list of SNix derivations to - ;; a list of name/variable pairs. - (match (and=> (find-attribute-by-name type attributes) - attribute-value) - (#f - '()) - ((inputs ...) - ;; Inputs can be either derivations or the null value. - (filter-map (match-lambda - (('derivation _ _ (attributes ...)) - (let* ((full-name - (attribute-value - (find-attribute-by-name "name" attributes))) - (name (package-name->name+version full-name))) - (list name - (list 'unquote (string->symbol name))))) - ('null #f)) - inputs)))) - - (define (maybe-inputs guix-name inputs) - (match inputs - (() - '()) - ((inputs ...) - (list (list guix-name - (list 'quasiquote inputs)))))) - - (define (pretty-uri uri version) - (if version - (match (factorize-uri uri version) - ((items ...) - `(string-append ,@items)) - (x x)) - uri)) - - (let* ((source (find-attribute-by-name "src" attributes)) - (urls (source-urls source)) - (sha256 (source-sha256 source)) - (meta (and=> (find-attribute-by-name "meta" attributes) - attribute-value))) - (values - `(package - (name ,name) - (version ,version) - (source (origin - (method url-fetch) - (uri ,(pretty-uri (car urls) version)) - (sha256 - (base32 - ,(bytevector->nix-base32-string sha256))))) - (build-system gnu-build-system) - - ;; When doing a native Nixpkgs build, `buildInputs' is empty and - ;; everything is in `nativeBuildInputs'. So we can't distinguish - ;; between both, here. - ;; - ;; Note that `nativeBuildInputs' was renamed from - ;; `buildNativeInputs' in Nixpkgs sometime around March 2013. - ,@(maybe-inputs 'inputs - (convert-inputs "nativeBuildInputs")) - ,@(maybe-inputs 'propagated-inputs - (convert-inputs "propagatedNativeBuildInputs")) - - (home-page ,(and=> (find-attribute-by-name "homepage" meta) - attribute-value)) - (synopsis - ;; For GNU packages, prefer the official synopsis. - ,(or (false-if-exception - (and=> (find (lambda (gnu-package) - (equal? (gnu-package-name gnu-package) - name)) - (official-gnu-packages)) - gnu-package-doc-summary)) - (and=> (find-attribute-by-name "description" meta) - attribute-value))) - (description - ;; Likewise, prefer the official description of GNU packages. - ,(or (false-if-exception - (and=> (find (lambda (gnu-package) - (equal? (gnu-package-name gnu-package) - name)) - (official-gnu-packages)) - gnu-package-doc-description)) - (and=> (find-attribute-by-name "longDescription" meta) - attribute-value))) - (license ,(and=> (find-attribute-by-name "license" meta) - (compose license-variable attribute-value)))) - loc)))))) - -(define (nixpkgs->guix-package nixpkgs attribute) - "Evaluate ATTRIBUTE in NIXPKGS, the file name of a Nixpkgs checkout, -and return the `package' s-expression corresponding to that package." - (let ((port (open-nixpkgs nixpkgs attribute))) - (match (xml->snix port) - (('snix loc (and drv ('derivation _ ...))) - (and (not (pipe-failed? port)) - (snix-derivation->guix-package drv))) - (_ - (not (pipe-failed? port)))))) - -;;; snix.scm ends here diff --git a/guix/lint.scm b/guix/lint.scm index a7d6bbba4f..1bebfe03d3 100644 --- a/guix/lint.scm +++ b/guix/lint.scm @@ -11,6 +11,7 @@ ;;; Copyright © 2018, 2019 Arun Isaac <arunisaac@systemreboot.net> ;;; Copyright © 2020 Chris Marusich <cmmarusich@gmail.com> ;;; Copyright © 2020 Timothy Sample <samplet@ngyro.com> +;;; Copyright © 2021 Xinglu Chen <public@yoctocell.xyz> ;;; ;;; This file is part of GNU Guix. ;;; @@ -81,6 +82,7 @@ check-synopsis-style check-derivation check-home-page + check-name check-source check-source-file-name check-source-unstable-tarball @@ -173,14 +175,20 @@ (define (check-name package) "Check whether PACKAGE's name matches our guidelines." (let ((name (package-name package))) - ;; Currently checks only whether the name is too short. - (if (and (<= (string-length name) 1) - (not (string=? name "r"))) ; common-sense exception - (list - (make-warning package - (G_ "name should be longer than a single character") - #:field 'name)) - '()))) + (cond + ;; Currently checks only whether the name is too short. + ((and (<= (string-length name) 1) + (not (string=? name "r"))) ; common-sense exception + (list + (make-warning package + (G_ "name should be longer than a single character") + #:field 'name))) + ((string-index name #\_) + (list + (make-warning package + (G_ "name should use hyphens instead of underscores") + #:field 'name))) + (else '())))) (define (properly-starts-sentence? s) (string-match "^[(\"'`[:upper:][:digit:]]" s)) diff --git a/guix/packages.scm b/guix/packages.scm index 3ae205b22a..ba19174646 100644 --- a/guix/packages.scm +++ b/guix/packages.scm @@ -807,7 +807,8 @@ specifies modules in scope when evaluating SNIPPET." "Return package ORIGINAL with PATCHES applied." (package (inherit original) (source (origin (inherit (package-source original)) - (patches patches))))) + (patches patches))) + (location (package-location original)))) (define (package-with-extra-patches original patches) "Return package ORIGINAL with all PATCHES appended to its list of patches." diff --git a/guix/profiles.scm b/guix/profiles.scm index 7a207589b0..2ec78b080a 100644 --- a/guix/profiles.scm +++ b/guix/profiles.scm @@ -4,7 +4,7 @@ ;;; Copyright © 2014, 2016 Alex Kost <alezost@gmail.com> ;;; Copyright © 2015 Mark H Weaver <mhw@netris.org> ;;; Copyright © 2015 Sou Bunnbu <iyzsong@gmail.com> -;;; Copyright © 2016, 2018, 2019 Ricardo Wurmus <rekado@elephly.net> +;;; Copyright © 2016, 2018, 2019, 2021 Ricardo Wurmus <rekado@elephly.net> ;;; Copyright © 2016 Chris Marusich <cmmarusich@gmail.com> ;;; Copyright © 2017 Huang Ying <huang.ying.caritas@gmail.com> ;;; Copyright © 2017 Maxim Cournoyer <maxim.cournoyer@gmail.com> @@ -1115,6 +1115,46 @@ MANIFEST. Single-file bundles are required by programs such as Git and Lynx." `((type . profile-hook) (hook . ca-certificate-bundle)))) +(define (emacs-subdirs manifest) + (define build + (with-imported-modules (source-module-closure + '((guix build profiles) + (guix build utils))) + #~(begin + (use-modules (guix build utils) + (guix build profiles) + (ice-9 ftw) ; scandir + (srfi srfi-1) ; append-map + (srfi srfi-26)) + + (let ((destdir (string-append #$output "/share/emacs/site-lisp")) + (subdirs + (append-map + (lambda (dir) + (filter + file-is-directory? + (map (cute string-append dir "/" <>) + (scandir dir (negate (cute member <> '("." ".."))))))) + (filter file-exists? + (map (cute string-append <> "/share/emacs/site-lisp") + '#$(manifest-inputs manifest)))))) + (mkdir-p destdir) + (with-directory-excursion destdir + (call-with-output-file "subdirs.el" + (lambda (port) + (write + `(normal-top-level-add-to-load-path + (list ,@subdirs)) + port) + (newline port) + #t))))))) + (gexp->derivation "emacs-subdirs" build + #:local-build? #t + #:substitutable? #f + #:properties + `((type . profile-hook) + (hook . emacs-subdirs)))) + (define (glib-schemas manifest) "Return a derivation that unions all schemas from manifest entries and creates the Glib 'gschemas.compiled' file." @@ -1625,6 +1665,7 @@ the entries in MANIFEST." fonts-dir-file ghc-package-cache-file ca-certificate-bundle + emacs-subdirs glib-schemas gtk-icon-themes gtk-im-modules diff --git a/guix/scripts/import.scm b/guix/scripts/import.scm index 98554ef79b..bbd9a3b190 100644 --- a/guix/scripts/import.scm +++ b/guix/scripts/import.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2012, 2013, 2014, 2020 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2012, 2013, 2014, 2020, 2021 Ludovic Courtès <ludo@gnu.org> ;;; Copyright © 2014 David Thompson <davet@gnu.org> ;;; Copyright © 2018 Kyle Meyer <kyle@kyleam.com> ;;; Copyright © 2019 Ricardo Wurmus <rekado@elephly.net> @@ -76,7 +76,7 @@ rather than \\n." ;;; Entry point. ;;; -(define importers '("gnu" "nix" "pypi" "cpan" "hackage" "stackage" "elpa" "gem" +(define importers '("gnu" "pypi" "cpan" "hackage" "stackage" "elpa" "gem" "go" "cran" "crate" "texlive" "json" "opam")) (define (resolve-importer name) diff --git a/guix/scripts/import/nix.scm b/guix/scripts/import/nix.scm deleted file mode 100644 index 45ca7e3fcf..0000000000 --- a/guix/scripts/import/nix.scm +++ /dev/null @@ -1,90 +0,0 @@ -;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2012, 2013, 2016 Ludovic Courtès <ludo@gnu.org> -;;; Copyright © 2014 David Thompson <davet@gnu.org> -;;; -;;; This file is part of GNU Guix. -;;; -;;; GNU Guix is free software; you can redistribute it and/or modify it -;;; under the terms of the GNU General Public License as published by -;;; the Free Software Foundation; either version 3 of the License, or (at -;;; your option) any later version. -;;; -;;; GNU Guix is distributed in the hope that it will be useful, but -;;; WITHOUT ANY WARRANTY; without even the implied warranty of -;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -;;; GNU General Public License for more details. -;;; -;;; You should have received a copy of the GNU General Public License -;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>. - -(define-module (guix scripts import nix) - #:use-module (guix ui) - #:use-module (guix utils) - #:use-module (guix scripts) - #:use-module (guix import snix) - #:use-module (guix scripts import) - #:use-module (srfi srfi-1) - #:use-module (srfi srfi-11) - #:use-module (srfi srfi-37) - #:use-module (ice-9 match) - #:export (guix-import-nix)) - - -;;; -;;; Command-line options. -;;; - -(define %default-options - '()) - -(define (show-help) - (display (G_ "Usage: guix import nix NIXPKGS ATTRIBUTE -Import and convert the Nix expression ATTRIBUTE of NIXPKGS.\n")) - (display (G_ " - -h, --help display this help and exit")) - (display (G_ " - -V, --version display version information and exit")) - (newline) - (show-bug-report-information)) - -(define %options - ;; Specification of the command-line options. - (cons* (option '(#\h "help") #f #f - (lambda args - (show-help) - (exit 0))) - (option '(#\V "version") #f #f - (lambda args - (show-version-and-exit "guix import nix"))) - %standard-import-options)) - - -;;; -;;; Entry point. -;;; - -(define (guix-import-nix . args) - (define (parse-options) - ;; Return the alist of option values. - (args-fold* args %options - (lambda (opt name arg result) - (leave (G_ "~A: unrecognized option~%") name)) - (lambda (arg result) - (alist-cons 'argument arg result)) - %default-options)) - - (let* ((opts (parse-options)) - (args (filter-map (match-lambda - (('argument . value) - value) - (_ #f)) - (reverse opts)))) - (match args - ((nixpkgs attribute) - (let-values (((expr loc) - (nixpkgs->guix-package nixpkgs attribute))) - (format #t ";; converted from ~a:~a~%~%" - (location-file loc) (location-line loc)) - expr)) - (x - (leave (G_ "wrong number of arguments~%")))))) diff --git a/guix/scripts/perform-download.scm b/guix/scripts/perform-download.scm index 8d409092ba..6889bcef79 100644 --- a/guix/scripts/perform-download.scm +++ b/guix/scripts/perform-download.scm @@ -54,7 +54,8 @@ actual output is different from that when we're doing a 'bmCheck' or (output* "out") (executable "executable") (mirrors "mirrors") - (content-addressed-mirrors "content-addressed-mirrors")) + (content-addressed-mirrors "content-addressed-mirrors") + (disarchive-mirrors "disarchive-mirrors")) (unless url (leave (G_ "~a: missing URL~%") (derivation-file-name drv))) @@ -79,6 +80,10 @@ actual output is different from that when we're doing a 'bmCheck' or (lambda (port) (eval (read port) %user-module))) '()) + #:disarchive-mirrors + (if disarchive-mirrors + (call-with-input-file disarchive-mirrors read) + '()) #:hashes `((,algo . ,hash)) ;; Since DRV's output hash is known, X.509 certificate diff --git a/guix/scripts/substitute.scm b/guix/scripts/substitute.scm index 48309f9b3a..8e4eae00b3 100755 --- a/guix/scripts/substitute.scm +++ b/guix/scripts/substitute.scm @@ -45,7 +45,7 @@ #:select (uri-abbreviation nar-uri-abbreviation (open-connection-for-uri . guix:open-connection-for-uri))) - #:autoload (gnutls) (error/invalid-session) + #:autoload (gnutls) (error/invalid-session error/again error/interrupted) #:use-module (guix progress) #:use-module ((guix build syscalls) #:select (set-thread-name)) @@ -417,7 +417,14 @@ server certificates." (if (or (and (eq? key 'system-error) (= EPIPE (system-error-errno `(,key ,@args)))) (and (eq? key 'gnutls-error) - (eq? (first args) error/invalid-session)) + (memq (first args) + (list error/invalid-session + + ;; XXX: These two are not properly handled in + ;; GnuTLS < 3.7.2, in + ;; 'write_to_session_record_port'; see + ;; <https://bugs.gnu.org/47867>. + error/again error/interrupted))) (memq key '(bad-response bad-header bad-header-component))) (proc (open-connection-for-uri/cached uri #:verify-certificate? #f diff --git a/guix/scripts/weather.scm b/guix/scripts/weather.scm index 5164fe0494..6d925d416c 100644 --- a/guix/scripts/weather.scm +++ b/guix/scripts/weather.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2017, 2018, 2019, 2020 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2017, 2018, 2019, 2020, 2021 Ludovic Courtès <ludo@gnu.org> ;;; Copyright © 2017 Ricardo Wurmus <rekado@elephly.net> ;;; Copyright © 2018 Kyle Meyer <kyle@kyleam.com> ;;; Copyright © 2020 Simon Tournier <zimon.toutoune@gmail.com> @@ -171,6 +171,16 @@ about the derivations queued, as is the case with Hydra." #f ;no derivation information (lset-intersection string=? queued items))) +(define (store-item-system store item) + "Return the system (a string such as \"aarch64-linux\")) ITEM targets, +or #f if it could not be determined." + (match (valid-derivers store item) + ((drv . _) + (and=> (false-if-exception (read-derivation-from-file drv)) + derivation-system)) + (() + #f))) + (define* (report-server-coverage server items #:key display-missing?) "Report the subset of ITEMS available as substitutes on SERVER. @@ -270,7 +280,22 @@ are queued~%") (when (and display-missing? (not (null? missing))) (newline) (format #t (G_ "Substitutes are missing for the following items:~%")) - (format #t "~{ ~a~%~}" missing)) + + ;; Display two columns: store items, and their system type. + (format #t "~:{ ~a ~a~%~}" + (zip (map (let ((width (max (- (current-terminal-columns) + 20) + 0))) + (lambda (item) + (if (> (string-length item) width) + item + (string-pad-right item width)))) + missing) + (with-store store + (map (lambda (item) + (or (store-item-system store item) + (G_ "unknown system"))) + missing))))) ;; Return the coverage ratio. (let ((total (length items))) diff --git a/guix/self.scm b/guix/self.scm index ec8b6c33cc..cdbb606a0b 100644 --- a/guix/self.scm +++ b/guix/self.scm @@ -878,7 +878,8 @@ itself." ("guix/store/schema.sql" ,(local-file "../guix/store/schema.sql"))) - #:extensions (list guile-gcrypt) + #:extensions (list guile-gcrypt + guile-json) ;for (guix swh) #:guile-for-build guile-for-build)) (define *extra-modules* diff --git a/guix/ssh.scm b/guix/ssh.scm index 457d1890f9..77a9732ce5 100644 --- a/guix/ssh.scm +++ b/guix/ssh.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2016, 2017, 2018, 2019, 2020 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2016, 2017, 2018, 2019, 2020, 2021, 2021 Ludovic Courtès <ludo@gnu.org> ;;; ;;; This file is part of GNU Guix. ;;; @@ -253,7 +253,22 @@ EXP never returns or calls 'primitive-exit' when it's done." (use-modules (ice-9 match) (rnrs io ports) (rnrs bytevectors)) - (let ((sock (socket AF_UNIX SOCK_STREAM 0)) + (define connect-to-daemon + ;; XXX: 'connect-to-daemon' used to be private and before that it + ;; didn't even exist, hence these shenanigans. + (let ((connect-to-daemon + (false-if-exception (module-ref (resolve-module '(guix store)) + 'connect-to-daemon)))) + (lambda (uri) + (if connect-to-daemon + (connect-to-daemon uri) + (let ((sock (socket AF_UNIX SOCK_STREAM 0))) + (connect sock AF_UNIX ,socket-name) + sock))))) + + ;; Use 'connect-to-daemon' to honor GUIX_DAEMON_SOCKET. + (let ((sock (connect-to-daemon (or (getenv "GUIX_DAEMON_SOCKET") + socket-name))) (stdin (current-input-port)) (stdout (current-output-port)) (select* (lambda (read write except) @@ -272,8 +287,6 @@ EXP never returns or calls 'primitive-exit' when it's done." (setvbuf stdin 'block 65536) (setvbuf sock 'block 65536) - (connect sock AF_UNIX ,socket-name) - (let loop () (match (select* (list stdin sock) '() '()) ((reads () ()) @@ -302,8 +315,13 @@ EXP never returns or calls 'primitive-exit' when it's done." "/var/guix/daemon-socket/socket")) "Connect to the remote build daemon listening on SOCKET-NAME over SESSION, an SSH session. Return a <store-connection> object." - (open-connection #:port (remote-daemon-channel session socket-name))) - + (guard (c ((store-connection-error? c) + ;; Raise a more focused error condition. + (raise (formatted-message + (G_ "failed to connect over SSH to daemon at '~a', socket ~a") + (session-get session 'host) + socket-name)))) + (open-connection #:port (remote-daemon-channel session socket-name)))) (define (store-import-channel session) "Return an output port to which archives to be exported to SESSION's store diff --git a/guix/status.scm b/guix/status.scm index 362ae2882c..1164c2a6e3 100644 --- a/guix/status.scm +++ b/guix/status.scm @@ -379,6 +379,8 @@ the current build phase." (G_ "building GHC package cache...")) ('ca-certificate-bundle (G_ "building CA certificate bundle...")) + ('emacs-subdirs + (G_ "listing Emacs sub-directories...")) ('glib-schemas (G_ "generating GLib schema cache...")) ('gtk-icon-themes @@ -552,12 +554,16 @@ substitutes being downloaded." (download-start download) #:transferred transferred)))))) (('substituter-succeeded item _ ...) - ;; If there are no jobs running, we already reported download completion - ;; so there's nothing left to do. - (unless (and (zero? (simultaneous-jobs status)) - (extended-build-trace-supported?)) - (format port (success (G_ "substitution of ~a complete")) item) - (newline port))) + (when (extended-build-trace-supported?) + ;; If there are no jobs running, we already reported download completion + ;; so there's nothing left to do. + (unless (zero? (simultaneous-jobs status)) + (format port (success (G_ "substitution of ~a complete")) item)) + + (when (and print-urls? (zero? (simultaneous-jobs status))) + ;; Leave a blank line after the "downloading ..." line and the + ;; progress bar (that's three lines in total). + (newline port)))) (('substituter-failed item _ ...) (format port (failure (G_ "substitution of ~a failed")) item) (newline port)) diff --git a/guix/store.scm b/guix/store.scm index 37ae6cfedd..9d706ae590 100644 --- a/guix/store.scm +++ b/guix/store.scm @@ -90,6 +90,7 @@ hash-algo build-mode + connect-to-daemon open-connection port->connection close-connection @@ -501,7 +502,10 @@ (define (connect-to-daemon uri) "Connect to the daemon at URI, a string that may be an actual URI or a file -name." +name, and return an input/output port. + +This is a low-level procedure that does not perform the initial handshake with +the daemon. Use 'open-connection' for that." (define (not-supported) (raise (condition (&store-connection-error (file uri) @@ -548,13 +552,16 @@ space on the file system so that the garbage collector can still operate, should the disk become full. When CPU-AFFINITY is true, it must be an integer corresponding to an OS-level CPU number to which the daemon's worker process for this connection will be pinned. Return a server object." + (define (handshake-error) + (raise (condition + (&store-connection-error (file (or port uri)) + (errno EPROTO)) + (&message (message "build daemon handshake failed"))))) + (guard (c ((nar-error? c) ;; One of the 'write-' or 'read-' calls below failed, but this is ;; really a connection error. - (raise (condition - (&store-connection-error (file (or port uri)) - (errno EPROTO)) - (&message (message "build daemon handshake failed")))))) + (handshake-error))) (let*-values (((port) (or port (connect-to-daemon uri))) ((output flush) @@ -562,32 +569,35 @@ for this connection will be pinned. Return a server object." (make-bytevector 8192)))) (write-int %worker-magic-1 port) (let ((r (read-int port))) - (and (= r %worker-magic-2) - (let ((v (read-int port))) - (and (= (protocol-major %protocol-version) - (protocol-major v)) - (begin - (write-int %protocol-version port) - (when (>= (protocol-minor v) 14) - (write-int (if cpu-affinity 1 0) port) - (when cpu-affinity - (write-int cpu-affinity port))) - (when (>= (protocol-minor v) 11) - (write-int (if reserve-space? 1 0) port)) - (letrec* ((built-in-builders - (delay (%built-in-builders conn))) - (conn - (%make-store-connection port - (protocol-major v) - (protocol-minor v) - output flush - (make-hash-table 100) - (make-hash-table 100) - vlist-null - built-in-builders))) - (let loop ((done? (process-stderr conn))) - (or done? (process-stderr conn))) - conn))))))))) + (unless (= r %worker-magic-2) + (handshake-error)) + + (let ((v (read-int port))) + (unless (= (protocol-major %protocol-version) + (protocol-major v)) + (handshake-error)) + + (write-int %protocol-version port) + (when (>= (protocol-minor v) 14) + (write-int (if cpu-affinity 1 0) port) + (when cpu-affinity + (write-int cpu-affinity port))) + (when (>= (protocol-minor v) 11) + (write-int (if reserve-space? 1 0) port)) + (letrec* ((built-in-builders + (delay (%built-in-builders conn))) + (conn + (%make-store-connection port + (protocol-major v) + (protocol-minor v) + output flush + (make-hash-table 100) + (make-hash-table 100) + vlist-null + built-in-builders))) + (let loop ((done? (process-stderr conn))) + (or done? (process-stderr conn))) + conn)))))) (define* (port->connection port #:key (version %protocol-version)) diff --git a/guix/swh.scm b/guix/swh.scm index f11b7ea2d5..3005323fd1 100644 --- a/guix/swh.scm +++ b/guix/swh.scm @@ -108,6 +108,7 @@ commit-id? + swh-download-directory swh-download)) ;;; Commentary: @@ -147,12 +148,20 @@ url (string-append url "/"))) -;; XXX: Work around a bug in Guile 3.0.2 where #:verify-certificate? would -;; be ignored (<https://bugs.gnu.org/40486>). -(define* (http-get* uri #:rest rest) - (apply http-request uri #:method 'GET rest)) -(define* (http-post* uri #:rest rest) - (apply http-request uri #:method 'POST rest)) +(cond-expand + (guile-3 + ;; XXX: Work around a bug in Guile 3.0.2 where #:verify-certificate? would + ;; be ignored (<https://bugs.gnu.org/40486>). + (define* (http-get* uri #:rest rest) + (apply http-request uri #:method 'GET rest)) + (define* (http-post* uri #:rest rest) + (apply http-request uri #:method 'POST rest))) + (else ;Guile 2.2 + ;; Guile 2.2 did not have #:verify-certificate? so ignore it. + (define* (http-get* uri #:key verify-certificate? streaming?) + (http-request uri #:method 'GET #:streaming? streaming?)) + (define* (http-post* uri #:key verify-certificate? streaming?) + (http-request uri #:method 'POST #:streaming? streaming?)))) (define %date-regexp ;; Match strings like "2014-11-17T22:09:38+01:00" or @@ -558,12 +567,6 @@ requested bundle cooking, waiting for completion...~%")) ;;; High-level interface. ;;; -(define (commit-id? reference) - "Return true if REFERENCE is likely a commit ID, false otherwise---e.g., if -it is a tag name. This is based on a simple heuristic so use with care!" - (and (= (string-length reference) 40) - (string-every char-set:hex-digit reference))) - (define (call-with-temporary-directory proc) ;FIXME: factorize "Call PROC with a name of a temporary directory; close the directory and delete it when leaving the dynamic extent of this call." @@ -577,6 +580,39 @@ delete it when leaving the dynamic extent of this call." (lambda () (false-if-exception (delete-file-recursively tmp-dir)))))) +(define* (swh-download-directory id output + #:key (log-port (current-error-port))) + "Download from Software Heritage the directory with the given ID, and +unpack it to OUTPUT. Return #t on success and #f on failure" + (call-with-temporary-directory + (lambda (directory) + (match (vault-fetch id 'directory #:log-port log-port) + (#f + (format log-port + "SWH: directory ~a could not be fetched from the vault~%" + id) + #f) + ((? port? input) + (let ((tar (open-pipe* OPEN_WRITE "tar" "-C" directory "-xzvf" "-"))) + (dump-port input tar) + (close-port input) + (let ((status (close-pipe tar))) + (unless (zero? status) + (error "tar extraction failure" status))) + + (match (scandir directory) + (("." ".." sub-directory) + (copy-recursively (string-append directory "/" sub-directory) + output + #:log (%make-void-port "w")) + #t)))))))) + +(define (commit-id? reference) + "Return true if REFERENCE is likely a commit ID, false otherwise---e.g., if +it is a tag name. This is based on a simple heuristic so use with care!" + (and (= (string-length reference) 40) + (string-every char-set:hex-digit reference))) + (define* (swh-download url reference output #:key (log-port (current-error-port))) "Download from Software Heritage a checkout of the Git tag or commit @@ -593,28 +629,7 @@ wait until it becomes available, which could take several minutes." (format log-port "SWH: found revision ~a with directory at '~a'~%" (revision-id revision) (swh-url (revision-directory-url revision))) - (call-with-temporary-directory - (lambda (directory) - (match (vault-fetch (revision-directory revision) 'directory - #:log-port log-port) - (#f - (format log-port - "SWH: directory ~a could not be fetched from the vault~%" - (revision-directory revision)) - #f) - ((? port? input) - (let ((tar (open-pipe* OPEN_WRITE "tar" "-C" directory "-xzvf" "-"))) - (dump-port input tar) - (close-port input) - (let ((status (close-pipe tar))) - (unless (zero? status) - (error "tar extraction failure" status))) - - (match (scandir directory) - (("." ".." sub-directory) - (copy-recursively (string-append directory "/" sub-directory) - output - #:log (%make-void-port "w")) - #t)))))))) + (swh-download-directory (revision-directory revision) output + #:log-port log-port)) (#f #f))) diff --git a/guix/ui.scm b/guix/ui.scm index 7fbd4c63a2..e2cf2f1f5e 100644 --- a/guix/ui.scm +++ b/guix/ui.scm @@ -376,12 +376,14 @@ ARGS is the list of arguments received by the 'throw' handler." (('system-error . rest) (let ((err (system-error-errno args))) (report-error (G_ "failed to load '~a': ~a~%") file (strerror err)))) - (('read-error "scm_i_lreadparen" message _ ...) + (('read-error _ message args ...) ;; Guile's missing-paren messages are obscure so we make them more ;; intelligible here. - (if (string-suffix? "end of file" message) - (let ((location (string-drop-right message - (string-length "end of file")))) + (if (or (string-suffix? "end of file" message) ;Guile < 3.0.6 + (and (string-contains message "unexpected end of input") + (member '(#\)) args))) + (let ((location (string-take message + (+ 2 (string-contains message ": "))))) (format (current-error-port) (G_ "~amissing closing parenthesis~%") location)) (apply throw args))) @@ -490,12 +492,11 @@ part." (lambda _ (setlocale LC_ALL "")) (lambda args - (display-hint (G_ "Consider installing the @code{glibc-utf8-locales} or -@code{glibc-locales} package and defining @code{GUIX_LOCPATH}, along these -lines: + (display-hint (G_ "Consider installing the @code{glibc-locales} package +and defining @code{GUIX_LOCPATH}, along these lines: @example -guix install glibc-utf8-locales +guix install glibc-locales export GUIX_LOCPATH=\"$HOME/.guix-profile/lib/locale\" @end example |