diff options
Diffstat (limited to 'guix')
| -rw-r--r-- | guix/avahi.scm | 14 | ||||
| -rw-r--r-- | guix/build/store-copy.scm | 133 | ||||
| -rw-r--r-- | guix/gexp.scm | 15 | ||||
| -rw-r--r-- | guix/import/cran.scm | 7 | ||||
| -rw-r--r-- | guix/import/elpa.scm | 189 | ||||
| -rw-r--r-- | guix/inferior.scm | 11 | ||||
| -rw-r--r-- | guix/monads.scm | 15 | ||||
| -rw-r--r-- | guix/nar.scm | 8 | ||||
| -rw-r--r-- | guix/progress.scm | 68 | ||||
| -rw-r--r-- | guix/scripts/archive.scm | 2 | ||||
| -rw-r--r-- | guix/scripts/challenge.scm | 1 | ||||
| -rw-r--r-- | guix/scripts/discover.scm | 58 | ||||
| -rw-r--r-- | guix/scripts/offload.scm | 12 | ||||
| -rw-r--r-- | guix/scripts/pack.scm | 260 | ||||
| -rw-r--r-- | guix/scripts/publish.scm | 26 | ||||
| -rwxr-xr-x | guix/scripts/substitute.scm | 70 | ||||
| -rw-r--r-- | guix/scripts/system.scm | 66 | ||||
| -rw-r--r-- | guix/serialization.scm | 42 | ||||
| -rw-r--r-- | guix/status.scm | 4 | ||||
| -rw-r--r-- | guix/store/database.scm | 58 | ||||
| -rw-r--r-- | guix/store/deduplication.scm | 199 |
21 files changed, 769 insertions, 489 deletions
diff --git a/guix/avahi.scm b/guix/avahi.scm index aa90a5cdd4..132e42f268 100644 --- a/guix/avahi.scm +++ b/guix/avahi.scm @@ -89,13 +89,19 @@ when STOP-LOOP? procedure returns true." (close-port socket) ip)) +(define never + ;; Never true. + (const #f)) + (define* (avahi-browse-service-thread proc #:key types (ignore-local? #t) (family AF_INET) - (stop-loop? (const #f)) - (timeout 100)) + (stop-loop? never) + (timeout (if (eq? stop-loop? never) + #f + 100))) "Browse services which type is part of the TYPES list, using Avahi. The search is restricted to services with the given FAMILY. Each time a service is found or removed, PROC is called and passed as argument the corresponding @@ -167,4 +173,6 @@ when STOP-LOOP? procedure returns true." client-callback))) (and (client? client) (while (not (stop-loop?)) - (iterate-simple-poll poll timeout))))) + (if timeout + (iterate-simple-poll poll timeout) + (iterate-simple-poll poll)))))) diff --git a/guix/build/store-copy.scm b/guix/build/store-copy.scm index ad551bca98..01e1f41870 100644 --- a/guix/build/store-copy.scm +++ b/guix/build/store-copy.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2013, 2014, 2017, 2018 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2013, 2014, 2017, 2018, 2020 Ludovic Courtès <ludo@gnu.org> ;;; ;;; This file is part of GNU Guix. ;;; @@ -17,9 +17,10 @@ ;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>. (define-module (guix build store-copy) - #:use-module (guix build utils) + #:use-module ((guix build utils) #:hide (copy-recursively)) #:use-module (guix sets) #:use-module (guix progress) + #:autoload (guix store deduplication) (copy-file/deduplicate) #:use-module (srfi srfi-1) #:use-module (srfi srfi-9) #:use-module (srfi srfi-26) @@ -37,6 +38,7 @@ file-size closure-size + copy-store-item populate-store)) ;;; Commentary: @@ -169,32 +171,104 @@ REFERENCE-GRAPHS, a list of reference-graph files." (reduce + 0 (map file-size items))) -(define (reset-permissions file) - "Reset the permissions on FILE and its sub-directories so that they are all -read-only." - ;; XXX: This procedure exists just to work around the inability of - ;; 'copy-recursively' to preserve permissions. - (file-system-fold (const #t) ;enter? - (lambda (file stat _) ;leaf - (unless (eq? 'symlink (stat:type stat)) - (chmod file - (if (zero? (logand (stat:mode stat) - #o100)) - #o444 - #o555)))) - (const #t) ;down - (lambda (directory stat _) ;up - (chmod directory #o555)) - (const #f) ;skip - (const #f) ;error +;; TODO: Remove when the one in (guix build utils) has #:keep-permissions?, +;; the fix for <https://bugs.gnu.org/44741>, and when #:keep-mtime? works for +;; symlinks. +(define* (copy-recursively source destination + #:key + (log (current-output-port)) + (follow-symlinks? #f) + (copy-file copy-file) + keep-mtime? keep-permissions?) + "Copy SOURCE directory to DESTINATION. Follow symlinks if FOLLOW-SYMLINKS? +is true; otherwise, just preserve them. Call COPY-FILE to copy regular files. +When KEEP-MTIME? is true, keep the modification time of the files in SOURCE on +those of DESTINATION. When KEEP-PERMISSIONS? is true, preserve file +permissions. Write verbose output to the LOG port." + (define AT_SYMLINK_NOFOLLOW + ;; Guile 2.0 did not define this constant, hence this hack. + (let ((variable (module-variable the-root-module 'AT_SYMLINK_NOFOLLOW))) + (if variable + (variable-ref variable) + 256))) ;for GNU/Linux + + (define (set-file-time file stat) + (utime file + (stat:atime stat) + (stat:mtime stat) + (stat:atimensec stat) + (stat:mtimensec stat) + AT_SYMLINK_NOFOLLOW)) + + (define strip-source + (let ((len (string-length source))) + (lambda (file) + (substring file len)))) + + (file-system-fold (const #t) ; enter? + (lambda (file stat result) ; leaf + (let ((dest (string-append destination + (strip-source file)))) + (format log "`~a' -> `~a'~%" file dest) + (case (stat:type stat) + ((symlink) + (let ((target (readlink file))) + (symlink target dest))) + (else + (copy-file file dest) + (when keep-permissions? + (chmod dest (stat:perms stat))))) + (when keep-mtime? + (set-file-time dest stat)))) + (lambda (dir stat result) ; down + (let ((target (string-append destination + (strip-source dir)))) + (mkdir-p target))) + (lambda (dir stat result) ; up + (let ((target (string-append destination + (strip-source dir)))) + (when keep-mtime? + (set-file-time target stat)) + (when keep-permissions? + (chmod target (stat:perms stat))))) + (const #t) ; skip + (lambda (file stat errno result) + (format (current-error-port) "i/o error: ~a: ~a~%" + file (strerror errno)) + #f) #t - file - lstat)) + source + + (if follow-symlinks? + stat + lstat))) + +(define* (copy-store-item item target + #:key + (deduplicate? #t) + (log-port (%make-void-port "w"))) + "Copy ITEM, a store item, to the store under TARGET, the target root +directory. When DEDUPLICATE? is true, deduplicate it within TARGET." + (define store + (string-append target (%store-directory))) + + (copy-recursively item (string-append target item) + #:keep-mtime? #t + #:keep-permissions? #t + #:copy-file + (if deduplicate? + (cut copy-file/deduplicate <> <> #:store store) + copy-file) + #:log log-port)) (define* (populate-store reference-graphs target - #:key (log-port (current-error-port))) + #:key + (deduplicate? #t) + (log-port (current-error-port))) "Populate the store under directory TARGET with the items specified in -REFERENCE-GRAPHS, a list of reference-graph files." +REFERENCE-GRAPHS, a list of reference-graph files. Items copied to TARGET +maintain timestamps and permissions. When DEDUPLICATE? is true, deduplicate +regular files as they are copied to TARGET." (define store (string-append target (%store-directory))) @@ -218,15 +292,8 @@ REFERENCE-GRAPHS, a list of reference-graph files." (call-with-progress-reporter progress (lambda (report) (for-each (lambda (thing) - (copy-recursively thing - (string-append target thing) - #:keep-mtime? #t - #:log (%make-void-port "w")) - - ;; XXX: Since 'copy-recursively' doesn't allow us to - ;; preserve permissions, we have to traverse TARGET to - ;; make sure everything is read-only. - (reset-permissions (string-append target thing)) + (copy-store-item thing target + #:deduplicate? deduplicate?) (report)) things))))) diff --git a/guix/gexp.scm b/guix/gexp.scm index 051831238e..764c89a187 100644 --- a/guix/gexp.scm +++ b/guix/gexp.scm @@ -1317,18 +1317,7 @@ and in the current monad setting (system type, etc.)" reference->sexp (gexp-references exp)))) (return (apply (gexp-proc exp) args)))) -(define-syntax-rule (define-syntax-parameter-once name proc) - ;; Like 'define-syntax-parameter' but ensure the top-level binding for NAME - ;; does not get redefined. This works around a race condition in a - ;; multi-threaded context with Guile <= 2.2.4: <https://bugs.gnu.org/27476>. - (eval-when (load eval expand compile) - (define name - (if (module-locally-bound? (current-module) 'name) - (module-ref (current-module) 'name) - (make-syntax-transformer 'name 'syntax-parameter - (list proc)))))) - -(define-syntax-parameter-once current-imported-modules +(define-syntax-parameter current-imported-modules ;; Current list of imported modules. (identifier-syntax '())) @@ -1339,7 +1328,7 @@ environment." (identifier-syntax modules))) body ...)) -(define-syntax-parameter-once current-imported-extensions +(define-syntax-parameter current-imported-extensions ;; Current list of extensions. (identifier-syntax '())) diff --git a/guix/import/cran.scm b/guix/import/cran.scm index 1d25a5125e..9d38be7a1e 100644 --- a/guix/import/cran.scm +++ b/guix/import/cran.scm @@ -141,6 +141,7 @@ package definition." `((,type (,'quasiquote ,(format-inputs package-inputs))))))) (define %cran-url "https://cran.r-project.org/web/packages/") +(define %cran-canonical-url "https://cran.r-project.org/package=") (define %bioconductor-url "https://bioconductor.org/packages/") ;; The latest Bioconductor release is 3.12. Bioconductor packages should be @@ -441,6 +442,10 @@ from the alist META, which was derived from the R package's DESCRIPTION file." ((bioconductor) %bioconductor-url) ((git) #f) ((hg) #f))) + (canonical-url-base (case repository + ((cran) %cran-canonical-url) + ((bioconductor) %bioconductor-url) + ((git) #f))) (uri-helper (case repository ((cran) cran-uri) ((bioconductor) bioconductor-uri) @@ -456,7 +461,7 @@ from the alist META, which was derived from the R package's DESCRIPTION file." ((hg) (assoc-ref meta 'hg)) (else (match (listify meta "URL") ((url rest ...) url) - (_ (string-append base-url name)))))) + (_ (string-append canonical-url-base name)))))) (source-url (case repository ((git) (assoc-ref meta 'git)) ((hg) (assoc-ref meta 'hg)) diff --git a/guix/import/elpa.scm b/guix/import/elpa.scm index c4e8e84aba..8922e57840 100644 --- a/guix/import/elpa.scm +++ b/guix/import/elpa.scm @@ -22,6 +22,7 @@ (define-module (guix import elpa) #:use-module (ice-9 match) #:use-module (ice-9 rdelim) + #:use-module (ice-9 regex) #:use-module (web uri) #:use-module (srfi srfi-1) #:use-module (srfi srfi-9) @@ -31,6 +32,8 @@ #:use-module ((guix download) #:select (download-to-store)) #:use-module (guix import utils) #:use-module (guix http-client) + #:use-module (guix git) + #:use-module ((guix serialization) #:select (write-file)) #:use-module (guix store) #:use-module (guix ui) #:use-module (gcrypt hash) @@ -196,10 +199,143 @@ include VERSION." url))) (_ #f)))) -(define* (elpa-package->sexp pkg #:optional license) +(define* (download-git-repository url ref) + "Fetch the given REF from the Git repository at URL." + (with-store store + (latest-repository-commit store url #:ref ref))) + +(define (package-name->melpa-recipe package-name) + "Fetch the MELPA recipe for PACKAGE-NAME, represented as an alist from +keywords to values." + (define recipe-url + (string-append "https://raw.githubusercontent.com/melpa/melpa/master/recipes/" + package-name)) + + (define (data->recipe data) + (match data + (() '()) + ((key value . tail) + (cons (cons key value) (data->recipe tail))))) + + (let* ((port (http-fetch/cached (string->uri recipe-url) + #:ttl (* 6 3600))) + (data (read port))) + (close-port port) + (data->recipe (cons ':name data)))) + +;; XXX adapted from (guix scripts hash) +(define (file-hash file select? recursive?) + ;; Compute the hash of FILE. + (if recursive? + (let-values (((port get-hash) (open-sha256-port))) + (write-file file port #:select? select?) + (force-output port) + (get-hash)) + (call-with-input-file file port-sha256))) + +;; XXX taken from (guix scripts hash) +(define (vcs-file? file stat) + (case (stat:type stat) + ((directory) + (member (basename file) '(".bzr" ".git" ".hg" ".svn" "CVS"))) + ((regular) + ;; Git sub-modules have a '.git' file that is a regular text file. + (string=? (basename file) ".git")) + (else + #f))) + +(define (git-repository->origin recipe url) + "Fetch origin details from the Git repository at URL for the provided MELPA +RECIPE." + (define ref + (cond + ((assoc-ref recipe #:branch) + => (lambda (branch) (cons 'branch branch))) + ((assoc-ref recipe #:commit) + => (lambda (commit) (cons 'commit commit))) + (else + '(branch . "master")))) + + (let-values (((directory commit) (download-git-repository url ref))) + `(origin + (method git-fetch) + (uri (git-reference + (url ,url) + (commit ,commit))) + (sha256 + (base32 + ,(bytevector->nix-base32-string + (file-hash directory (negate vcs-file?) #t))))))) + +(define* (melpa-recipe->origin recipe) + "Fetch origin details from the MELPA recipe and associated repository for +the package named PACKAGE-NAME." + (define (github-repo->url repo) + (string-append "https://github.com/" repo ".git")) + (define (gitlab-repo->url repo) + (string-append "https://gitlab.com/" repo ".git")) + + (match (assq-ref recipe ':fetcher) + ('github (git-repository->origin recipe (github-repo->url (assq-ref recipe ':repo)))) + ('gitlab (git-repository->origin recipe (gitlab-repo->url (assq-ref recipe ':repo)))) + ('git (git-repository->origin recipe (assq-ref recipe ':url))) + (#f #f) ; if we're not using melpa then this stops us printing a warning + (_ (warning (G_ "Unsupported MELPA fetcher: ~a, falling back to unstable MELPA source.~%") + (assq-ref recipe ':fetcher)) + #f))) + +(define default-files-spec + ;; This contains more than just the things contained in %default-include and + ;; %default-exclude, presumably because this includes source files (*.in, + ;; *.texi, etc.) which have already been processed for releases. + ;; + ;; Taken from: + ;; https://github.com/melpa/melpa/blob/e8dc709d0ab2b4a68c59315f42858bcb86095f11/package-build/package-build.el#L580-L585 + '("*.el" "*.el.in" "dir" + "*.info" "*.texi" "*.texinfo" + "doc/dir" "doc/*.info" "doc/*.texi" "doc/*.texinfo" + (:exclude ".dir-locals.el" "test.el" "tests.el" "*-test.el" "*-tests.el"))) + +(define* (melpa-recipe->maybe-arguments melpa-recipe) + "Extract arguments for the build system from MELPA-RECIPE." + (define (glob->regexp glob) + (string-append + "^" + (regexp-substitute/global #f "\\*\\*?" glob + 'pre + (lambda (m) + (if (string= (match:substring m 0) "**") + ".*" + "[^/]+")) + 'post) + "$")) + + (let ((files (assq-ref melpa-recipe ':files))) + (if files + (let* ((with-default (apply append (map (lambda (entry) + (if (eq? ':defaults entry) + default-files-spec + (list entry))) + files))) + (inclusions (remove pair? with-default)) + (exclusions (apply append (map (match-lambda + ((':exclude . values) + values) + (_ '())) + with-default)))) + `((arguments '(#:include ',(map glob->regexp inclusions) + #:exclude ',(map glob->regexp exclusions))))) + '()))) + +(define* (elpa-package->sexp pkg #:optional license repo) "Return the `package' S-expression for the Emacs package PKG, a record of type '<elpa-package>'." + (define melpa-recipe + (if (eq? repo 'melpa) + (package-name->melpa-recipe (elpa-package-name pkg)) + #f)) + (define name (elpa-package-name pkg)) (define version (elpa-package-version pkg)) @@ -224,27 +360,34 @@ type '<elpa-package>'." (list (list input-type (list 'quasiquote inputs)))))) - (let ((tarball (with-store store - (download-to-store store source-url)))) - (values - `(package - (name ,(elpa-name->package-name name)) - (version ,version) - (source (origin - (method url-fetch) - (uri (string-append ,@(factorize-uri source-url version))) - (sha256 - (base32 - ,(if tarball - (bytevector->nix-base32-string (file-sha256 tarball)) - "failed to download package"))))) - (build-system emacs-build-system) - ,@(maybe-inputs 'propagated-inputs dependencies) - (home-page ,(elpa-package-home-page pkg)) - (synopsis ,(elpa-package-synopsis pkg)) - (description ,(elpa-package-description pkg)) - (license ,license)) - dependencies-names))) + (define melpa-source + (melpa-recipe->origin melpa-recipe)) + + (values + `(package + (name ,(elpa-name->package-name name)) + (version ,version) + (source ,(or melpa-source + (let ((tarball (with-store store + (download-to-store store source-url)))) + `(origin + (method url-fetch) + (uri (string-append ,@(factorize-uri source-url version))) + (sha256 + (base32 + ,(if tarball + (bytevector->nix-base32-string (file-sha256 tarball)) + "failed to download package"))))))) + (build-system emacs-build-system) + ,@(maybe-inputs 'propagated-inputs dependencies) + ,@(if melpa-source + (melpa-recipe->maybe-arguments melpa-recipe) + '()) + (home-page ,(elpa-package-home-page pkg)) + (synopsis ,(elpa-package-synopsis pkg)) + (description ,(elpa-package-description pkg)) + (license ,license)) + dependencies-names)) (define* (elpa->guix-package name #:key (repo 'gnu) version) "Fetch the package NAME from REPO and produce a Guix package S-expression." @@ -254,7 +397,7 @@ type '<elpa-package>'." ;; ELPA is known to contain only GPLv3+ code. Other repos may contain ;; code under other license but there's no license metadata. (let ((license (and (memq repo '(gnu gnu/http)) 'license:gpl3+))) - (elpa-package->sexp package license))))) + (elpa-package->sexp package license repo))))) ;;; diff --git a/guix/inferior.scm b/guix/inferior.scm index 77820872b3..2fe91beaab 100644 --- a/guix/inferior.scm +++ b/guix/inferior.scm @@ -469,22 +469,13 @@ is similar to the sexp returned by 'package-provenance' for regular packages." "Proxy communication between CLIENT and BACKEND until CLIENT closes the connection, at which point CLIENT is closed (both CLIENT and BACKEND must be input/output ports.)" - (define (select* read write except) - ;; This is a workaround for <https://bugs.gnu.org/30365> in Guile < 2.2.4: - ;; since 'select' sometimes returns non-empty sets for no good reason, - ;; call 'select' a second time with a zero timeout to filter out incorrect - ;; replies. - (match (select read write except) - ((read write except) - (select read write except 0)))) - ;; Use buffered ports so that 'get-bytevector-some' returns up to the ;; whole buffer like read(2) would--see <https://bugs.gnu.org/30066>. (setvbuf client 'block 65536) (setvbuf backend 'block 65536) (let loop () - (match (select* (list client backend) '() '()) + (match (select (list client backend) '() '()) ((reads () ()) (when (memq client reads) (match (get-bytevector-some client) diff --git a/guix/monads.scm b/guix/monads.scm index 6924471345..6ae616aca9 100644 --- a/guix/monads.scm +++ b/guix/monads.scm @@ -274,23 +274,12 @@ more optimizations." (_ #'generic-name)))))))))) -(define-syntax-rule (define-syntax-parameter-once name proc) - ;; Like 'define-syntax-parameter' but ensure the top-level binding for NAME - ;; does not get redefined. This works around a race condition in a - ;; multi-threaded context with Guile <= 2.2.4: <https://bugs.gnu.org/27476>. - (eval-when (load eval expand compile) - (define name - (if (module-locally-bound? (current-module) 'name) - (module-ref (current-module) 'name) - (make-syntax-transformer 'name 'syntax-parameter - (list proc)))))) - -(define-syntax-parameter-once >>= +(define-syntax-parameter >>= ;; The name 'bind' is already taken, so we choose this (obscure) symbol. (lambda (s) (syntax-violation '>>= ">>= (bind) used outside of 'with-monad'" s))) -(define-syntax-parameter-once return +(define-syntax-parameter return (lambda (s) (syntax-violation 'return "return used outside of 'with-monad'" s))) diff --git a/guix/nar.scm b/guix/nar.scm index a23af2e5de..a817b56007 100644 --- a/guix/nar.scm +++ b/guix/nar.scm @@ -27,6 +27,7 @@ ;; (guix store) since this is "daemon-side" code. #:use-module (guix store) #:use-module (guix store database) + #:use-module ((guix store deduplication) #:select (dump-file/deduplicate)) #:use-module ((guix build store-copy) #:select (store-info)) #:use-module (guix i18n) @@ -114,8 +115,8 @@ held." ;; Install the new TARGET. (rename-file source target) - ;; Register TARGET. As a side effect, it resets the timestamps of all - ;; its files, recursively, and runs a deduplication pass. + ;; Register TARGET. The 'restore-file' call took care of + ;; deduplication, timestamps, and permissions. (register-items db (list (store-info target deriver references)))) @@ -210,7 +211,8 @@ s-expression")) (let-values (((port get-hash) (open-sha256-input-port port))) (with-temporary-store-file temp - (restore-file port temp) + (restore-file port temp + #:dump-file dump-file/deduplicate) (let ((magic (read-int port))) (unless (= magic %export-magic) diff --git a/guix/progress.scm b/guix/progress.scm index cd80ae620a..334bd40547 100644 --- a/guix/progress.scm +++ b/guix/progress.scm @@ -184,44 +184,54 @@ move the cursor to the beginning of the line." (define* (display-download-progress file size #:key + (tty? #t) start-time (transferred 0) (log-port (current-error-port))) "Write the progress report to LOG-PORT. Use START-TIME (a SRFI-19 time object) and TRANSFERRED (a total number of bytes) to determine the -throughput." +throughput. When TTY? is false, assume LOG-PORT is not a tty and do not emit +ANSI escape codes." (define elapsed (duration->seconds (time-difference (current-time (time-type start-time)) start-time))) - (if (and (number? size) (not (zero? size))) - (let* ((% (* 100.0 (/ transferred size))) - (throughput (/ transferred elapsed)) - (left (format #f " ~a ~a" file - (byte-count->string size))) - (right (format #f "~a/s ~a ~a~6,1f%" - (byte-count->string throughput) - (seconds->string elapsed) - (progress-bar %) %))) - (erase-current-line log-port) - (display (string-pad-middle left right - (current-terminal-columns)) - log-port) - (force-output log-port)) - ;; If we don't know the total size, the last transfer will have a 0B - ;; size. Don't display it. - (unless (zero? transferred) - (let* ((throughput (/ transferred elapsed)) - (left (format #f " ~a" file)) - (right (format #f "~a/s ~a | ~a transferred" - (byte-count->string throughput) - (seconds->string elapsed) - (byte-count->string transferred)))) - (erase-current-line log-port) - (display (string-pad-middle left right - (current-terminal-columns)) - log-port) - (force-output log-port))))) + (cond ((and (not tty?) + size (not (zero? size)) + transferred) + ;; Display a dot for at most every 10%. + (when (zero? (modulo (round (* 100. (/ transferred size))) 10)) + (display "." log-port) + (force-output log-port))) + ((and (number? size) (not (zero? size))) + (let* ((% (* 100.0 (/ transferred size))) + (throughput (/ transferred elapsed)) + (left (format #f " ~a ~a" file + (byte-count->string size))) + (right (format #f "~a/s ~a ~a~6,1f%" + (byte-count->string throughput) + (seconds->string elapsed) + (progress-bar %) %))) + (erase-current-line log-port) + (display (string-pad-middle left right + (current-terminal-columns)) + log-port) + (force-output log-port))) + (else + ;; If we don't know the total size, the last transfer will have a 0B + ;; size. Don't display it. + (unless (zero? transferred) + (let* ((throughput (/ transferred elapsed)) + (left (format #f " ~a" file)) + (right (format #f "~a/s ~a | ~a transferred" + (byte-count->string throughput) + (seconds->string elapsed) + (byte-count->string transferred)))) + (erase-current-line log-port) + (display (string-pad-middle left right + (current-terminal-columns)) + log-port) + (force-output log-port)))))) (define %progress-interval ;; Default interval between subsequent outputs for rate-limited displays. diff --git a/guix/scripts/archive.scm b/guix/scripts/archive.scm index c04baf9784..1f73fff711 100644 --- a/guix/scripts/archive.scm +++ b/guix/scripts/archive.scm @@ -347,6 +347,8 @@ output port." (match type ('directory (format #t "D ~a~%" file)) + ('directory-complete + #t) ('symlink (format #t "S ~a -> ~a~%" file content)) ((or 'regular 'executable) diff --git a/guix/scripts/challenge.scm b/guix/scripts/challenge.scm index 39bd2c1c0f..d0a456ac1d 100644 --- a/guix/scripts/challenge.scm +++ b/guix/scripts/challenge.scm @@ -210,6 +210,7 @@ taken since we do not import the archives." (cons `(,file ,type ,(port-sha256* port size)) result)))) ('directory result) + ('directory-complete result) ('symlink (cons `(,file ,type ,contents) result)))) '() diff --git a/guix/scripts/discover.scm b/guix/scripts/discover.scm index 007db0d49d..6aade81ed1 100644 --- a/guix/scripts/discover.scm +++ b/guix/scripts/discover.scm @@ -21,6 +21,7 @@ #:use-module (guix config) #:use-module (guix scripts) #:use-module (guix ui) + #:use-module (guix utils) #:use-module (guix build syscalls) #:use-module (guix build utils) #:use-module (guix scripts publish) @@ -78,47 +79,27 @@ CACHE-DIRECTORY." (define* (write-publish-file #:key (file (%publish-file))) "Dump the content of %PUBLISH-SERVICES hash table into FILE. Use a write lock on FILE to synchronize with any potential readers." - (with-file-lock file - (call-with-output-file file - (lambda (port) - (hash-for-each - (lambda (name service) - (format port "http://~a:~a~%" - (avahi-service-address service) - (avahi-service-port service))) - %publish-services))) - (chmod file #o644))) - -(define (call-with-read-file-lock file thunk) - "Call THUNK with a read lock on FILE." - (let ((port #f)) - (dynamic-wind - (lambda () - (set! port - (let ((port (open-file file "r0"))) - (fcntl-flock port 'read-lock) - port))) - thunk - (lambda () - (when port - (unlock-file port)))))) - -(define-syntax-rule (with-read-file-lock file exp ...) - "Wait to acquire a read lock on FILE and evaluate EXP in that context." - (call-with-read-file-lock file (lambda () exp ...))) + (with-atomic-file-output file + (lambda (port) + (hash-for-each + (lambda (name service) + (format port "http://~a:~a~%" + (avahi-service-address service) + (avahi-service-port service))) + %publish-services))) + (chmod file #o644)) (define* (read-substitute-urls #:key (file (%publish-file))) "Read substitute urls list from FILE and return it. Use a read lock on FILE to synchronize with the writer." (if (file-exists? file) - (with-read-file-lock file - (call-with-input-file file - (lambda (port) - (let loop ((url (read-line port)) - (urls '())) - (if (eof-object? url) - urls - (loop (read-line port) (cons url urls))))))) + (call-with-input-file file + (lambda (port) + (let loop ((url (read-line port)) + (urls '())) + (if (eof-object? url) + urls + (loop (read-line port) (cons url urls)))))) '())) @@ -156,9 +137,6 @@ to synchronize with the writer." (publish-file (publish-file cache))) (parameterize ((%publish-file publish-file)) (mkdir-p (dirname publish-file)) + (false-if-exception (delete-file publish-file)) (avahi-browse-service-thread service-proc #:types %services))))) - -;;; Local Variables: -;;; eval: (put 'with-read-file-lock 'scheme-indent-function 1) -;;; End: diff --git a/guix/scripts/offload.scm b/guix/scripts/offload.scm index 6366556647..58ee53e85c 100644 --- a/guix/scripts/offload.scm +++ b/guix/scripts/offload.scm @@ -2,6 +2,7 @@ ;;; Copyright © 2014, 2015, 2016, 2017, 2018, 2019, 2020 Ludovic Courtès <ludo@gnu.org> ;;; Copyright © 2017 Ricardo Wurmus <rekado@elephly.net> ;;; Copyright © 2020 Maxim Cournoyer <maxim.cournoyer@gmail.com> +;;; Copyright © 2020 Julien Lepiller <julien@lepiller.eu> ;;; ;;; This file is part of GNU Guix. ;;; @@ -65,6 +66,7 @@ build-machine-overload-threshold build-machine-systems build-machine-features + build-machine-location build-requirements build-requirements? @@ -112,11 +114,17 @@ (speed build-machine-speed ; inexact real (default 1.0)) (features build-machine-features ; list of strings - (default '()))) + (default '())) + (location build-machine-location + (default (and=> (current-source-location) + source-properties->location)) + (innate))) ;;; Deprecated. (define (build-machine-system machine) - (warning (G_ "The 'system' field is deprecated, \ + (warning + (build-machine-location machine) + (G_ "The 'system' field is deprecated, \ please use 'systems' instead.~%")) (%build-machine-system machine)) diff --git a/guix/scripts/pack.scm b/guix/scripts/pack.scm index ba9a6dc1b2..8ecdcb823f 100644 --- a/guix/scripts/pack.scm +++ b/guix/scripts/pack.scm @@ -167,8 +167,6 @@ dependencies are registered." (let ((items (append-map read-closure '#$labels))) (with-database db-file db (register-items db items - #:deduplicate? #f - #:reset-timestamps? #f #:registration-time %epoch))))))) (computed-file "store-database" build @@ -204,12 +202,19 @@ added to the pack." #+(file-append glibc-utf8-locales "/lib/locale")) (setlocale LC_ALL "en_US.utf8")))) + (define (import-module? module) + ;; Since we don't use deduplication support in 'populate-store', don't + ;; import (guix store deduplication) and its dependencies, which includes + ;; Guile-Gcrypt. That way we can run tests with '--bootstrap'. + (and (not-config? module) + (not (equal? '(guix store deduplication) module)))) + (define build (with-imported-modules (source-module-closure `((guix build utils) (guix build union) (gnu build install)) - #:select? not-config?) + #:select? import-module?) #~(begin (use-modules (guix build utils) ((guix build union) #:select (relative-file-name)) @@ -383,138 +388,139 @@ added to the pack." `(("/bin" -> "bin") ,@symlinks))) (define build - (with-imported-modules (source-module-closure - '((guix build utils) - (guix build store-copy) - (guix build union) - (gnu build install)) - #:select? not-config?) - #~(begin - (use-modules (guix build utils) - (guix build store-copy) - ((guix build union) #:select (relative-file-name)) - (gnu build install) - (srfi srfi-1) - (srfi srfi-26) - (ice-9 match)) + (with-extensions (list guile-gcrypt) + (with-imported-modules (source-module-closure + '((guix build utils) + (guix build store-copy) + (guix build union) + (gnu build install)) + #:select? not-config?) + #~(begin + (use-modules (guix build utils) + (guix build store-copy) + ((guix build union) #:select (relative-file-name)) + (gnu build install) + (srfi srfi-1) + (srfi srfi-26) + (ice-9 match)) - (define database #+database) - (define entry-point #$entry-point) + (define database #+database) + (define entry-point #$entry-point) - (define (mksquashfs args) - (apply invoke "mksquashfs" - `(,@args + (define (mksquashfs args) + (apply invoke "mksquashfs" + `(,@args - ;; Do not create a "recovery file" when appending to the - ;; file system since it's useless in this case. - "-no-recovery" + ;; Do not create a "recovery file" when appending to the + ;; file system since it's useless in this case. + "-no-recovery" - ;; Do not attempt to store extended attributes. - ;; See <https://bugs.gnu.org/40043>. - "-no-xattrs" + ;; Do not attempt to store extended attributes. + ;; See <https://bugs.gnu.org/40043>. + "-no-xattrs" - ;; Set file times and the file system creation time to - ;; one second after the Epoch. - "-all-time" "1" "-mkfs-time" "1" + ;; Set file times and the file system creation time to + ;; one second after the Epoch. + "-all-time" "1" "-mkfs-time" "1" - ;; Reset all UIDs and GIDs. - "-force-uid" "0" "-force-gid" "0"))) + ;; Reset all UIDs and GIDs. + "-force-uid" "0" "-force-gid" "0"))) - (setenv "PATH" #+(file-append archiver "/bin")) + (setenv "PATH" #+(file-append archiver "/bin")) - ;; We need an empty file in order to have a valid file argument when - ;; we reparent the root file system. Read on for why that's - ;; necessary. - (with-output-to-file ".empty" (lambda () (display ""))) - - ;; Create the squashfs image in several steps. - ;; Add all store items. Unfortunately mksquashfs throws away all - ;; ancestor directories and only keeps the basename. We fix this - ;; in the following invocations of mksquashfs. - (mksquashfs `(,@(map store-info-item - (call-with-input-file "profile" - read-reference-graph)) - #$environment - ,#$output - - ;; Do not perform duplicate checking because we - ;; don't have any dupes. - "-no-duplicates" - "-comp" - ,#+(compressor-name compressor))) - - ;; Here we reparent the store items. For each sub-directory of - ;; the store prefix we need one invocation of "mksquashfs". - (for-each (lambda (dir) - (mksquashfs `(".empty" - ,#$output - "-root-becomes" ,dir))) - (reverse (string-tokenize (%store-directory) - (char-set-complement (char-set #\/))))) - - ;; Add symlinks and mount points. - (mksquashfs - `(".empty" - ,#$output - ;; Create SYMLINKS via pseudo file definitions. - ,@(append-map - (match-lambda - ((source '-> target) - ;; Create relative symlinks to work around a bug in - ;; Singularity 2.x: - ;; https://bugs.gnu.org/34913 - ;; https://github.com/sylabs/singularity/issues/1487 - (let ((target (string-append #$profile "/" target))) - (list "-p" - (string-join - ;; name s mode uid gid symlink - (list source - "s" "777" "0" "0" - (relative-file-name (dirname source) - target))))))) - '#$symlinks*) - - "-p" "/.singularity.d d 555 0 0" - - ;; Create the environment file. - "-p" "/.singularity.d/env d 555 0 0" - "-p" ,(string-append - "/.singularity.d/env/90-environment.sh s 777 0 0 " - (relative-file-name "/.singularity.d/env" - #$environment)) - - ;; Create /.singularity.d/actions, and optionally the 'run' - ;; script, used by 'singularity run'. - "-p" "/.singularity.d/actions d 555 0 0" - - ,@(if entry-point - `(;; This one if for Singularity 2.x. - "-p" - ,(string-append - "/.singularity.d/actions/run s 777 0 0 " - (relative-file-name "/.singularity.d/actions" - (string-append #$profile "/" - entry-point))) - - ;; This one is for Singularity 3.x. - "-p" - ,(string-append - "/.singularity.d/runscript s 777 0 0 " - (relative-file-name "/.singularity.d" - (string-append #$profile "/" - entry-point)))) - '()) - - ;; Create empty mount points. - "-p" "/proc d 555 0 0" - "-p" "/sys d 555 0 0" - "-p" "/dev d 555 0 0" - "-p" "/home d 555 0 0")) - - (when database - ;; Initialize /var/guix. - (install-database-and-gc-roots "var-etc" database #$profile) - (mksquashfs `("var-etc" ,#$output)))))) + ;; We need an empty file in order to have a valid file argument when + ;; we reparent the root file system. Read on for why that's + ;; necessary. + (with-output-to-file ".empty" (lambda () (display ""))) + + ;; Create the squashfs image in several steps. + ;; Add all store items. Unfortunately mksquashfs throws away all + ;; ancestor directories and only keeps the basename. We fix this + ;; in the following invocations of mksquashfs. + (mksquashfs `(,@(map store-info-item + (call-with-input-file "profile" + read-reference-graph)) + #$environment + ,#$output + + ;; Do not perform duplicate checking because we + ;; don't have any dupes. + "-no-duplicates" + "-comp" + ,#+(compressor-name compressor))) + + ;; Here we reparent the store items. For each sub-directory of + ;; the store prefix we need one invocation of "mksquashfs". + (for-each (lambda (dir) + (mksquashfs `(".empty" + ,#$output + "-root-becomes" ,dir))) + (reverse (string-tokenize (%store-directory) + (char-set-complement (char-set #\/))))) + + ;; Add symlinks and mount points. + (mksquashfs + `(".empty" + ,#$output + ;; Create SYMLINKS via pseudo file definitions. + ,@(append-map + (match-lambda + ((source '-> target) + ;; Create relative symlinks to work around a bug in + ;; Singularity 2.x: + ;; https://bugs.gnu.org/34913 + ;; https://github.com/sylabs/singularity/issues/1487 + (let ((target (string-append #$profile "/" target))) + (list "-p" + (string-join + ;; name s mode uid gid symlink + (list source + "s" "777" "0" "0" + (relative-file-name (dirname source) + target))))))) + '#$symlinks*) + + "-p" "/.singularity.d d 555 0 0" + + ;; Create the environment file. + "-p" "/.singularity.d/env d 555 0 0" + "-p" ,(string-append + "/.singularity.d/env/90-environment.sh s 777 0 0 " + (relative-file-name "/.singularity.d/env" + #$environment)) + + ;; Create /.singularity.d/actions, and optionally the 'run' + ;; script, used by 'singularity run'. + "-p" "/.singularity.d/actions d 555 0 0" + + ,@(if entry-point + `( ;; This one if for Singularity 2.x. + "-p" + ,(string-append + "/.singularity.d/actions/run s 777 0 0 " + (relative-file-name "/.singularity.d/actions" + (string-append #$profile "/" + entry-point))) + + ;; This one is for Singularity 3.x. + "-p" + ,(string-append + "/.singularity.d/runscript s 777 0 0 " + (relative-file-name "/.singularity.d" + (string-append #$profile "/" + entry-point)))) + '()) + + ;; Create empty mount points. + "-p" "/proc d 555 0 0" + "-p" "/sys d 555 0 0" + "-p" "/dev d 555 0 0" + "-p" "/home d 555 0 0")) + + (when database + ;; Initialize /var/guix. + (install-database-and-gc-roots "var-etc" database #$profile) + (mksquashfs `("var-etc" ,#$output))))))) (gexp->derivation (string-append name (compressor-extension compressor) diff --git a/guix/scripts/publish.scm b/guix/scripts/publish.scm index c31cef3181..5a865c838d 100644 --- a/guix/scripts/publish.scm +++ b/guix/scripts/publish.scm @@ -824,32 +824,6 @@ example: \"/foo/bar\" yields '(\"foo\" \"bar\")." (define %http-write (@@ (web server http) http-write)) -(match (list (major-version) (minor-version) (micro-version)) - (("2" "2" "5") ;Guile 2.2.5 - (let () - (define %read-line (@ (ice-9 rdelim) %read-line)) - (define bad-header (@@ (web http) bad-header)) - - ;; XXX: Work around <https://bugs.gnu.org/36350> by reverting to the - ;; definition of 'read-header-line' as found in 2.2.4 and earlier. - (define (read-header-line port) - "Read an HTTP header line and return it without its final CRLF or LF. -Raise a 'bad-header' exception if the line does not end in CRLF or LF, -or if EOF is reached." - (match (%read-line port) - (((? string? line) . #\newline) - ;; '%read-line' does not consider #\return a delimiter; so if it's - ;; there, remove it. We are more tolerant than the RFC in that we - ;; tolerate LF-only endings. - (if (string-suffix? "\r" line) - (string-drop-right line 1) - line)) - ((line . _) ;EOF or missing delimiter - (bad-header 'read-header-line line)))) - - (set! (@@ (web http) read-header-line) read-header-line))) - (_ #t)) - (define (strip-headers response) "Return RESPONSE's headers minus 'Content-Length' and our internal headers." (fold alist-delete diff --git a/guix/scripts/substitute.scm b/guix/scripts/substitute.scm index 25075eedff..38702d0c4b 100755 --- a/guix/scripts/substitute.scm +++ b/guix/scripts/substitute.scm @@ -26,7 +26,10 @@ #:use-module (guix combinators) #:use-module (guix config) #:use-module (guix records) - #:use-module ((guix serialization) #:select (restore-file)) + #:use-module (guix diagnostics) + #:use-module (guix i18n) + #:use-module ((guix serialization) #:select (restore-file dump-file)) + #:autoload (guix store deduplication) (dump-file/deduplicate) #:autoload (guix scripts discover) (read-substitute-urls) #:use-module (gcrypt hash) #:use-module (guix base32) @@ -256,6 +259,18 @@ connection (typically PORT) is kept open once data has been fetched from URI." ;; for more information. (contents narinfo-contents)) +(define (narinfo-hash-algorithm+value narinfo) + "Return two values: the hash algorithm used by NARINFO and its value as a +bytevector." + (match (string-tokenize (narinfo-hash narinfo) + (char-set-complement (char-set #\:))) + ((algorithm base32) + (values (lookup-hash-algorithm (string->symbol algorithm)) + (nix-base32-string->bytevector base32))) + (_ + (raise (formatted-message + (G_ "invalid narinfo hash: ~s") (narinfo-hash narinfo)))))) + (define (narinfo-hash->sha256 hash) "If the string HASH denotes a sha256 hash, return it as a bytevector. Otherwise return #f." @@ -1031,22 +1046,33 @@ one. Return #f if URI's scheme is 'file' or #f." (call-with-cached-connection uri (lambda (port) exp ...))) (define* (process-substitution store-item destination - #:key cache-urls acl print-build-trace?) + #:key cache-urls acl + deduplicate? print-build-trace?) "Substitute STORE-ITEM (a store file name) from CACHE-URLS, and write it to -DESTINATION as a nar file. Verify the substitute against ACL." +DESTINATION as a nar file. Verify the substitute against ACL, and verify its +hash against what appears in the narinfo. When DEDUPLICATE? is true, and if +DESTINATION is in the store, deduplicate its files. Print a status line on +the current output port." (define narinfo (lookup-narinfo cache-urls store-item (cut valid-narinfo? <> acl))) + (define destination-in-store? + (string-prefix? (string-append (%store-prefix) "/") + destination)) + + (define (dump-file/deduplicate* . args) + ;; Make sure deduplication looks at the right store (necessary in test + ;; environments). + (apply dump-file/deduplicate + (append args (list #:store (%store-prefix))))) + (unless narinfo (leave (G_ "no valid substitute for '~a'~%") store-item)) (let-values (((uri compression file-size) (narinfo-best-uri narinfo))) - ;; Tell the daemon what the expected hash of the Nar itself is. - (format #t "~a~%" (narinfo-hash narinfo)) - (unless print-build-trace? (format (current-error-port) (G_ "Downloading ~a...~%") (uri->string uri))) @@ -1079,9 +1105,20 @@ DESTINATION as a nar file. Verify the substitute against ACL." ;; closed here, while the child process doing the ;; reporting will close it upon exit. (decompressed-port (string->symbol compression) - progress))) + progress)) + + ;; Compute the actual nar hash as we read it. + ((algorithm expected) + (narinfo-hash-algorithm+value narinfo)) + ((hashed get-hash) + (open-hash-input-port algorithm input))) ;; Unpack the Nar at INPUT into DESTINATION. - (restore-file input destination) + (restore-file hashed destination + #:dump-file (if (and destination-in-store? + deduplicate?) + dump-file/deduplicate* + dump-file)) + (close-port hashed) (close-port input) ;; Wait for the reporter to finish. @@ -1091,8 +1128,17 @@ DESTINATION as a nar file. Verify the substitute against ACL." ;; one to visually separate substitutions. (display "\n\n" (current-error-port)) - ;; Tell the daemon that we're done. - (display "success\n" (current-output-port))))) + ;; Check whether we got the data announced in NARINFO. + (let ((actual (get-hash))) + (if (bytevector=? actual expected) + ;; Tell the daemon that we're done. + (format (current-output-port) "success ~a ~a~%" + (narinfo-hash narinfo) (narinfo-size narinfo)) + ;; The actual data has a different hash than that in NARINFO. + (format (current-output-port) "hash-mismatch ~a ~a ~a~%" + (hash-algorithm-name algorithm) + (bytevector->nix-base32-string expected) + (bytevector->nix-base32-string actual))))))) ;;; @@ -1219,6 +1265,9 @@ default value." ((= string->number number) (> number 0)) (_ #f))) + (define deduplicate? + (find-daemon-option "deduplicate")) + ;; The daemon's agent code opens file descriptor 4 for us and this is where ;; stderr should go. (parameterize ((current-error-port (if (%error-to-file-descriptor-4?) @@ -1278,6 +1327,7 @@ default value." (process-substitution store-path destination #:cache-urls (substitute-urls) #:acl (current-acl) + #:deduplicate? deduplicate? #:print-build-trace? print-build-trace?) (loop)))))) diff --git a/guix/scripts/system.scm b/guix/scripts/system.scm index db80e0be8f..0dcf2b3afe 100644 --- a/guix/scripts/system.scm +++ b/guix/scripts/system.scm @@ -29,7 +29,10 @@ #:use-module (guix ui) #:use-module ((guix status) #:select (with-status-verbosity)) #:use-module (guix store) - #:autoload (guix store database) (register-path) + #:autoload (guix base16) (bytevector->base16-string) + #:autoload (guix store database) + (sqlite-register store-database-file call-with-database) + #:autoload (guix build store-copy) (copy-store-item) #:use-module (guix describe) #:use-module (guix grafts) #:use-module (guix gexp) @@ -129,12 +132,11 @@ BODY..., and restore them." (store-lift topologically-sorted)) -(define* (copy-item item references target +(define* (copy-item item info target db #:key (log-port (current-error-port))) - "Copy ITEM to the store under root directory TARGET and register it with -REFERENCES as its set of references." - (let ((dest (string-append target item)) - (state (string-append target "/var/guix"))) + "Copy ITEM to the store under root directory TARGET and populate DB with the +given INFO, a <path-info> record." + (let ((dest (string-append target item))) (format log-port "copying '~a'...~%" item) ;; Remove DEST if it exists to make sure that (1) we do not fail badly @@ -147,44 +149,48 @@ REFERENCES as its set of references." #:directories? #t)) (delete-file-recursively dest)) - (copy-recursively item dest - #:log (%make-void-port "w")) + (copy-store-item item target + #:deduplicate? #t) - ;; Register ITEM; as a side-effect, it resets timestamps, etc. - ;; Explicitly use "TARGET/var/guix" as the state directory, to avoid - ;; reproducing the user's current settings; see - ;; <http://bugs.gnu.org/18049>. - (unless (register-path item - #:prefix target - #:state-directory state - #:references references) - (leave (G_ "failed to register '~a' under '~a'~%") - item target)))) + (sqlite-register db + #:path item + #:references (path-info-references info) + #:deriver (path-info-deriver info) + #:hash (string-append + "sha256:" + (bytevector->base16-string (path-info-hash info))) + #:nar-size (path-info-nar-size info)))) (define* (copy-closure item target #:key (log-port (current-error-port))) "Copy ITEM and all its dependencies to the store under root directory TARGET, and register them." (mlet* %store-monad ((to-copy (topologically-sorted* (list item))) - (refs (mapm %store-monad references* to-copy)) - (info (mapm %store-monad query-path-info* - (delete-duplicates - (append to-copy (concatenate refs))))) + (info (mapm %store-monad query-path-info* to-copy)) (size -> (reduce + 0 (map path-info-nar-size info)))) (define progress-bar (progress-reporter/bar (length to-copy) (format #f (G_ "copying to '~a'...") target))) + (define state + (string-append target "/var/guix")) + (check-available-space size target) - (call-with-progress-reporter progress-bar - (lambda (report) - (let ((void (%make-void-port "w"))) - (for-each (lambda (item refs) - (copy-item item refs target #:log-port void) - (report)) - to-copy refs)))) + ;; Explicitly use "TARGET/var/guix" as the state directory to avoid + ;; reproducing the user's current settings; see + ;; <http://bugs.gnu.org/18049>. + (call-with-database (store-database-file #:prefix target + #:state-directory state) + (lambda (db) + (call-with-progress-reporter progress-bar + (lambda (report) + (let ((void (%make-void-port "w"))) + (for-each (lambda (item info) + (copy-item item info target db #:log-port void) + (report)) + to-copy info)))))) (return *unspecified*))) @@ -385,6 +391,7 @@ STORE is an open connection to the store." (params (first (profile-boot-parameters %system-profile (list number)))) (locale (boot-parameters-locale params)) + (store-crypto-devices (boot-parameters-store-crypto-devices params)) (store-directory-prefix (boot-parameters-store-directory-prefix params)) (old-generations @@ -400,6 +407,7 @@ STORE is an open connection to the store." ((bootloader-configuration-file-generator bootloader) bootloader-config entries #:locale locale + #:store-crypto-devices store-crypto-devices #:store-directory-prefix store-directory-prefix #:old-entries old-entries))) (drvs -> (list bootcfg))) diff --git a/guix/serialization.scm b/guix/serialization.scm index 836ad06caf..59cd93fb18 100644 --- a/guix/serialization.scm +++ b/guix/serialization.scm @@ -51,7 +51,8 @@ write-file write-file-tree fold-archive - restore-file)) + restore-file + dump-file)) ;;; Comment: ;;; @@ -444,7 +445,8 @@ depends on TYPE." (file file) (token x)))))) (loop (read-string port) result))))) - (")" result) ;done with DIR + (")" ;done with DIR + (proc file 'directory-complete #f result)) (x (raise (condition @@ -456,23 +458,43 @@ depends on TYPE." (&message (message "unsupported nar entry type")) (&nar-read-error (port port) (file file) (token x))))))))) -(define (restore-file port file) +(define (dump-file file input size type) + "Dump SIZE bytes from INPUT to FILE. + +This procedure is suitable for use as the #:dump-file argument to +'restore-file'." + (call-with-output-file file + (lambda (output) + (dump input output size)))) + +(define* (restore-file port file + #:key (dump-file dump-file)) "Read a file (possibly a directory structure) in Nar format from PORT. -Restore it as FILE." +Restore it as FILE with canonical permissions and timestamps. To write a +regular or executable file, call: + + (DUMP-FILE FILE INPUT SIZE TYPE) + +The default is to dump SIZE bytes from INPUT to FILE, but callers can provide +a custom procedure, for instance to deduplicate FILE on the fly." (fold-archive (lambda (file type content result) (match type ('directory (mkdir file)) + ('directory-complete + (chmod file #o555) + (utime file 1 1 0 0)) ('symlink - (symlink content file)) + (symlink content file) + (utime file 1 1 0 0 AT_SYMLINK_NOFOLLOW)) ((or 'regular 'executable) (match content ((input . size) - (call-with-output-file file - (lambda (output) - (dump input output size) - (when (eq? type 'executable) - (chmod output #o755))))))))) + (dump-file file input size type) + (chmod file (if (eq? type 'executable) + #o555 + #o444)) + (utime file 1 1 0 0)))))) #t port file)) diff --git a/guix/status.scm b/guix/status.scm index f40d5d59b9..9ca6d92470 100644 --- a/guix/status.scm +++ b/guix/status.scm @@ -423,6 +423,9 @@ addition to build events." (cute colorize-string <> (color RED BOLD)) identity)) + (define tty? + (isatty?* port)) + (define (report-build-progress phase %) (let ((% (min (max % 0) 100))) ;sanitize (erase-current-line port) @@ -542,6 +545,7 @@ addition to build events." (nar-uri-abbreviation uri) (basename uri)))) (display-download-progress uri size + #:tty? tty? #:start-time (download-start download) #:transferred transferred)))))) diff --git a/guix/store/database.scm b/guix/store/database.scm index b36b127630..0a84bbddb9 100644 --- a/guix/store/database.scm +++ b/guix/store/database.scm @@ -39,10 +39,10 @@ #:export (sql-schema %default-database-file store-database-file + call-with-database with-database path-id sqlite-register - register-path register-items %epoch reset-timestamps)) @@ -323,8 +323,19 @@ ids of items referred to." (sqlite-fold cons '() stmt)) references))) +(define (timestamp) + "Return a timestamp, either the current time of SOURCE_DATE_EPOCH." + (match (getenv "SOURCE_DATE_EPOCH") + (#f + (current-time time-utc)) + ((= string->number seconds) + (if seconds + (make-time time-utc 0 seconds) + (current-time time-utc))))) + (define* (sqlite-register db #:key path (references '()) - deriver hash nar-size time) + deriver hash nar-size + (time (timestamp))) "Registers this stuff in DB. PATH is the store item to register and REFERENCES is the list of store items PATH refers to; DERIVER is the '.drv' that produced PATH, HASH is the base16-encoded Nix sha256 hash of @@ -337,9 +348,7 @@ Every store item in REFERENCES must already be registered." #:deriver deriver #:hash hash #:nar-size nar-size - #:time (time-second - (or time - (current-time time-utc)))))) + #:time (time-second time)))) ;; Call 'path-id' on each of REFERENCES. This ensures we get a ;; "non-NULL constraint" failure if one of REFERENCES is unregistered. (add-references db id @@ -382,44 +391,13 @@ is true." (chmod file (if (executable-file? file) #o555 #o444))) (utime file 1 1 0 0))))) -(define* (register-path path - #:key (references '()) deriver prefix - state-directory (deduplicate? #t) - (reset-timestamps? #t) - (schema (sql-schema))) - "Register PATH as a valid store file, with REFERENCES as its list of -references, and DERIVER as its deriver (.drv that led to it.) If PREFIX is -given, it must be the name of the directory containing the new store to -initialize; if STATE-DIRECTORY is given, it must be a string containing the -absolute file name to the state directory of the store being initialized. -Return #t on success. - -Use with care as it directly modifies the store! This is primarily meant to -be used internally by the daemon's build hook. - -PATH must be protected from GC and locked during execution of this, typically -by adding it as a temp-root." - (define db-file - (store-database-file #:prefix prefix - #:state-directory state-directory)) - - (parameterize ((sql-schema schema)) - (with-database db-file db - (register-items db (list (store-info path deriver references)) - #:prefix prefix - #:deduplicate? deduplicate? - #:reset-timestamps? reset-timestamps? - #:log-port (%make-void-port "w"))))) - (define %epoch ;; When it all began. (make-time time-utc 0 1)) (define* (register-items db items #:key prefix - (deduplicate? #t) - (reset-timestamps? #t) - registration-time + (registration-time (timestamp)) (log-port (current-error-port))) "Register all of ITEMS, a list of <store-info> records as returned by 'read-reference-graph', in DB. ITEMS must be in topological order (with @@ -452,8 +430,6 @@ typically by adding them as temp-roots." ;; significant differences when 'register-closures' is called ;; consecutively for overlapping closures such as 'system' and 'bootcfg'. (unless (path-id db to-register) - (when reset-timestamps? - (reset-timestamps real-file-name)) (let-values (((hash nar-size) (nar-sha256 real-file-name))) (call-with-retrying-transaction db (lambda () @@ -464,9 +440,7 @@ typically by adding them as temp-roots." "sha256:" (bytevector->base16-string hash)) #:nar-size nar-size - #:time registration-time))) - (when deduplicate? - (deduplicate real-file-name hash #:store store-dir))))) + #:time registration-time)))))) (let* ((prefix (format #f "registering ~a items" (length items))) (progress (progress-reporter/bar (length items) diff --git a/guix/store/deduplication.scm b/guix/store/deduplication.scm index 0655ceb890..cd9660174c 100644 --- a/guix/store/deduplication.scm +++ b/guix/store/deduplication.scm @@ -26,45 +26,25 @@ #:use-module (guix build syscalls) #:use-module (guix base32) #:use-module (srfi srfi-11) + #:use-module (srfi srfi-34) + #:use-module (srfi srfi-35) #:use-module (rnrs io ports) #:use-module (ice-9 ftw) #:use-module (ice-9 match) #:use-module (guix serialization) #:export (nar-sha256 - deduplicate)) - -;; XXX: This port is used as a workaround on Guile <= 2.2.4 where -;; 'port-position' throws to 'out-of-range' when the offset is great than or -;; equal to 2^32: <https://bugs.gnu.org/32161>. -(define (counting-wrapper-port output-port) - "Return two values: an output port that wraps OUTPUT-PORT, and a thunk to -retrieve the number of bytes written to OUTPUT-PORT." - (let ((byte-count 0)) - (values (make-custom-binary-output-port "counting-wrapper" - (lambda (bytes offset count) - (put-bytevector output-port bytes - offset count) - (set! byte-count - (+ byte-count count)) - count) - (lambda () - byte-count) - #f - (lambda () - (close-port output-port))) - (lambda () - byte-count)))) + deduplicate + dump-file/deduplicate + copy-file/deduplicate)) (define (nar-sha256 file) "Gives the sha256 hash of a file and the size of the file in nar form." - (let*-values (((port get-hash) (open-sha256-port)) - ((wrapper get-size) (counting-wrapper-port port))) - (write-file file wrapper) - (force-output wrapper) + (let-values (((port get-hash) (open-sha256-port))) + (write-file file port) (force-output port) (let ((hash (get-hash)) - (size (get-size))) - (close-port wrapper) + (size (port-position port))) + (close-port port) (values hash size)))) (define (tempname-in directory) @@ -155,49 +135,118 @@ under STORE." (define links-directory (string-append store "/.links")) - (mkdir-p links-directory) - (let loop ((path path) - (type (stat:type (lstat path))) - (hash hash)) - (if (eq? 'directory type) - ;; Can't hardlink directories, so hardlink their atoms. - (for-each (match-lambda - ((file . properties) - (unless (member file '("." "..")) - (let* ((file (string-append path "/" file)) - (type (match (assoc-ref properties 'type) - ((or 'unknown #f) - (stat:type (lstat file))) - (type type)))) - (loop file type - (and (not (eq? 'directory type)) - (nar-sha256 file))))))) - (scandir* path)) - (let ((link-file (string-append links-directory "/" - (bytevector->nix-base32-string hash)))) - (if (file-exists? link-file) - (replace-with-link link-file path - #:swap-directory links-directory - #:store store) - (catch 'system-error - (lambda () - (link path link-file)) - (lambda args - (let ((errno (system-error-errno args))) - (cond ((= errno EEXIST) - ;; Someone else put an entry for PATH in - ;; LINKS-DIRECTORY before we could. Let's use it. - (replace-with-link path link-file - #:swap-directory - links-directory - #:store store)) - ((= errno ENOSPC) - ;; There's not enough room in the directory index for - ;; more entries in .links, but that's fine: we can - ;; just stop. - #f) - ((= errno EMLINK) - ;; PATH has reached the maximum number of links, but - ;; that's OK: we just can't deduplicate it more. - #f) - (else (apply throw args))))))))))) + (let loop ((path path) + (type (stat:type (lstat path))) + (hash hash)) + (if (eq? 'directory type) + ;; Can't hardlink directories, so hardlink their atoms. + (for-each (match-lambda + ((file . properties) + (unless (member file '("." "..")) + (let* ((file (string-append path "/" file)) + (type (match (assoc-ref properties 'type) + ((or 'unknown #f) + (stat:type (lstat file))) + (type type)))) + (loop file type + (and (not (eq? 'directory type)) + (nar-sha256 file))))))) + (scandir* path)) + (let ((link-file (string-append links-directory "/" + (bytevector->nix-base32-string hash)))) + (if (file-exists? link-file) + (replace-with-link link-file path + #:swap-directory links-directory + #:store store) + (catch 'system-error + (lambda () + (link path link-file)) + (lambda args + (let ((errno (system-error-errno args))) + (cond ((= errno EEXIST) + ;; Someone else put an entry for PATH in + ;; LINKS-DIRECTORY before we could. Let's use it. + (replace-with-link path link-file + #:swap-directory + links-directory + #:store store)) + ((= errno ENOENT) + ;; This most likely means that LINKS-DIRECTORY does + ;; not exist. Attempt to create it and try again. + (mkdir-p links-directory) + (loop path type hash)) + ((= errno ENOSPC) + ;; There's not enough room in the directory index for + ;; more entries in .links, but that's fine: we can + ;; just stop. + #f) + ((= errno EMLINK) + ;; PATH has reached the maximum number of links, but + ;; that's OK: we just can't deduplicate it more. + #f) + (else (apply throw args))))))))))) + +(define (tee input len output) + "Return a port that reads up to LEN bytes from INPUT and writes them to +OUTPUT as it goes." + (define bytes-read 0) + + (define (fail) + ;; Reached EOF before we had read LEN bytes from INPUT. + (raise (condition + (&nar-error (port input) + (file (port-filename output)))))) + + (define (read! bv start count) + ;; Read at most LEN bytes in total. + (let ((count (min count (- len bytes-read)))) + (let loop ((ret (get-bytevector-n! input bv start count))) + (cond ((eof-object? ret) + (if (= bytes-read len) + 0 ; EOF + (fail))) + ((and (zero? ret) (> count 0)) + ;; Do not return zero since zero means EOF, so try again. + (loop (get-bytevector-n! input bv start count))) + (else + (put-bytevector output bv start ret) + (set! bytes-read (+ bytes-read ret)) + ret))))) + + (make-custom-binary-input-port "tee input port" read! #f #f #f)) + +(define* (dump-file/deduplicate file input size type + #:key (store (%store-directory))) + "Write SIZE bytes read from INPUT to FILE. TYPE is a symbol, either +'regular or 'executable. + +This procedure is suitable as a #:dump-file argument to 'restore-file'. When +used that way, it deduplicates files on the fly as they are restored, thereby +removing the need to a deduplication pass that would re-read all the files +down the road." + (define hash + (call-with-output-file file + (lambda (output) + (let-values (((hash-port get-hash) + (open-hash-port (hash-algorithm sha256)))) + (write-file-tree file hash-port + #:file-type+size (lambda (_) (values type size)) + #:file-port + (const (tee input size output))) + (close-port hash-port) + (get-hash))))) + + (deduplicate file hash #:store store)) + +(define* (copy-file/deduplicate source target + #:key (store (%store-directory))) + "Like 'copy-file', but additionally deduplicate TARGET in STORE." + (call-with-input-file source + (lambda (input) + (let ((stat (stat input))) + (dump-file/deduplicate target input (stat:size stat) + (if (zero? (logand (stat:mode stat) + #o100)) + 'regular + 'executable) + #:store store))))) |