diff options
author | Ludovic Courtès <ludo@gnu.org> | 2021-09-07 11:04:44 +0200 |
---|---|---|
committer | Ludovic Courtès <ludo@gnu.org> | 2021-09-07 14:19:08 +0200 |
commit | d9dfbf886ddbb92dfdaa118bb9765e78aad5c53a (patch) | |
tree | 2732020de20a38c09b66a60b0cb36022799f7c2e /guix | |
parent | b949f34f31a045eb0fb242b81a223178fb6994d3 (diff) | |
parent | 49922efb11da0f0e9d4f5979d081de5ea8c99d25 (diff) |
Merge branch 'master' into core-updates-frozen
Diffstat (limited to 'guix')
-rw-r--r-- | guix/build-system/chicken.scm | 10 | ||||
-rw-r--r-- | guix/build-system/dune.scm | 3 | ||||
-rw-r--r-- | guix/build-system/minetest.scm | 99 | ||||
-rw-r--r-- | guix/build/dune-build-system.scm | 8 | ||||
-rw-r--r-- | guix/build/minetest-build-system.scm | 229 | ||||
-rw-r--r-- | guix/ci.scm | 34 | ||||
-rw-r--r-- | guix/import/egg.scm | 5 | ||||
-rw-r--r-- | guix/import/go.scm | 50 | ||||
-rw-r--r-- | guix/import/minetest.scm | 456 | ||||
-rw-r--r-- | guix/import/opam.scm | 142 | ||||
-rw-r--r-- | guix/import/utils.scm | 44 | ||||
-rw-r--r-- | guix/scripts/import.scm | 3 | ||||
-rw-r--r-- | guix/scripts/import/gem.scm | 3 | ||||
-rw-r--r-- | guix/scripts/import/go.scm | 6 | ||||
-rw-r--r-- | guix/scripts/import/minetest.scm | 117 | ||||
-rw-r--r-- | guix/scripts/import/opam.scm | 8 | ||||
-rw-r--r-- | guix/scripts/publish.scm | 21 | ||||
-rw-r--r-- | guix/scripts/system.scm | 20 | ||||
-rw-r--r-- | guix/scripts/system/reconfigure.scm | 22 | ||||
-rw-r--r-- | guix/scripts/weather.scm | 22 | ||||
-rw-r--r-- | guix/swh.scm | 43 |
21 files changed, 1202 insertions, 143 deletions
diff --git a/guix/build-system/chicken.scm b/guix/build-system/chicken.scm index c6978266fc..07666d1321 100644 --- a/guix/build-system/chicken.scm +++ b/guix/build-system/chicken.scm @@ -1,6 +1,7 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2020 raingloom <raingloom@riseup.net> ;;; Copyright © 2021 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2021 Xinglu Chen <public@yoctocell.xyz> ;;; ;;; This file is part of GNU Guix. ;;; @@ -29,7 +30,14 @@ #:use-module (ice-9 match) #:export (%chicken-build-system-modules chicken-build - chicken-build-system)) + chicken-build-system + egg-uri)) + +(define* (egg-uri name version #:optional (extension ".tar.gz")) + "Return a URI string for the CHICKEN egg corresponding to NAME and VERSION. +EXTENSION is the file name extension, such as '.tar.gz'." + (string-append "https://code.call-cc.org/egg-tarballs/5/" + name "/" name "-" version extension)) (define %chicken-build-system-modules ;; Build-side modules imported and used by default. diff --git a/guix/build-system/dune.scm b/guix/build-system/dune.scm index 8c33e096f5..303b5f76c6 100644 --- a/guix/build-system/dune.scm +++ b/guix/build-system/dune.scm @@ -2,6 +2,7 @@ ;;; Copyright © 2016, 2017, 2018 Julien Lepiller <julien@lepiller.eu> ;;; Copyright © 2017 Ben Woodcroft <donttrustben@gmail.com> ;;; Copyright © 2021 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2021 pukkamustard <pukkamustard@posteo.net> ;;; ;;; This file is part of GNU Guix. ;;; @@ -90,6 +91,7 @@ (out-of-source? #t) (jbuild? #f) (package #f) + (profile "release") (tests? #t) (test-flags ''()) (test-target "test") @@ -129,6 +131,7 @@ provides a 'setup.ml' file as its build system." #:out-of-source? #$out-of-source? #:jbuild? #$jbuild? #:package #$package + #:profile #$profile #:tests? #$tests? #:test-target #$test-target #:install-target #$install-target diff --git a/guix/build-system/minetest.scm b/guix/build-system/minetest.scm new file mode 100644 index 0000000000..1fae3a47e9 --- /dev/null +++ b/guix/build-system/minetest.scm @@ -0,0 +1,99 @@ +;;; GNU Guix --- Functional package management for GNU +;;; Copyright © 2021 Maxime Devos <maximedevos@telenet.be> +;;; +;;; This file is part of GNU Guix. +;;; +;;; GNU Guix is free software; you can redistribute it and/or modify it +;;; under the terms of the GNU General Public License as published by +;;; the Free Software Foundation; either version 3 of the License, or (at +;;; your option) any later version. +;;; +;;; GNU Guix is distributed in the hope that it will be useful, but +;;; WITHOUT ANY WARRANTY; without even the implied warranty of +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;;; GNU General Public License for more details. +;;; +;;; You should have received a copy of the GNU General Public License +;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>. + +(define-module (guix build-system minetest) + #:use-module (guix build-system copy) + #:use-module (guix build-system gnu) + #:use-module (guix build-system) + #:use-module (guix utils) + #:export (minetest-mod-build-system)) + +;; +;; Build procedure for minetest mods. This is implemented as an extension +;; of ‘copy-build-system’. +;; +;; Code: + +;; Lazily resolve the bindings to avoid circular dependencies. +(define (default-optipng) + ;; Lazily resolve the binding to avoid a circular dependency. + (module-ref (resolve-interface '(gnu packages image)) 'optipng)) + +(define (default-minetest) + (module-ref (resolve-interface '(gnu packages minetest)) 'minetest)) + +(define (default-xvfb-run) + (module-ref (resolve-interface '(gnu packages xorg)) 'xvfb-run)) + +(define %minetest-build-system-modules + ;; Build-side modules imported by default. + `((guix build minetest-build-system) + ,@%copy-build-system-modules)) + +(define %default-modules + ;; Modules in scope in the build-side environment. + '((guix build gnu-build-system) + (guix build minetest-build-system) + (guix build utils))) + +(define (standard-minetest-packages) + "Return the list of (NAME PACKAGE OUTPUT) or (NAME PACKAGE) tuples of +standard packages used as implicit inputs of the Minetest build system." + `(("xvfb-run" ,(default-xvfb-run)) + ("optipng" ,(default-optipng)) + ("minetest" ,(default-minetest)) + ,@(filter (lambda (input) + (member (car input) + '("libc" "tar" "gzip" "bzip2" "xz" "locales"))) + (standard-packages)))) + +(define* (lower-mod name #:key (implicit-inputs? #t) #:allow-other-keys + #:rest arguments) + (define lower (build-system-lower gnu-build-system)) + (apply lower + name + (substitute-keyword-arguments arguments + ;; minetest-mod-build-system adds implicit inputs by itself, + ;; so don't let gnu-build-system add its own implicit inputs + ;; as well. + ((#:implicit-inputs? implicit-inputs? #t) + #f) + ((#:implicit-cross-inputs? implicit-cross-inputs? #t) + #f) + ((#:imported-modules imported-modules %minetest-build-system-modules) + imported-modules) + ((#:modules modules %default-modules) + modules) + ((#:phases phases '%standard-phases) + phases) + ;; Ensure nothing sneaks into the closure. + ((#:allowed-references allowed-references '()) + allowed-references) + ;; Add the implicit inputs. + ((#:native-inputs native-inputs '()) + (if implicit-inputs? + (append native-inputs (standard-minetest-packages)) + native-inputs))))) + +(define minetest-mod-build-system + (build-system + (name 'minetest-mod) + (description "The build system for minetest mods") + (lower lower-mod))) + +;;; minetest.scm ends here diff --git a/guix/build/dune-build-system.scm b/guix/build/dune-build-system.scm index 7e2ec1e3e1..6a0c2593ac 100644 --- a/guix/build/dune-build-system.scm +++ b/guix/build/dune-build-system.scm @@ -1,6 +1,7 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2018 Julien Lepiller <julien@lepiller.eu> ;;; Copyright © 2019 Gabriel Hondet <gabrielhondet@gmail.com> +;;; Copyright © 2021 pukkamustard <pukkamustard@posteo.net> ;;; ;;; This file is part of GNU Guix. ;;; @@ -31,11 +32,14 @@ ;; Code: (define* (build #:key (build-flags '()) (jbuild? #f) - (use-make? #f) (package #f) #:allow-other-keys) + (use-make? #f) (package #f) + (profile "release") #:allow-other-keys) "Build the given package." (let ((program (if jbuild? "jbuilder" "dune"))) (apply invoke program "build" "@install" - (append (if package (list "-p" package) '()) build-flags))) + (append (if package (list "-p" package) '()) + `("--profile" ,profile) + build-flags))) #t) (define* (check #:key (test-flags '()) (test-target "test") tests? diff --git a/guix/build/minetest-build-system.scm b/guix/build/minetest-build-system.scm new file mode 100644 index 0000000000..477cc3d1d0 --- /dev/null +++ b/guix/build/minetest-build-system.scm @@ -0,0 +1,229 @@ +;;; Copyright © 2021 Maxime Devos <maximedevos@telenet.be> +;;; +;;; This file is part of GNU Guix. +;;; +;;; GNU Guix is free software; you can redistribute it and/or modify it +;;; under the terms of the GNU General Public License as published by +;;; the Free Software Foundation; either version 3 of the License, or (at +;;; your option) any later version. +;;; +;;; GNU Guix is distributed in the hope that it will be useful, but +;;; WITHOUT ANY WARRANTY; without even the implied warranty of +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;;; GNU General Public License for more details. +;;; +;;; You should have received a copy of the GNU General Public License +;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>. + +(define-module (guix build minetest-build-system) + #:use-module (guix build utils) + #:use-module (srfi srfi-1) + #:use-module (ice-9 format) + #:use-module (ice-9 match) + #:use-module (ice-9 rdelim) + #:use-module (ice-9 receive) + #:use-module (ice-9 regex) + #:use-module ((guix build gnu-build-system) #:prefix gnu:) + #:use-module ((guix build copy-build-system) #:prefix copy:) + #:export (%standard-phases + mod-install-plan minimise-png read-mod-name check)) + +;; (guix build copy-build-system) does not export 'install'. +(define copy:install + (assoc-ref copy:%standard-phases 'install)) + +(define (mod-install-plan mod-name) + `(("." ,(string-append "share/minetest/mods/" mod-name) + ;; Only install files that will actually be used at run time. + ;; This can save a little disk space. + ;; + ;; See <https://github.com/minetest/minetest/blob/master/doc/lua_api.txt> + ;; for an incomple list of files that can be found in mods. + #:include ("mod.conf" "modpack.conf" "settingtypes.txt" "depends.txt" + "description.txt") + #:include-regexp (".lua$" ".png$" ".ogg$" ".obj$" ".b3d$" ".tr$" + ".mts$")))) + +(define* (guess-mod-name #:key inputs #:allow-other-keys) + "Try to determine the name of the mod or modpack that is being built. +If it is unknown, make an educated guess." + ;; Minetest doesn't care about the directory names in "share/minetest/mods" + ;; so there is no technical problem if the directory names don't match + ;; the mod names. The directory can appear in the GUI if the modpack + ;; doesn't have the 'name' set though, so try to make a guess. + (define (guess) + (let* ((source (assoc-ref inputs "source")) + ;; Don't retain a reference to the store. + (file-name (strip-store-file-name source)) + ;; The "minetest-" prefix is not informative, so strip it. + (file-name (if (string-prefix? "minetest-" file-name) + (substring file-name (string-length "minetest-")) + file-name)) + ;; Strip "-checkout" suffixes of git checkouts. + (file-name (if (string-suffix? "-checkout" file-name) + (substring file-name + 0 + (- (string-length file-name) + (string-length "-checkout"))) + file-name)) + (first-dot (string-index file-name #\.)) + ;; If the source code is in an archive (.tar.gz, .zip, ...), + ;; strip the extension. + (file-name (if first-dot + (substring file-name 0 first-dot) + file-name))) + (format (current-error-port) + "warning: the modpack ~a did not set 'name' in 'modpack.conf'~%" + file-name) + file-name)) + (cond ((file-exists? "mod.conf") + ;; Mods must have 'name' set in "mod.conf", so don't guess. + (read-mod-name "mod.conf")) + ((file-exists? "modpack.conf") + ;; While it is recommended to have 'name' set in 'modpack.conf', + ;; it is optional, so guess a name if necessary. + (read-mod-name "modpack.conf" guess)) + (#t (guess)))) + +(define* (install #:key inputs #:allow-other-keys #:rest arguments) + (apply copy:install + #:install-plan (mod-install-plan (apply guess-mod-name arguments)) + arguments)) + +(define %png-magic-bytes + ;; Magic bytes of PNG images, see ‘5.2 PNG signatures’ in + ;; ‘Portable Network Graphics (PNG) Specification (Second Edition)’ + ;; on <https://www.w3.org/TR/PNG/>. + #vu8(137 80 78 71 13 10 26 10)) + +(define png-file? + ((@@ (guix build utils) file-header-match) %png-magic-bytes)) + +(define* (minimise-png #:key inputs native-inputs #:allow-other-keys) + "Minimise PNG images found in the working directory." + (define optipng (which "optipng")) + (define (optimise image) + (format #t "Optimising ~a~%" image) + (make-file-writable (dirname image)) + (make-file-writable image) + (define old-size (stat:size (stat image))) + ;; The mod "technic" has a file "technic_music_player_top.png" that + ;; actually is a JPEG file, see + ;; <https://github.com/minetest-mods/technic/issues/590>. + (if (png-file? image) + (invoke optipng "-o4" "-quiet" image) + (format #t "warning: skipping ~a because it's not actually a PNG image~%" + image)) + (define new-size (stat:size (stat image))) + (values old-size new-size)) + (define files (find-files "." ".png$")) + (let loop ((total-old-size 0) + (total-new-size 0) + (images (find-files "." ".png$"))) + (cond ((pair? images) + (receive (old-size new-size) + (optimise (car images)) + (loop (+ total-old-size old-size) + (+ total-new-size new-size) + (cdr images)))) + ((= total-old-size 0) + (format #t "There were no PNG images to minimise.")) + (#t + (format #t "Minimisation reduced size of images by ~,2f% (~,2f MiB to ~,2f MiB)~%" + (* 100.0 (- 1 (/ total-new-size total-old-size))) + (/ total-old-size (expt 1024 2)) + (/ total-new-size (expt 1024 2))))))) + +(define name-regexp (make-regexp "^name[ ]*=(.+)$")) + +(define* (read-mod-name mod.conf #:optional not-found) + "Read the name of a mod from MOD.CONF. If MOD.CONF +does not have a name field and NOT-FOUND is #false, raise an +error. If NOT-FOUND is TRUE, call NOT-FOUND instead." + (call-with-input-file mod.conf + (lambda (port) + (let loop () + (define line (read-line port)) + (if (eof-object? line) + (if not-found + (not-found) + (error "~a does not have a 'name' field" mod.conf)) + (let ((match (regexp-exec name-regexp line))) + (if (regexp-match? match) + (string-trim-both (match:substring match 1) #\ ) + (loop)))))))) + +(define* (check #:key outputs tests? #:allow-other-keys) + "Test whether the mod loads. The mod must first be installed first." + (define (all-mod-names directories) + (append-map + (lambda (directory) + (map read-mod-name (find-files directory "mod.conf"))) + directories)) + (when tests? + (mkdir "guix_testworld") + ;; Add the mod to the mod search path, such that Minetest can find it. + (setenv "MINETEST_MOD_PATH" + (list->search-path-as-string + (cons + (string-append (assoc-ref outputs "out") "/share/minetest/mods") + (search-path-as-string->list + (or (getenv "MINETEST_MOD_PATH") ""))) + ":")) + (with-directory-excursion "guix_testworld" + (setenv "HOME" (getcwd)) + ;; Create a world in which all mods are loaded. + (call-with-output-file "world.mt" + (lambda (port) + (display + "gameid = minetest +world_name = guix_testworld +backend = sqlite3 +player_backend = sqlite3 +auth_backend = sqlite3 +" port) + (for-each + (lambda (mod) + (format port "load_mod_~a = true~%" mod)) + (all-mod-names (search-path-as-string->list + (getenv "MINETEST_MOD_PATH")))))) + (receive (port pid) + ((@@ (guix build utils) open-pipe-with-stderr) + "xvfb-run" "--" "minetest" "--info" "--world" "." "--go") + (format #t "Started Minetest with all mods loaded for testing~%") + ;; Scan the output for error messages. + ;; When the player has joined the server, stop minetest. + (define (error? line) + (and (string? line) + (string-contains line ": ERROR["))) + (define (stop? line) + (and (string? line) + (string-contains line "ACTION[Server]: singleplayer [127.0.0.1] joins game."))) + (let loop () + (match (read-line port) + ((? error? line) + (error "minetest raised an error: ~a" line)) + ((? stop?) + (kill pid SIGINT) + (close-port port) + (waitpid pid)) + ((? string? line) + (display line) + (newline) + (loop)) + ((? eof-object?) + (error "minetest didn't start")))))))) + +(define %standard-phases + (modify-phases gnu:%standard-phases + (delete 'bootstrap) + (delete 'configure) + (add-before 'build 'minimise-png minimise-png) + (delete 'build) + (delete 'check) + (replace 'install install) + ;; The 'check' phase requires the mod to be installed, + ;; so move the 'check' phase after the 'install' phase. + (add-after 'install 'check check))) + +;;; minetest-build-system.scm ends here diff --git a/guix/ci.scm b/guix/ci.scm index 6a3af8b42c..01b493b3af 100644 --- a/guix/ci.scm +++ b/guix/ci.scm @@ -59,6 +59,11 @@ job-status job-name + history? + history-evaluation + history-checkouts + history-jobs + %query-limit queued-builds latest-builds @@ -66,6 +71,7 @@ evaluation-jobs build job-build + jobs-history latest-evaluations evaluations-for-commit @@ -127,6 +133,18 @@ integer->build-status) (name job-name)) ;string +(define-json-mapping <history> make-history history? + json->history + (evaluation history-evaluation) ;integer + (checkouts history-checkouts "checkouts" ;<checkout>* + (lambda (checkouts) + (map json->checkout + (vector->list checkouts)))) + (jobs history-jobs "jobs" + (lambda (jobs) + (map json->job + (vector->list jobs))))) + (define-json-mapping <checkout> make-checkout checkout? json->checkout (commit checkout-commit) ;string (SHA1) @@ -247,8 +265,20 @@ found (404)." "Return the build associated with JOB." (build url (job-build-id job))) -;; TODO: job history: -;; https://ci.guix.gnu.org/api/jobs/history?spec=master&names=coreutils.x86_64-linux&nr=10 +(define* (jobs-history url jobs + #:key + (specification "master") + (limit 20)) + "Return the job history for the SPECIFICATION jobs which names are part of +the JOBS list, from the CI server at URL. Limit the history to the latest +LIMIT evaluations. " + (let ((names (string-join jobs ","))) + (map json->history + (vector->list + (json->scm + (http-fetch + (format #f "~a/api/jobs/history?spec=~a&names=~a&nr=~a" + url specification names (number->string limit)))))))) (define (find-latest-commit-with-substitutes url) "Return the latest commit with available substitutes for the Guix package diff --git a/guix/import/egg.scm b/guix/import/egg.scm index 86b54ff56f..75b7659944 100644 --- a/guix/import/egg.scm +++ b/guix/import/egg.scm @@ -88,7 +88,7 @@ (define (egg-source-url name version) "Return the URL to the source tarball for version VERSION of the CHICKEN egg NAME." - (string-append (%eggs-url) "/" name "/" name "-" version ".tar.gz")) + `(egg-uri ,name version)) (define (egg-name->guix-name name) "Return the package name for CHICKEN egg NAME." @@ -198,7 +198,8 @@ not work." (tarball (if source #f (with-store store - (download-to-store store source-url))))) + (download-to-store + store (egg-uri name version)))))) (define egg-home-page (string-append (%eggs-home-page) "/" name)) diff --git a/guix/import/go.scm b/guix/import/go.scm index 617a0d0e23..4755571209 100644 --- a/guix/import/go.scm +++ b/guix/import/go.scm @@ -6,6 +6,7 @@ ;;; Copyright © 2021 Ludovic Courtès <ludo@gnu.org> ;;; Copyright © 2021 Xinglu Chen <public@yoctocell.xyz> ;;; Copyright © 2021 Sarah Morgensen <iskarian@mgsn.dev> +;;; Copyright © 2021 Simon Tournier <zimon.toutoune@gmail.com> ;;; ;;; This file is part of GNU Guix. ;;; @@ -63,6 +64,7 @@ #:use-module (web uri) #:export (go-module->guix-package + go-module->guix-package* go-module-recursive-import)) ;;; Commentary: @@ -646,7 +648,28 @@ hint: use one of the following available versions ~a\n" dependencies+versions dependencies)))) -(define go-module->guix-package* (memoize go-module->guix-package)) +(define go-module->guix-package* + (lambda args + ;; Disable output buffering so that the following warning gets printed + ;; consistently. + (setvbuf (current-error-port) 'none) + (let ((package-name (match args ((name _ ...) name)))) + (guard (c ((http-get-error? c) + (warning (G_ "Failed to import package ~s. +reason: ~s could not be fetched: HTTP error ~a (~s). +This package and its dependencies won't be imported.~%") + package-name + (uri->string (http-get-error-uri c)) + (http-get-error-code c) + (http-get-error-reason c)) + (values #f '())) + (else + (warning (G_ "Failed to import package ~s. +reason: ~s.~%") + package-name + (exception-args c)) + (values #f '()))) + (apply go-module->guix-package args))))) (define* (go-module-recursive-import package-name #:key (goproxy "https://proxy.golang.org") @@ -656,23 +679,12 @@ hint: use one of the following available versions ~a\n" (recursive-import package-name #:repo->guix-package - (lambda* (name #:key version repo) - ;; Disable output buffering so that the following warning gets printed - ;; consistently. - (setvbuf (current-error-port) 'none) - (guard (c ((http-get-error? c) - (warning (G_ "Failed to import package ~s. -reason: ~s could not be fetched: HTTP error ~a (~s). -This package and its dependencies won't be imported.~%") - name - (uri->string (http-get-error-uri c)) - (http-get-error-code c) - (http-get-error-reason c)) - (values '() '()))) - (receive (package-sexp dependencies) - (go-module->guix-package* name #:goproxy goproxy - #:version version - #:pin-versions? pin-versions?) - (values package-sexp dependencies)))) + (memoize + (lambda* (name #:key version repo) + (receive (package-sexp dependencies) + (go-module->guix-package* name #:goproxy goproxy + #:version version + #:pin-versions? pin-versions?) + (values package-sexp dependencies)))) #:guix-name go-module->guix-package-name #:version version)) diff --git a/guix/import/minetest.scm b/guix/import/minetest.scm new file mode 100644 index 0000000000..e1f8487b75 --- /dev/null +++ b/guix/import/minetest.scm @@ -0,0 +1,456 @@ +;;; GNU Guix --- Functional package management for GNU +;;; Copyright © 2021 Maxime Devos <maximedevos@telenet.be> +;;; +;;; This file is part of GNU Guix. +;;; +;;; GNU Guix is free software; you can redistribute it and/or modify it +;;; under the terms of the GNU General Public License as published by +;;; the Free Software Foundation; either version 3 of the License, or (at +;;; your option) any later version. +;;; +;;; GNU Guix is distributed in the hope that it will be useful, but +;;; WITHOUT ANY WARRANTY; without even the implied warranty of +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;;; GNU General Public License for more details. +;;; +;;; You should have received a copy of the GNU General Public License +;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>. + +(define-module (guix import minetest) + #:use-module (ice-9 match) + #:use-module (ice-9 receive) + #:use-module (ice-9 threads) + #:use-module (ice-9 hash-table) + #:use-module (srfi srfi-1) + #:use-module (srfi srfi-2) + #:use-module (srfi srfi-11) + #:use-module (srfi srfi-26) + #:use-module (guix utils) + #:use-module (guix ui) + #:use-module (guix i18n) + #:use-module (guix memoization) + #:use-module (guix serialization) + #:use-module (guix import utils) + #:use-module (guix import json) + #:use-module ((gcrypt hash) #:select (open-sha256-port port-sha256)) + #:use-module (json) + #:use-module (guix base32) + #:use-module (guix git) + #:use-module (guix store) + #:export (%default-sort-key + %contentdb-api + json->package + contentdb-fetch + elaborate-contentdb-name + minetest->guix-package + minetest-recursive-import + sort-packages)) + +;; The ContentDB API is documented at +;; <https://content.minetest.net>. + +(define %contentdb-api + (make-parameter "https://content.minetest.net/api/")) + +(define (string-or-false x) + (and (string? x) x)) + +(define (natural-or-false x) + (and (exact-integer? x) (>= x 0) x)) + +;; Descriptions on ContentDB use carriage returns, but Guix doesn't. +(define (delete-cr text) + (string-delete #\cr text)) + + + +;;; +;;; JSON mappings +;;; + +;; Minetest package. +;; +;; API endpoint: /packages/AUTHOR/NAME/ +(define-json-mapping <package> make-package package? + json->package + (author package-author) ; string + (creation-date package-creation-date ; string + "created_at") + (downloads package-downloads) ; integer + (forums package-forums "forums" natural-or-false) + (issue-tracker package-issue-tracker "issue_tracker") ; string + (license package-license) ; string + (long-description package-long-description "long_description") ; string + (maintainers package-maintainers ; list of strings + "maintainers" vector->list) + (media-license package-media-license "media_license") ; string + (name package-name) ; string + (provides package-provides ; list of strings + "provides" vector->list) + (release package-release) ; integer + (repository package-repository "repo" string-or-false) + (score package-score) ; flonum + (screenshots package-screenshots "screenshots" vector->list) ; list of strings + (short-description package-short-description "short_description") ; string + (state package-state) ; string + (tags package-tags "tags" vector->list) ; list of strings + (thumbnail package-thumbnail) ; string + (title package-title) ; string + (type package-type) ; string + (url package-url) ; string + (website package-website "website" string-or-false)) + +(define-json-mapping <release> make-release release? + json->release + ;; If present, a git commit identified by its hash + (commit release-commit "commit" string-or-false) + (downloads release-downloads) ; integer + (id release-id) ; integer + (max-minetest-version release-max-minetest-version string-or-false) + (min-minetest-version release-min-minetest-version string-or-false) + (release-date release-data) ; string + (title release-title) ; string + (url release-url)) ; string + +(define-json-mapping <dependency> make-dependency dependency? + json->dependency + (optional? dependency-optional? "is_optional") ; bool + (name dependency-name) ; string + (packages dependency-packages "packages" vector->list)) ; list of strings + +;; A structure returned by the /api/packages/?fmt=keys endpoint +(define-json-mapping <package-keys> make-package-keys package-keys? + json->package-keys + (author package-keys-author) ; string + (name package-keys-name) ; string + (type package-keys-type)) ; string + +(define (package-mod? package) + "Is the ContentDB package PACKAGE a mod?" + ;; ContentDB also has ‘games’ and ‘texture packs’. + (string=? (package-type package) "mod")) + + + +;;; +;;; Manipulating names of packages +;;; +;;; There are three kind of names: +;;; +;;; * names of guix packages, e.g. minetest-basic-materials. +;;; * names of mods on ContentDB, e.g. basic_materials +;;; * a combination of author and mod name on ContentDB, e.g. VanessaE/basic_materials +;;; + +(define (%construct-full-name author name) + (string-append author "/" name)) + +(define (package-full-name package) + "Given a <package> object, return the corresponding AUTHOR/NAME string." + (%construct-full-name (package-author package) (package-name package))) + +(define (package-keys-full-name package) + "Given a <package-keys> object, return the corresponding AUTHOR/NAME string." + (%construct-full-name (package-keys-author package) + (package-keys-name package))) + +(define (contentdb->package-name author/name) + "Given the AUTHOR/NAME of a package on ContentDB, return a Guix-compliant +name for the package." + ;; The author is not included, as the names of popular mods + ;; tend to be unique. + (string-append "minetest-" (snake-case (author/name->name author/name)))) + +(define (author/name->name author/name) + "Extract NAME from the AUTHOR/NAME string, or raise an error if AUTHOR/NAME +is ill-formatted." + (match (string-split author/name #\/) + ((author name) + (when (string-null? author) + (leave + (G_ "In ~a: author names must consist of at least a single character.~%") + author/name)) + (when (string-null? name) + (leave + (G_ "In ~a: mod names must consist of at least a single character.~%") + author/name)) + name) + ((too many . components) + (leave + (G_ "In ~a: author names and mod names may not contain forward slashes.~%") + author/name)) + ((name) + (if (string-null? name) + (leave (G_ "mod names may not be empty.~%")) + (leave (G_ "The name of the author is missing in ~a.~%") + author/name))))) + +(define* (elaborate-contentdb-name name #:key (sort %default-sort-key)) + "If NAME is an AUTHOR/NAME string, return it. Otherwise, try to determine +the author and return an appropriate AUTHOR/NAME string. If that fails, +raise an exception." + (if (or (string-contains name "/") (string-null? name)) + ;; Call 'author/name->name' to verify that NAME seems reasonable + ;; and raise an appropriate exception if it isn't. + (begin + (author/name->name name) + name) + (let* ((package-keys (contentdb-query-packages name #:sort sort)) + (correctly-named + (filter (lambda (package-key) + (string=? name (package-keys-name package-key))) + package-keys))) + (match correctly-named + ((one) (package-keys-full-name one)) + ((too . many) + (warning (G_ "~a is ambigious, presuming ~a (other options include: ~a)~%") + name (package-keys-full-name too) + (map package-keys-full-name many)) + (package-keys-full-name too)) + (() + (leave (G_ "No mods with name ~a were found.~%") name)))))) + + + +;;; +;;; API endpoints +;;; + +(define contentdb-fetch + (mlambda (author/name) + "Return a <package> record for package AUTHOR/NAME, or #f on failure." + (and=> (json-fetch + (string-append (%contentdb-api) "packages/" author/name "/")) + json->package))) + +(define (contentdb-fetch-releases author/name) + "Return a list of <release> records for package NAME by AUTHOR, or #f +on failure." + (and=> (json-fetch (string-append (%contentdb-api) "packages/" author/name + "/releases/")) + (lambda (json) + (map json->release (vector->list json))))) + +(define (latest-release author/name) + "Return the latest source release for package NAME by AUTHOR, +or #f if this package does not exist." + (and=> (contentdb-fetch-releases author/name) + car)) + +(define (contentdb-fetch-dependencies author/name) + "Return an alist of lists of <dependency> records for package NAME by AUTHOR +and possibly some other packages as well, or #f on failure." + (define url (string-append (%contentdb-api) "packages/" author/name + "/dependencies/")) + (and=> (json-fetch url) + (lambda (json) + (map (match-lambda + ((key . value) + (cons key (map json->dependency (vector->list value))))) + json)))) + +(define* (contentdb-query-packages q #:key + (type "mod") + (limit 50) + (sort %default-sort-key) + (order "desc")) + "Search ContentDB for Q (a string). Sort by SORT, in ascending order +if ORDER is \"asc\" or descending order if ORDER is \"desc\". TYPE must +be \"mod\", \"game\" or \"txp\", restricting thes search results to +respectively mods, games and texture packs. Limit to at most LIMIT +results. The return value is a list of <package-keys> records." + ;; XXX does Guile have something for constructing (and, when necessary, + ;; escaping) query strings? + (define url (string-append (%contentdb-api) "packages/?type=" type + "&q=" q "&fmt=keys" + "&limit=" (number->string limit) + "&order=" order + "&sort=" sort)) + (let ((json (json-fetch url))) + (if json + (map json->package-keys (vector->list json)) + (leave + (G_ "The package search API doesn't exist anymore.~%"))))) + + + +;; XXX copied from (guix import elpa) +(define* (download-git-repository url ref) + "Fetch the given REF from the Git repository at URL." + (with-store store + (latest-repository-commit store url #:ref ref))) + +;; XXX adapted from (guix scripts hash) +(define (file-hash file) + "Compute the hash of FILE." + (let-values (((port get-hash) (open-sha256-port))) + (write-file file port) + (force-output port) + (get-hash))) + +(define (make-minetest-sexp author/name version repository commit + inputs home-page synopsis + description media-license license) + "Return a S-expression for the minetest package with the given author/NAME, +VERSION, REPOSITORY, COMMIT, INPUTS, HOME-PAGE, SYNOPSIS, DESCRIPTION, +MEDIA-LICENSE and LICENSE." + `(package + (name ,(contentdb->package-name author/name)) + (version ,version) + (source + (origin + (method git-fetch) + (uri (git-reference + (url ,repository) + (commit ,commit))) + (sha256 + (base32 + ;; The git commit is not always available. + ,(and commit + (bytevector->nix-base32-string + (file-hash + (download-git-repository repository + `(commit . ,commit))))))) + (file-name (git-file-name name version)))) + (build-system minetest-mod-build-system) + ,@(maybe-propagated-inputs (map contentdb->package-name inputs)) + (home-page ,home-page) + (synopsis ,(delete-cr synopsis)) + (description ,(delete-cr description)) + (license ,(if (eq? media-license license) + license + `(list ,media-license ,license))) + ;; The Minetest updater (not yet in Guix; it requires not-yet-submitted + ;; patches to (guix upstream) that require some work) needs to know both + ;; the author name and mod name for efficiency. + (properties ,(list 'quasiquote `((upstream-name . ,author/name)))))) + +(define (package-home-page package) + "Guess the home page of the ContentDB package PACKAGE. + +In order of preference, try the 'website', the forum topic on the +official Minetest forum and the Git repository (if any)." + (define (topic->url-sexp topic) + ;; 'minetest-topic' is a procedure defined in (gnu packages minetest) + `(minetest-topic ,topic)) + (or (package-website package) + (and=> (package-forums package) topic->url-sexp) + (package-repository package))) + +;; If the default sort key is changed, make sure to modify 'show-help' +;; in (guix scripts import minetest) appropriately as well. +(define %default-sort-key "score") + +(define* (sort-packages packages #:key (sort %default-sort-key)) + "Sort PACKAGES by SORT, in descending order." + (define package->key + (match sort + ("score" package-score) + ("downloads" package-downloads))) + (define (greater x y) + (> (package->key x) (package->key y))) + (sort-list packages greater)) + +(define builtin-mod? + (let ((%builtin-mods + (alist->hash-table + (map (lambda (x) (cons x #t)) + '("beds" "binoculars" "boats" "bones" "bucket" "butterflies" + "carts" "creative" "default" "doors" "dungeon_loot" "dye" + "env_sounds" "farming" "fire" "fireflies" "flowers" + "game_commands" "give_initial_stuff" "map" "mtg_craftguide" + "player_api" "screwdriver" "sethome" "sfinv" "spawn" "stairs" + "tnt" "vessels" "walls" "weather" "wool" "xpanes"))))) + (lambda (mod) + "Is MOD provided by the default minetest subgame?" + (hash-ref %builtin-mods mod)))) + +(define* (important-dependencies dependencies author/name + #:key (sort %default-sort-key)) + "Return the hard dependencies of AUTHOR/NAME in the association list +DEPENDENCIES as a list of AUTHOR/NAME strings." + (define dependency-list + (assoc-ref dependencies author/name)) + (filter-map + (lambda (dependency) + (and (not (dependency-optional? dependency)) + (not (builtin-mod? (dependency-name dependency))) + ;; The dependency information contains symbolic names + ;; that can be ‘provided’ by multiple mods, so we need to choose one + ;; of the implementations. + (let* ((implementations + (par-map contentdb-fetch (dependency-packages dependency))) + ;; Fetching package information about the packages is racy: + ;; some packages might be removed from ContentDB between the + ;; construction of DEPENDENCIES and the call to + ;; 'contentdb-fetch'. So filter out #f. + ;; + ;; Filter out ‘games’ that include the requested mod -- it's + ;; the mod itself we want. + (mods (filter (lambda (p) (and=> p package-mod?)) + implementations)) + (sorted-mods (sort-packages mods #:sort sort))) + (match sorted-mods + ((package) (package-full-name package)) + ((too . many) + (warning + (G_ "The dependency ~a of ~a has multiple different implementations ~a.~%") + (dependency-name dependency) + author/name + (map package-full-name sorted-mods)) + (match sort + ("score" + (warning + (G_ "The implementation with the highest score will be choosen!~%"))) + ("downloads" + (warning + (G_ "The implementation that has been downloaded the most will be choosen!~%")))) + (package-full-name too)) + (() + (warning + (G_ "The dependency ~a of ~a does not have any implementation. It will be ignored!~%") + (dependency-name dependency) author/name) + #f))))) + dependency-list)) + +(define* (%minetest->guix-package author/name #:key (sort %default-sort-key)) + "Fetch the metadata for AUTHOR/NAME from https://content.minetest.net, and +return the 'package' S-expression corresponding to that package, or raise an +exception on failure. On success, also return the upstream dependencies as a +list of AUTHOR/NAME strings." + ;; Call 'author/name->name' to verify that AUTHOR/NAME seems reasonable. + (author/name->name author/name) + (define package (contentdb-fetch author/name)) + (unless package + (leave (G_ "no package metadata for ~a on ContentDB~%") author/name)) + (define dependencies (contentdb-fetch-dependencies author/name)) + (unless dependencies + (leave (G_ "no dependency information for ~a on ContentDB~%") author/name)) + (define release (latest-release author/name)) + (unless release + (leave (G_ "no release of ~a on ContentDB~%") author/name)) + (define important-upstream-dependencies + (important-dependencies dependencies author/name #:sort sort)) + (values (make-minetest-sexp author/name + (release-title release) ; version + (package-repository package) + (release-commit release) + important-upstream-dependencies + (package-home-page package) + (package-short-description package) + (package-long-description package) + (spdx-string->license + (package-media-license package)) + (spdx-string->license + (package-license package))) + important-upstream-dependencies)) + +(define minetest->guix-package + (memoize %minetest->guix-package)) + +(define* (minetest-recursive-import author/name #:key (sort %default-sort-key)) + (define* (minetest->guix-package* author/name #:key repo version) + (minetest->guix-package author/name #:sort sort)) + (recursive-import author/name + #:repo->guix-package minetest->guix-package* + #:guix-name contentdb->package-name)) diff --git a/guix/import/opam.scm b/guix/import/opam.scm index c42b608ec9..f8402ff5ba 100644 --- a/guix/import/opam.scm +++ b/guix/import/opam.scm @@ -3,6 +3,7 @@ ;;; Copyright © 2020 Martin Becze <mjbecze@riseup.net> ;;; Copyright © 2021 Xinglu Chen <public@yoctocell.xyz> ;;; Copyright © 2021 Sarah Morgensen <iskarian@mgsn.dev> +;;; Copyright © 2021 Alice Brenon <alice.brenon@ens-lyon.fr> ;;; ;;; This file is part of GNU Guix. ;;; @@ -23,21 +24,24 @@ #:use-module (ice-9 ftw) #:use-module (ice-9 match) #:use-module (ice-9 peg) + #:use-module ((ice-9 popen) #:select (open-pipe*)) #:use-module (ice-9 receive) - #:use-module ((ice-9 rdelim) #:select (read-line)) #:use-module (ice-9 textual-ports) #:use-module (ice-9 vlist) #:use-module (srfi srfi-1) #:use-module (srfi srfi-2) - #:use-module (web uri) + #:use-module ((srfi srfi-26) #:select (cut)) + #:use-module ((web uri) #:select (string->uri uri->string)) + #:use-module ((guix build utils) #:select (dump-port find-files mkdir-p)) #:use-module (guix build-system) #:use-module (guix build-system ocaml) #:use-module (guix http-client) - #:use-module (guix git) #:use-module (guix ui) #:use-module (guix packages) #:use-module (guix upstream) - #:use-module (guix utils) + #:use-module ((guix utils) #:select (cache-directory + version>? + call-with-temporary-output-file)) #:use-module (guix import utils) #:use-module ((guix licenses) #:prefix license:) #:export (opam->guix-package @@ -122,51 +126,83 @@ (define-peg-pattern condition-string all (and QUOTE (* STRCHR) QUOTE)) (define-peg-pattern condition-var all (+ (or (range #\a #\z) "-" ":"))) -(define* (get-opam-repository #:optional repo) +(define (opam-cache-directory path) + (string-append (cache-directory) "/opam/" path)) + +(define known-repositories + '((opam . "https://opam.ocaml.org") + (coq . "https://coq.inria.fr/opam/released") + (coq-released . "https://coq.inria.fr/opam/released") + (coq-core-dev . "https://coq.inria.fr/opam/core-dev") + (coq-extra-dev . "https://coq.inria.fr/opam/extra-dev") + (grew . "http://opam.grew.fr"))) + +(define (get-uri repo-root) + (let ((archive-file (string-append repo-root "/index.tar.gz"))) + (or (string->uri archive-file) + (begin + (warning (G_ "'~a' is not a valid URI~%") archive-file) + 'bad-repo)))) + +(define (repo-type repo) + (match (assoc-ref known-repositories (string->symbol repo)) + (#f (if (file-exists? repo) + `(local ,repo) + `(remote ,(get-uri repo)))) + (url `(remote ,(get-uri url))))) + +(define (update-repository input) + "Make sure the cache for opam repository INPUT is up-to-date" + (let* ((output (opam-cache-directory (basename (port-filename input)))) + (cached-date (if (file-exists? output) + (stat:mtime (stat output)) + (begin (mkdir-p output) 0)))) + (when (> (stat:mtime (stat input)) cached-date) + (call-with-port + (open-pipe* OPEN_WRITE "tar" "xz" "-C" output "-f" "-") + (cut dump-port input <>))) + output)) + +(define* (get-opam-repository #:optional (repo "opam")) "Update or fetch the latest version of the opam repository and return the path to the repository." - (let ((url (cond - ((or (not repo) (equal? repo 'opam)) - "https://github.com/ocaml/opam-repository") - ((string-prefix? "coq-" (symbol->string repo)) - "https://github.com/coq/opam-coq-archive") - ((equal? repo 'coq) "https://github.com/coq/opam-coq-archive") - (else (throw 'unknown-repository repo))))) - (receive (location commit _) - (update-cached-checkout url) - (cond - ((or (not repo) (equal? repo 'opam)) - location) - ((equal? repo 'coq) - (string-append location "/released")) - ((string-prefix? "coq-" (symbol->string repo)) - (string-append location "/" (substring (symbol->string repo) 4))) - (else location))))) + (match (repo-type repo) + (('local p) p) + (('remote 'bad-repo) #f) ; to weed it out during filter-map in opam-fetch + (('remote r) (call-with-port (http-fetch/cached r) update-repository)))) ;; Prevent Guile 3 from inlining this procedure so we can mock it in tests. (set! get-opam-repository get-opam-repository) -(define (latest-version versions) - "Find the most recent version from a list of versions." - (fold (lambda (a b) (if (version>? a b) a b)) (car versions) versions)) +(define (get-version-and-file path) + "Analyse a candidate path and return an list containing information for proper + version comparison as well as the source path for metadata." + (and-let* ((metadata-file (string-append path "/opam")) + (filename (basename path)) + (version (string-join (cdr (string-split filename #\.)) "."))) + (and (file-exists? metadata-file) + (eq? 'regular (stat:type (stat metadata-file))) + (if (string-prefix? "v" version) + `(V ,(substring version 1) ,metadata-file) + `(digits ,version ,metadata-file))))) + +(define (keep-max-version a b) + "Version comparison on the lists returned by the previous function taking the + janestreet re-versioning into account (v-prefixed come first)." + (match (cons a b) + ((('V va _) . ('V vb _)) (if (version>? va vb) a b)) + ((('V _ _) . _) a) + ((_ . ('V _ _)) b) + ((('digits va _) . ('digits vb _)) (if (version>? va vb) a b)))) (define (find-latest-version package repository) "Get the latest version of a package as described in the given repository." - (let* ((dir (string-append repository "/packages/" package)) - (versions (scandir dir (lambda (name) (not (string-prefix? "." name)))))) - (if versions - (let ((versions (map - (lambda (dir) - (string-join (cdr (string-split dir #\.)) ".")) - versions))) - ;; Workaround for janestreet re-versionning - (let ((v-versions (filter (lambda (version) (string-prefix? "v" version)) versions))) - (if (null? v-versions) - (latest-version versions) - (string-append "v" (latest-version (map (lambda (version) (substring version 1)) v-versions)))))) - (begin - (format #t (G_ "Package not found in opam repository: ~a~%") package) - #f)))) + (let ((packages (string-append repository "/packages")) + (filter (make-regexp (string-append "^" package "\\.")))) + (reduce keep-max-version #f + (filter-map + get-version-and-file + (find-files packages filter #:directories? #t))))) (define (get-metadata opam-file) (with-input-from-file opam-file @@ -267,26 +303,28 @@ path to the repository." (define (depends->native-inputs depends) (filter (lambda (name) (not (equal? "" name))) - (map dependency->native-input depends))) + (map dependency->native-input depends))) (define (dependency-list->inputs lst) (map string->symbol (ocaml-names->guix-names lst))) -(define* (opam-fetch name #:optional (repository (get-opam-repository))) - (and-let* ((repository repository) - (version (find-latest-version name repository)) - (file (string-append repository "/packages/" name "/" name "." version "/opam"))) - `(("metadata" ,@(get-metadata file)) - ("version" . ,(if (string-prefix? "v" version) - (substring version 1) - version))))) +(define* (opam-fetch name #:optional (repositories-specs '("opam"))) + (or (fold (lambda (repository others) + (match (find-latest-version name repository) + ((_ version file) `(("metadata" ,@(get-metadata file)) + ("version" . ,version))) + (_ others))) + #f + (filter-map get-opam-repository repositories-specs)) + (leave (G_ "package '~a' not found~%") name))) (define* (opam->guix-package name #:key (repo 'opam) version) "Import OPAM package NAME from REPOSITORY (a directory name) or, if REPOSITORY is #f, from the official OPAM repository. Return a 'package' sexp or #f on failure." - (and-let* ((opam-file (opam-fetch name (get-opam-repository repo))) + (and-let* ((with-opam (if (member "opam" repo) repo (cons "opam" repo))) + (opam-file (opam-fetch name with-opam)) (version (assoc-ref opam-file "version")) (opam-content (assoc-ref opam-file "metadata")) (url-dict (metadata-ref opam-content "url")) @@ -311,9 +349,7 @@ or #f on failure." (values `(package (name ,(ocaml-name->guix-name name)) - (version ,(if (string-prefix? "v" version) - (substring version 1) - version)) + (version ,version) (source (origin (method url-fetch) diff --git a/guix/import/utils.scm b/guix/import/utils.scm index d817318a91..a180742ca3 100644 --- a/guix/import/utils.scm +++ b/guix/import/utils.scm @@ -8,6 +8,7 @@ ;;; Copyright © 2020 Helio Machado <0x2b3bfa0+guix@googlemail.com> ;;; Copyright © 2020 Martin Becze <mjbecze@riseup.net> ;;; Copyright © 2021 Maxim Cournoyer <maxim.cournoyer@gmail.com> +;;; Copyright © 2021 Sarah Morgensen <iskarian@mgsn.dev> ;;; ;;; This file is part of GNU Guix. ;;; @@ -133,8 +134,14 @@ of the string VERSION is replaced by the symbol 'version." ;; Please update guix/licenses.scm when modifying ;; this list to avoid mismatches. (match str + ;; "GPL-N+" has been deprecated in favour of "GPL-N-or-later". + ;; "GPL-N" has been deprecated in favour of "GPL-N-only" + ;; or "GPL-N-or-later" as appropriate. Likewise for LGPL + ;; and AGPL ("AGPL-1.0" 'license:agpl1) ("AGPL-3.0" 'license:agpl3) + ("AGPL-3.0-only" 'license:agpl3) + ("AGPL-3.0-or-later" 'license:agpl3+) ("Apache-1.1" 'license:asl1.1) ("Apache-2.0" 'license:asl2.0) ("BSL-1.0" 'license:boost1.0) @@ -161,11 +168,17 @@ of the string VERSION is replaced by the symbol 'version." ("GFDL-1.3" 'license:fdl1.3+) ("Giftware" 'license:giftware) ("GPL-1.0" 'license:gpl1) + ("GPL-1.0-only" 'license:gpl1) ("GPL-1.0+" 'license:gpl1+) + ("GPL-1.0-or-later" 'license:gpl1+) ("GPL-2.0" 'license:gpl2) + ("GPL-2.0-only" 'license:gpl2) ("GPL-2.0+" 'license:gpl2+) + ("GPL-2.0-or-later" 'license:gpl2+) ("GPL-3.0" 'license:gpl3) + ("GPL-3.0-only" 'license:gpl3) ("GPL-3.0+" 'license:gpl3+) + ("GPL-3.0-or-later" 'license:gpl3+) ("ISC" 'license:isc) ("IJG" 'license:ijg) ("Imlib2" 'license:imlib2) @@ -173,11 +186,17 @@ of the string VERSION is replaced by the symbol 'version." ("IPL-1.0" 'license:ibmpl1.0) ("LAL-1.3" 'license:lal1.3) ("LGPL-2.0" 'license:lgpl2.0) + ("LGPL-2.0-only" 'license:lgpl2.0) ("LGPL-2.0+" 'license:lgpl2.0+) + ("LGPL-2.0-or-later" 'license:lgpl2.0+) ("LGPL-2.1" 'license:lgpl2.1) + ("LGPL-2.1-only" 'license:lgpl2.1) ("LGPL-2.1+" 'license:lgpl2.1+) + ("LGPL-2.1-or-later" 'license:lgpl2.1+) ("LGPL-3.0" 'license:lgpl3) + ("LGPL-3.0-only" 'license:lgpl3) ("LGPL-3.0+" 'license:lgpl3+) + ("LGPL-3.0-or-later" 'license:lgpl3+) ("MPL-1.0" 'license:mpl1.0) ("MPL-1.1" 'license:mpl1.1) ("MPL-2.0" 'license:mpl2.0) @@ -471,15 +490,16 @@ to obtain the Guix package name corresponding to the upstream name." (name (list name #f))) dependencies))) (make-node name version package normalized-deps))) - (map node-package - (topological-sort (list (lookup-node package-name version)) - (lambda (node) - (map (lambda (name-version) - (apply lookup-node name-version)) - (remove (lambda (name-version) - (apply exists? name-version)) - (node-dependencies node)))) - (lambda (node) - (string-append - (node-name node) - (or (node-version node) "")))))) + (filter-map + node-package + (topological-sort (list (lookup-node package-name version)) + (lambda (node) + (map (lambda (name-version) + (apply lookup-node name-version)) + (remove (lambda (name-version) + (apply exists? name-version)) + (node-dependencies node)))) + (lambda (node) + (string-append + (node-name node) + (or (node-version node) "")))))) diff --git a/guix/scripts/import.scm b/guix/scripts/import.scm index f53d1ac1f4..b369a362d0 100644 --- a/guix/scripts/import.scm +++ b/guix/scripts/import.scm @@ -77,7 +77,8 @@ rather than \\n." ;;; (define importers '("gnu" "pypi" "cpan" "hackage" "stackage" "egg" "elpa" - "gem" "go" "cran" "crate" "texlive" "json" "opam")) + "gem" "go" "cran" "crate" "texlive" "json" "opam" + "minetest")) (define (resolve-importer name) (let ((module (resolve-interface diff --git a/guix/scripts/import/gem.scm b/guix/scripts/import/gem.scm index 65d2bf10b4..328d20b946 100644 --- a/guix/scripts/import/gem.scm +++ b/guix/scripts/import/gem.scm @@ -2,6 +2,7 @@ ;;; Copyright © 2015 David Thompson <davet@gnu.org> ;;; Copyright © 2018 Oleg Pykhalov <go.wigust@gmail.com> ;;; Copyright © 2021 Sarah Morgensen <iskarian@mgsn.dev> +;;; Copyright © 2021 Brice Waegeneire <brice@waegenei.re> ;;; ;;; This file is part of GNU Guix. ;;; @@ -60,7 +61,7 @@ Import and convert the RubyGems package for PACKAGE-NAME.\n")) (exit 0))) (option '(#\V "version") #f #f (lambda args - (show-version-and-exit "guix import pypi"))) + (show-version-and-exit "guix import gem"))) (option '(#\r "recursive") #f #f (lambda (opt name arg result) (alist-cons 'recursive #t result))) diff --git a/guix/scripts/import/go.scm b/guix/scripts/import/go.scm index e08a1e427e..f5cfea8683 100644 --- a/guix/scripts/import/go.scm +++ b/guix/scripts/import/go.scm @@ -112,10 +112,10 @@ that are not yet in Guix")) (map package->definition* (apply go-module-recursive-import arguments)) ;; Single import. - (let ((sexp (apply go-module->guix-package arguments))) + (let ((sexp (apply go-module->guix-package* arguments))) (unless sexp - (leave (G_ "failed to download meta-data for module '~a'~%") - module-name)) + (leave (G_ "failed to download meta-data for module '~a'.~%") + name)) (package->definition* sexp)))))) (() (leave (G_ "too few arguments~%"))) diff --git a/guix/scripts/import/minetest.scm b/guix/scripts/import/minetest.scm new file mode 100644 index 0000000000..5f204d90fc --- /dev/null +++ b/guix/scripts/import/minetest.scm @@ -0,0 +1,117 @@ +;;; GNU Guix --- Functional package management for GNU +;;; Copyright © 2014 David Thompson <davet@gnu.org> +;;; Copyright © 2018 Ricardo Wurmus <rekado@elephly.net> +;;; Copyright © 2021 Maxime Devos <maximedevos@telenet.be> +;;; +;;; This file is part of GNU Guix. +;;; +;;; GNU Guix is free software; you can redistribute it and/or modify it +;;; under the terms of the GNU General Public License as published by +;;; the Free Software Foundation; either version 3 of the License, or (at +;;; your option) any later version. +;;; +;;; GNU Guix is distributed in the hope that it will be useful, but +;;; WITHOUT ANY WARRANTY; without even the implied warranty of +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;;; GNU General Public License for more details. +;;; +;;; You should have received a copy of the GNU General Public License +;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>. + +(define-module (guix scripts import minetest) + #:use-module (guix ui) + #:use-module (guix utils) + #:use-module (guix scripts) + #:use-module (guix import minetest) + #:use-module (guix import utils) + #:use-module (guix scripts import) + #:use-module (srfi srfi-1) + #:use-module (srfi srfi-11) + #:use-module (srfi srfi-37) + #:use-module (ice-9 match) + #:use-module (ice-9 format) + #:export (guix-import-minetest)) + + +;;; +;;; Command-line options. +;;; + +(define %default-options + `((sort . ,%default-sort-key))) + +(define (show-help) + (display (G_ "Usage: guix import minetest AUTHOR/NAME +Import and convert the Minetest mod NAME by AUTHOR from ContentDB.\n")) + (display (G_ " + -h, --help display this help and exit")) + (display (G_ " + -r, --recursive import packages recursively")) + (display (G_ " + -V, --version display version information and exit")) + (display (G_ " + --sort=KEY when choosing between multiple implementations, + choose the one with the highest value for KEY + (one of \"score\" (standard) or \"downloads\")")) + (newline) + (show-bug-report-information)) + +(define (verify-sort-order sort) + "Verify SORT can be used to sort mods by." + (unless (member sort '("score" "downloads" "reviews")) + (leave (G_ "~a: not a valid key to sort by~%") sort)) + sort) + +(define %options + ;; Specification of the command-line options. + (cons* (option '(#\h "help") #f #f + (lambda args + (show-help) + (exit 0))) + (option '(#\V "version") #f #f + (lambda args + (show-version-and-exit "guix import minetest"))) + (option '(#\r "recursive") #f #f + (lambda (opt name arg result) + (alist-cons 'recursive #t result))) + (option '("sort") #t #f + (lambda (opt name arg result) + (alist-cons 'sort (verify-sort-order arg) result))) + %standard-import-options)) + + +;;; +;;; Entry point. +;;; + +(define (guix-import-minetest . args) + (define (parse-options) + ;; Return the alist of option values. + (args-fold* args %options + (lambda (opt name arg result) + (leave (G_ "~A: unrecognized option~%") name)) + (lambda (arg result) + (alist-cons 'argument arg result)) + %default-options)) + + (let* ((opts (parse-options)) + (args (filter-map (match-lambda + (('argument . value) + value) + (_ #f)) + (reverse opts)))) + (match args + ((name) + (with-error-handling + (let* ((sort (assoc-ref opts 'sort)) + (author/name (elaborate-contentdb-name name #:sort sort))) + (if (assoc-ref opts 'recursive) + ;; Recursive import + (filter-map package->definition + (minetest-recursive-import author/name #:sort sort)) + ;; Single import + (minetest->guix-package author/name #:sort sort))))) + (() + (leave (G_ "too few arguments~%"))) + ((many ...) + (leave (G_ "too many arguments~%")))))) diff --git a/guix/scripts/import/opam.scm b/guix/scripts/import/opam.scm index 64164e7cc4..834ac34cb0 100644 --- a/guix/scripts/import/opam.scm +++ b/guix/scripts/import/opam.scm @@ -1,6 +1,7 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2018 Julien Lepiller <julien@lepiller.eu> ;;; Copyright © 2021 Sarah Morgensen <iskarian@mgsn.dev> +;;; Copyright © 2021 Alice Brenon <alice.brenon@ens-lyon.fr> ;;; ;;; This file is part of GNU Guix. ;;; @@ -46,7 +47,8 @@ Import and convert the opam package for PACKAGE-NAME.\n")) (display (G_ " -r, --recursive import packages recursively")) (display (G_ " - --repo import packages from this opam repository")) + --repo import packages from this opam repository (name, URL or local path) + can be used more than once")) (display (G_ " -V, --version display version information and exit")) (newline) @@ -81,7 +83,9 @@ Import and convert the opam package for PACKAGE-NAME.\n")) #:build-options? #f)) (let* ((opts (parse-options)) - (repo (and=> (assoc-ref opts 'repo) string->symbol)) + (repo (filter-map (match-lambda + (('repo . name) name) + (_ #f)) opts)) (args (filter-map (match-lambda (('argument . value) value) diff --git a/guix/scripts/publish.scm b/guix/scripts/publish.scm index 913cbd4fda..25846b7dc2 100644 --- a/guix/scripts/publish.scm +++ b/guix/scripts/publish.scm @@ -4,6 +4,7 @@ ;;; Copyright © 2015, 2016, 2017, 2018, 2019, 2020, 2021 Ludovic Courtès <ludo@gnu.org> ;;; Copyright © 2020 Maxim Cournoyer <maxim.cournoyer@gmail.com> ;;; Copyright © 2021 Simon Tournier <zimon.toutoune@gmail.com> +;;; Copyright © 2021 Mathieu Othacehe <othacehe@gnu.org> ;;; ;;; This file is part of GNU Guix. ;;; @@ -358,23 +359,13 @@ References: ~a~%" compression))) compressions) hash size references)) - ;; Do not render a "Deriver" or "System" line if we are rendering - ;; info for a derivation. + ;; Do not render a "Deriver" line if we are rendering info for a + ;; derivation. Also do not render a "System" line that would be + ;; expensive to compute and is currently unused. (info (if (not deriver) base-info - (catch 'system-error - (lambda () - (let ((drv (read-derivation-from-file deriver))) - (format #f "~aSystem: ~a~%Deriver: ~a~%" - base-info (derivation-system drv) - (basename deriver)))) - (lambda args - ;; DERIVER might be missing, but that's fine: - ;; it's only used for <substitutable> where it's - ;; optional. 'System' is currently unused. - (if (= ENOENT (system-error-errno args)) - base-info - (apply throw args)))))) + (format #f "~aDeriver: ~a~%" + base-info (basename deriver)))) (signature (base64-encode-string (canonical-sexp->string (signed-string info))))) (format #f "~aSignature: 1;~a;~a~%" info (gethostname) signature))) diff --git a/guix/scripts/system.scm b/guix/scripts/system.scm index 40401d7e03..83bbefd3dc 100644 --- a/guix/scripts/system.scm +++ b/guix/scripts/system.scm @@ -253,7 +253,7 @@ the ownership of '~a' may be incorrect!~%") #:target target) (return (info (G_ "bootloader successfully installed on '~a'~%") - (bootloader-configuration-target bootloader)))))))) + (bootloader-configuration-targets bootloader)))))))) ;;; @@ -768,14 +768,13 @@ and TARGET arguments." skip-safety-checks? install-bootloader? dry-run? derivations-only? - use-substitutes? bootloader-target target + use-substitutes? target full-boot? container-shared-network? (mappings '()) (gc-root #f)) "Perform ACTION for IMAGE. INSTALL-BOOTLOADER? specifies whether to install -bootloader; BOOTLOADER-TAGET is the target for the bootloader; TARGET is the -target root directory. +bootloader; TARGET is the target root directory. FULL-BOOT? is used for the 'vm' action; it determines whether to boot directly to the kernel or to the bootloader. CONTAINER-SHARED-NETWORK? @@ -856,13 +855,13 @@ static checks." #:target (or target "/")) (return (info (G_ "bootloader successfully installed on '~a'~%") - (bootloader-configuration-target bootloader)))) + (bootloader-configuration-targets bootloader)))) (with-shepherd-error-handling - (upgrade-shepherd-services local-eval os) - (return (format #t (G_ "\ + (upgrade-shepherd-services local-eval os) + (return (format #t (G_ "\ To complete the upgrade, run 'herd restart SERVICE' to stop, upgrade, and restart each service that was not automatically restarted.\n"))) - (return (format #t (G_ "\ + (return (format #t (G_ "\ Run 'herd status' to view the list of services on your system.\n")))))) ((init) (newline) @@ -1218,9 +1217,9 @@ resulting from command-line parsing." (target-file (match args ((first second) second) (_ #f))) - (bootloader-target + (bootloader-targets (and bootloader? - (bootloader-configuration-target + (bootloader-configuration-targets (operating-system-bootloader os))))) (define (graph-backend) @@ -1269,7 +1268,6 @@ resulting from command-line parsing." opts) #:install-bootloader? bootloader? #:target target-file - #:bootloader-target bootloader-target #:gc-root (assoc-ref opts 'gc-root))))) #:target target #:system system))) diff --git a/guix/scripts/system/reconfigure.scm b/guix/scripts/system/reconfigure.scm index 49da6ecb16..bf23fb06af 100644 --- a/guix/scripts/system/reconfigure.scm +++ b/guix/scripts/system/reconfigure.scm @@ -207,10 +207,10 @@ services as defined by OS." (define (install-bootloader-program installer disk-installer bootloader-package bootcfg - bootcfg-file device target) + bootcfg-file devices target) "Return an executable store item that, upon being evaluated, will install -BOOTCFG to BOOTCFG-FILE, a target file name, on DEVICE, a file system device, -at TARGET, a mount point, and subsequently run INSTALLER from +BOOTCFG to BOOTCFG-FILE, a target file name, on DEVICES, a list of file system +devices, at TARGET, a mount point, and subsequently run INSTALLER from BOOTLOADER-PACKAGE." (program-file "install-bootloader.scm" @@ -254,11 +254,17 @@ BOOTLOADER-PACKAGE." ;; The bootloader might not support installation on a ;; mounted directory using the BOOTLOADER-INSTALLER ;; procedure. In that case, fallback to installing the - ;; bootloader directly on DEVICE using the + ;; bootloader directly on DEVICES using the ;; BOOTLOADER-DISK-IMAGE-INSTALLER procedure. (if #$installer - (#$installer #$bootloader-package #$device #$target) - (#$disk-installer #$bootloader-package 0 #$device))) + (for-each (lambda (device) + (#$installer #$bootloader-package device + #$target)) + '#$devices) + (for-each (lambda (device) + (#$disk-installer #$bootloader-package + 0 device)) + '#$devices))) (lambda args (delete-file new-gc-root) (match args @@ -284,7 +290,7 @@ additional configurations specified by MENU-ENTRIES can be selected." (disk-installer (and run-installer? (bootloader-disk-image-installer bootloader))) (package (bootloader-package bootloader)) - (device (bootloader-configuration-target configuration)) + (devices (bootloader-configuration-targets configuration)) (bootcfg-file (bootloader-configuration-file bootloader))) (eval #~(parameterize ((current-warning-port (%make-void-port "w"))) (primitive-load #$(install-bootloader-program installer @@ -292,7 +298,7 @@ additional configurations specified by MENU-ENTRIES can be selected." package bootcfg bootcfg-file - device + devices target)))))) diff --git a/guix/scripts/weather.scm b/guix/scripts/weather.scm index 06312d65a2..60a697d1ac 100644 --- a/guix/scripts/weather.scm +++ b/guix/scripts/weather.scm @@ -54,16 +54,18 @@ (define (all-packages) "Return the list of public packages we are going to query." - (fold-packages (lambda (package result) - (match (package-replacement package) - ((? package? replacement) - (cons* replacement package result)) - (#f - (cons package result)))) - '() - - ;; Dismiss deprecated packages but keep hidden packages. - #:select? (negate package-superseded))) + (delete-duplicates + (fold-packages (lambda (package result) + (match (package-replacement package) + ((? package? replacement) + (cons* replacement package result)) + (#f + (cons package result)))) + '() + + ;; Dismiss deprecated packages but keep hidden packages. + #:select? (negate package-superseded)) + eq?)) (define (call-with-progress-reporter reporter proc) "This is a variant of 'call-with-progress-reporter' that works with monadic diff --git a/guix/swh.scm b/guix/swh.scm index b5c800011d..922d781a7b 100644 --- a/guix/swh.scm +++ b/guix/swh.scm @@ -55,6 +55,11 @@ visit-number visit-snapshot + snapshot? + snapshot-id + snapshot-branches + lookup-snapshot-branch + branch? branch-name branch-target @@ -183,6 +188,12 @@ Software Heritage." (ref 10)))))) str)) ;oops! +(define (maybe-null proc) + (match-lambda + ((? null?) #f) + ('null #f) + (obj (proc obj)))) + (define string* ;; Converts "string or #nil" coming from JSON to "string or #f". (match-lambda @@ -287,6 +298,7 @@ FALSE-IF-404? is true, return #f upon 404 responses." ;; <https://archive.softwareheritage.org/api/1/snapshot/4334c3ed4bb208604ed780d8687fe523837f1bd1/> (define-json-mapping <snapshot> make-snapshot snapshot? json->snapshot + (id snapshot-id) (branches snapshot-branches "branches" json->branches)) ;; This is used for the "branches" field of snapshots. @@ -316,10 +328,13 @@ FALSE-IF-404? is true, return #f upon 404 responses." (target-url release-target-url "target_url")) ;; <https://archive.softwareheritage.org/api/1/revision/359fdda40f754bbf1b5dc261e7427b75463b59be/> +;; Note: Some revisions, such as those for "nixguix" origins (e.g., +;; <https://archive.softwareheritage.org/api/1/revision/b8dbc65475bbedde8e015d4730ade8864c38fad3/>), +;; have their 'date' field set to null. (define-json-mapping <revision> make-revision revision? json->revision (id revision-id) - (date revision-date "date" string->date*) + (date revision-date "date" (maybe-null string->date*)) (directory revision-directory) (directory-url revision-directory-url "directory_url")) @@ -426,6 +441,32 @@ available." (call (swh-url (visit-snapshot-url visit)) json->snapshot))) +(define (snapshot-url snapshot branch-count first-branch) + "Return the URL of SNAPSHOT such that it contains information for +BRANCH-COUNT branches, starting at FIRST-BRANCH." + (string-append (swh-url "/api/1/snapshot" (snapshot-id snapshot)) + "?branches_count=" (number->string branch-count) + "&branches_from=" (uri-encode first-branch))) + +(define (lookup-snapshot-branch snapshot name) + "Look up branch NAME on SNAPSHOT. Return the branch, or return #f if it +could not be found." + (or (find (lambda (branch) + (string=? (branch-name branch) name)) + (snapshot-branches snapshot)) + + ;; There's no API entry point to look up a snapshot branch by name. + ;; Work around that by using the paginated list of branches provided by + ;; the /api/1/snapshot API: ask for one branch, and start pagination at + ;; NAME. + (let ((snapshot (call (snapshot-url snapshot 1 name) + json->snapshot))) + (match (snapshot-branches snapshot) + ((branch) + (and (string=? (branch-name branch) name) + branch)) + (_ #f))))) + (define (branch-target branch) "Return the target of BRANCH, either a <revision> or a <release>." (match (branch-target-type branch) |