From f862f38ade0c6a648d5a8711d636281314b2db62 Mon Sep 17 00:00:00 2001 From: Paul van der Walt Date: Fri, 23 Oct 2015 19:33:43 +0200 Subject: build-system/haskell: CONFIG_SHELL env variable. For Cabal packages with "build-type: Configure", a configure shell script is run to set up build parameters. These scripts need the CONFIG_SHELL environment variable to be set to function properly. * guix/build/haskell-build-system.scm (configure): Set CONFIG_SHELL if necessary. --- guix/build/haskell-build-system.scm | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'guix/build') diff --git a/guix/build/haskell-build-system.scm b/guix/build/haskell-build-system.scm index c0cb789581..4506e96af9 100644 --- a/guix/build/haskell-build-system.scm +++ b/guix/build/haskell-build-system.scm @@ -1,5 +1,6 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2015 Federico Beffa +;;; Copyright © 2015 Paul van der Walt ;;; ;;; This file is part of GNU Guix. ;;; @@ -96,6 +97,14 @@ and parameters ~s~%" '("--enable-tests") '()) configure-flags))) + ;; For packages where the Cabal build-type is set to "Configure", + ;; ./configure will be executed. In these cases, the following + ;; environment variable is needed to be able to find the shell executable. + ;; For other package types, the configure script isn't present. For more + ;; information, see the Build Information section of + ;; . + (when (file-exists? "configure") + (setenv "CONFIG_SHELL" "sh")) (run-setuphs "configure" params))) (define* (build #:rest empty) -- cgit v1.2.3 From 60fd51222f9d7ec90bdad37bca921f40f7f5b104 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Thu, 12 Nov 2015 22:37:32 +0100 Subject: download: Add timeout parameter for connections. * guix/build/download.scm (ensure-uri): New procedure. (current-http-proxy): New variable. (open-socket-for-uri): Copy from Guile commit aaea5b2, but add #:timeout parameter and use 'connect*' instead of 'connect'. (open-connection-for-uri): Add #:timeout parameter and pass it to 'open-socket-for-uri'. --- guix/build/download.scm | 82 ++++++++++++++++++++++++++++++++++++------------- 1 file changed, 61 insertions(+), 21 deletions(-) (limited to 'guix/build') diff --git a/guix/build/download.scm b/guix/build/download.scm index 240e79ee8d..17e8f8cb9e 100644 --- a/guix/build/download.scm +++ b/guix/build/download.scm @@ -20,6 +20,7 @@ (define-module (guix build download) #:use-module (web uri) + #:use-module (web http) #:use-module ((web client) #:hide (open-socket-for-uri)) #:use-module (web response) #:use-module (guix ftp-client) @@ -277,26 +278,65 @@ host name without trailing dot." (add-weak-reference record port) record))) -(define (open-socket-for-uri uri) - "Return an open port for URI. This variant works around - which affects Guile's 'open-socket-for-uri' up to -2.0.11 included." - (define rmem-max - ;; The maximum size for a receive buffer on Linux, see socket(7). - "/proc/sys/net/core/rmem_max") - - (define buffer-size - (if (file-exists? rmem-max) - (call-with-input-file rmem-max read) - 126976)) ;the default for Linux, per 'rmem_default' - - (let ((s ((@ (web client) open-socket-for-uri) uri))) - ;; Work around by restoring a decent - ;; buffer size. - (setsockopt s SOL_SOCKET SO_RCVBUF buffer-size) - s)) - -(define (open-connection-for-uri uri) +(define (ensure-uri uri-or-string) ;XXX: copied from (web http) + (cond + ((string? uri-or-string) (string->uri uri-or-string)) + ((uri? uri-or-string) uri-or-string) + (else (error "Invalid URI" uri-or-string)))) + +(define current-http-proxy + ;; XXX: Add a dummy definition for Guile < 2.0.10; this is used in + ;; 'open-socket-for-uri'. + (or (and=> (module-variable (resolve-interface '(web client)) + 'current-http-proxy) + variable-ref) + (const #f))) + +(define* (open-socket-for-uri uri-or-string #:key timeout) + "Return an open input/output port for a connection to URI. When TIMEOUT is +not #f, it must be a (possibly inexact) number denoting the maximum duration +in seconds to wait for the connection to complete; passed TIMEOUT, an +ETIMEDOUT error is raised." + ;; Includes a fix for which affects Guile's + ;; 'open-socket-for-uri' up to 2.0.11 included, and uses 'connect*' instead + ;; of 'connect'. + + (define http-proxy (current-http-proxy)) + (define uri (ensure-uri (or http-proxy uri-or-string))) + (define addresses + (let ((port (uri-port uri))) + (delete-duplicates + (getaddrinfo (uri-host uri) + (cond (port => number->string) + (else (symbol->string (uri-scheme uri)))) + (if port + AI_NUMERICSERV + 0)) + (lambda (ai1 ai2) + (equal? (addrinfo:addr ai1) (addrinfo:addr ai2)))))) + + (let loop ((addresses addresses)) + (let* ((ai (car addresses)) + (s (with-fluids ((%default-port-encoding #f)) + ;; Restrict ourselves to TCP. + (socket (addrinfo:fam ai) SOCK_STREAM IPPROTO_IP)))) + (catch 'system-error + (lambda () + (connect* s (addrinfo:addr ai) timeout) + + ;; Buffer input and output on this port. + (setvbuf s _IOFBF) + ;; If we're using a proxy, make a note of that. + (when http-proxy (set-http-proxy-port?! s #t)) + s) + (lambda args + ;; Connection failed, so try one of the other addresses. + (close s) + (if (null? (cdr addresses)) + (apply throw args) + (loop (cdr addresses)))))))) + +(define* (open-connection-for-uri uri #:key timeout) "Like 'open-socket-for-uri', but also handle HTTPS connections." (define https? (eq? 'https (uri-scheme uri))) @@ -319,7 +359,7 @@ host name without trailing dot." (thunk)) (thunk))))))) (with-https-proxy - (let ((s (open-socket-for-uri uri))) + (let ((s (open-socket-for-uri uri #:timeout timeout))) ;; Buffer input and output on this port. (setvbuf s _IOFBF %http-receive-buffer-size) -- cgit v1.2.3 From 1b9aefa394a57dabe38e0658a3b612e962d3fc5e Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Thu, 12 Nov 2015 23:15:18 +0100 Subject: download: Always use AI_ADDRCONFIG when resolving host names. * guix/build/download.scm (open-socket-for-uri): Always pass AI_ADDRCONFIG to 'getaddrinfo' as recommended in the fine Guile manual. * guix/ftp-client.scm (ftp-open): Ditto. --- guix/build/download.scm | 10 +++++----- guix/ftp-client.scm | 4 +++- 2 files changed, 8 insertions(+), 6 deletions(-) (limited to 'guix/build') diff --git a/guix/build/download.scm b/guix/build/download.scm index 17e8f8cb9e..8843804c40 100644 --- a/guix/build/download.scm +++ b/guix/build/download.scm @@ -298,8 +298,8 @@ not #f, it must be a (possibly inexact) number denoting the maximum duration in seconds to wait for the connection to complete; passed TIMEOUT, an ETIMEDOUT error is raised." ;; Includes a fix for which affects Guile's - ;; 'open-socket-for-uri' up to 2.0.11 included, and uses 'connect*' instead - ;; of 'connect'. + ;; 'open-socket-for-uri' up to 2.0.11 included, uses 'connect*' instead + ;; of 'connect', and uses AI_ADDRCONFIG. (define http-proxy (current-http-proxy)) (define uri (ensure-uri (or http-proxy uri-or-string))) @@ -309,9 +309,9 @@ ETIMEDOUT error is raised." (getaddrinfo (uri-host uri) (cond (port => number->string) (else (symbol->string (uri-scheme uri)))) - (if port - AI_NUMERICSERV - 0)) + (if (number? port) + (logior AI_ADDRCONFIG AI_NUMERICSERV) + AI_ADDRCONFIG)) (lambda (ai1 ai2) (equal? (addrinfo:addr ai1) (addrinfo:addr ai2)))))) diff --git a/guix/ftp-client.scm b/guix/ftp-client.scm index 9ea878a145..f02d460061 100644 --- a/guix/ftp-client.scm +++ b/guix/ftp-client.scm @@ -134,7 +134,9 @@ TIMEOUT, an ETIMEDOUT error is raised." (define addresses (getaddrinfo host (if (number? port) (number->string port) port) - (if (number? port) AI_NUMERICSERV 0))) + (if (number? port) + (logior AI_ADDRCONFIG AI_NUMERICSERV) + AI_ADDRCONFIG))) (let loop ((addresses addresses)) (let* ((ai (car addresses)) -- cgit v1.2.3 From 9c88f655e6533e2f84ebf7ee546596c85031441d Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Mon, 16 Nov 2015 14:16:22 +0100 Subject: graft: Graft files in a deterministic order. * guix/build/graft.scm (rewrite-directory)[rewrite-leaf]: Change to take a single parameter. Add call to 'lstat'. Factorize result of 'destination'. Use 'find-files' instead of 'file-system-fold'. --- guix/build/graft.scm | 60 +++++++++++++++++++++++----------------------------- 1 file changed, 26 insertions(+), 34 deletions(-) (limited to 'guix/build') diff --git a/guix/build/graft.scm b/guix/build/graft.scm index 55f0f9410d..d29e671c67 100644 --- a/guix/build/graft.scm +++ b/guix/build/graft.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2014 Ludovic Courtès +;;; Copyright © 2014, 2015 Ludovic Courtès ;;; ;;; This file is part of GNU Guix. ;;; @@ -21,7 +21,6 @@ #:use-module (rnrs bytevectors) #:use-module (rnrs io ports) #:use-module (ice-9 match) - #:use-module (ice-9 ftw) #:export (replace-store-references rewrite-directory)) @@ -93,38 +92,31 @@ file name pairs." (define (destination file) (string-append output (string-drop file prefix-len))) - (define (rewrite-leaf file stat result) - (case (stat:type stat) - ((symlink) - (let ((target (readlink file))) - (symlink (call-with-output-string - (lambda (output) - (replace-store-references (open-input-string target) - output mapping - store))) - (destination file)))) - ((regular) - (with-fluids ((%default-port-encoding #f)) - (call-with-input-file file - (lambda (input) - (call-with-output-file (destination file) - (lambda (output) - (replace-store-references input output mapping - store) - (chmod output (stat:perms stat)))))))) - (else - (error "unsupported file type" stat)))) + (define (rewrite-leaf file) + (let ((stat (lstat file)) + (dest (destination file))) + (mkdir-p (dirname dest)) + (case (stat:type stat) + ((symlink) + (let ((target (readlink file))) + (symlink (call-with-output-string + (lambda (output) + (replace-store-references (open-input-string target) + output mapping + store))) + dest))) + ((regular) + (with-fluids ((%default-port-encoding #f)) + (call-with-input-file file + (lambda (input) + (call-with-output-file dest + (lambda (output) + (replace-store-references input output mapping + store) + (chmod output (stat:perms stat)))))))) + (else + (error "unsupported file type" stat))))) - (file-system-fold (const #t) - rewrite-leaf - (lambda (directory stat result) ;down - (mkdir (destination directory))) - (const #t) ;up - (const #f) ;skip - (lambda (file stat errno result) ;error - (error "read error" file stat errno)) - #f - directory - lstat)) + (for-each rewrite-leaf (find-files directory))) ;;; graft.scm ends here -- cgit v1.2.3 From 333c376c4586d16b215b994240ad4a5ddaa74d03 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Mon, 16 Nov 2015 14:22:13 +0100 Subject: graft: Graft files in parallel. * guix/build/graft.scm (rewrite-directory): Use 'n-par-for-each' instead of 'for-each'. --- guix/build/graft.scm | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'guix/build') diff --git a/guix/build/graft.scm b/guix/build/graft.scm index d29e671c67..0a9cd3260c 100644 --- a/guix/build/graft.scm +++ b/guix/build/graft.scm @@ -21,6 +21,7 @@ #:use-module (rnrs bytevectors) #:use-module (rnrs io ports) #:use-module (ice-9 match) + #:use-module (ice-9 threads) #:export (replace-store-references rewrite-directory)) @@ -117,6 +118,7 @@ file name pairs." (else (error "unsupported file type" stat))))) - (for-each rewrite-leaf (find-files directory))) + (n-par-for-each (parallel-job-count) + rewrite-leaf (find-files directory))) ;;; graft.scm ends here -- cgit v1.2.3 From dedc8320701b3f5d23ccc213c1c0381e00b33785 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Sat, 14 Nov 2015 00:08:35 +0100 Subject: gnu: python-2: Honor 'SOURCE_DATE_EPOCH'. * gnu/packages/patches/python-2.7-source-date-epoch.patch: New file. * gnu/packages/python.scm (python-2)[source]: Use it. [arguments]: Set SOURCE_DATE_EPOCH in 'patch-lib-shells' phase. * guix/build/python-build-system.scm (set-SOURCE-DATE-EPOCH): New procedure. (%standard-phases): Add it. * gnu-system.am (dist_patch_DATA): Add patch. --- gnu-system.am | 1 + .../patches/python-2.7-source-date-epoch.patch | 33 ++++++++++++++++++++++ gnu/packages/python.scm | 9 +++++- guix/build/python-build-system.scm | 7 +++++ 4 files changed, 49 insertions(+), 1 deletion(-) create mode 100644 gnu/packages/patches/python-2.7-source-date-epoch.patch (limited to 'guix/build') diff --git a/gnu-system.am b/gnu-system.am index 1fdbd00731..c7a17a42e9 100644 --- a/gnu-system.am +++ b/gnu-system.am @@ -670,6 +670,7 @@ dist_patch_DATA = \ gnu/packages/patches/pybugz-stty.patch \ gnu/packages/patches/pyqt-configure.patch \ gnu/packages/patches/python-2.7-search-paths.patch \ + gnu/packages/patches/python-2.7-source-date-epoch.patch \ gnu/packages/patches/python-3-search-paths.patch \ gnu/packages/patches/python-disable-ssl-test.patch \ gnu/packages/patches/python-fix-tests.patch \ diff --git a/gnu/packages/patches/python-2.7-source-date-epoch.patch b/gnu/packages/patches/python-2.7-source-date-epoch.patch new file mode 100644 index 0000000000..be1f8e010e --- /dev/null +++ b/gnu/packages/patches/python-2.7-source-date-epoch.patch @@ -0,0 +1,33 @@ +Honor the 'SOURCE_DATE_EPOCH' environment variable to allow for +determinitic builds. + +--- a/Lib/py_compile.py ++++ b/Lib/py_compile.py +@@ -105,7 +105,10 @@ def compile(file, cfile=None, dfile=None, doraise=False): + """ + with open(file, 'U') as f: + try: +- timestamp = long(os.fstat(f.fileno()).st_mtime) ++ if 'SOURCE_DATE_EPOCH' in os.environ: ++ timestamp = long(os.environ['SOURCE_DATE_EPOCH']) ++ else: ++ timestamp = long(os.fstat(f.fileno()).st_mtime) + except AttributeError: + timestamp = long(os.stat(file).st_mtime) + codestring = f.read() +diff --git a/Python/import.c b/Python/import.c +index e47ce63..7eecf9c 100644 +--- a/Python/import.c ++++ b/Python/import.c +@@ -945,6 +945,11 @@ write_compiled_module(PyCodeObject *co, char *cpathname, struct stat *srcstat, t + /* Now write the true mtime (as a 32-bit field) */ + fseek(fp, 4L, 0); + assert(mtime <= 0xFFFFFFFF); ++ if (Py_GETENV("SOURCE_DATE_EPOCH") != NULL) { ++ const char *epoch = Py_GETENV("SOURCE_DATE_EPOCH"); ++ mtime = atoi(epoch); ++ } ++ + PyMarshal_WriteLongToFile((long)mtime, fp, Py_MARSHAL_VERSION); + fflush(fp); + fclose(fp); diff --git a/gnu/packages/python.scm b/gnu/packages/python.scm index 001d00e9be..07725c727a 100644 --- a/gnu/packages/python.scm +++ b/gnu/packages/python.scm @@ -92,7 +92,9 @@ (sha256 (base32 "1h7zbrf9pkj29hlm18b10548ch9757f75m64l47sy75rh43p7lqw")) - (patches (list (search-patch "python-2.7-search-paths.patch"))))) + (patches (map search-patch + '("python-2.7-search-paths.patch" + "python-2.7-source-date-epoch.patch"))))) (build-system gnu-build-system) (arguments `(#:tests? #f @@ -158,6 +160,11 @@ "Lib/distutils/tests/test_spawn.py" "Lib/test/test_subprocess.py")) (("/bin/sh") (which "sh"))) + + ;; Use zero as the timestamp in .pyc files so that builds are + ;; deterministic. TODO: Remove it when this variable is set in + ;; gnu-build-system.scm. + (setenv "SOURCE_DATE_EPOCH" "0") #t)) (add-before 'check 'pre-check diff --git a/guix/build/python-build-system.scm b/guix/build/python-build-system.scm index 1ae42c00b4..6775cc4396 100644 --- a/guix/build/python-build-system.scm +++ b/guix/build/python-build-system.scm @@ -136,11 +136,18 @@ installed with setuptools." #t)) #t)) +(define* (set-SOURCE-DATE-EPOCH #:rest _) + "Set the 'SOURCE_DATE_EPOCH' environment variable." + ;; Use zero as the timestamp in .pyc files so that builds are deterministic. + ;; TODO: Remove it when this variable is set in GNU:%STANDARD-PHASES. + (setenv "SOURCE_DATE_EPOCH" "0")) + (define %standard-phases ;; 'configure' and 'build' phases are not needed. Everything is done during ;; 'install'. (modify-phases gnu:%standard-phases (add-after 'unpack 'ensure-no-mtimes-pre-1980 ensure-no-mtimes-pre-1980) + (add-after 'unpack 'set-SOURCE-DATE-EPOCH set-SOURCE-DATE-EPOCH) (delete 'configure) (replace 'install install) (replace 'check check) -- cgit v1.2.3 From a665996f58764a0c6e805016915225f911294989 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Sat, 14 Nov 2015 16:52:17 +0100 Subject: gnu: python: Set SOURCE_DATE_EPOCH to 1 instead of 0. * gnu/packages/python.scm (python-2)[arguments]: Set SOURCE_DATE_EPOCH to 1 to match what the daemon does. * guix/build/python-build-system.scm (set-SOURCE-DATE-EPOCH): Likewise. --- gnu/packages/python.scm | 2 +- guix/build/python-build-system.scm | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'guix/build') diff --git a/gnu/packages/python.scm b/gnu/packages/python.scm index 07725c727a..7f7fa388a1 100644 --- a/gnu/packages/python.scm +++ b/gnu/packages/python.scm @@ -164,7 +164,7 @@ ;; Use zero as the timestamp in .pyc files so that builds are ;; deterministic. TODO: Remove it when this variable is set in ;; gnu-build-system.scm. - (setenv "SOURCE_DATE_EPOCH" "0") + (setenv "SOURCE_DATE_EPOCH" "1") #t)) (add-before 'check 'pre-check diff --git a/guix/build/python-build-system.scm b/guix/build/python-build-system.scm index 6775cc4396..8025b7fec6 100644 --- a/guix/build/python-build-system.scm +++ b/guix/build/python-build-system.scm @@ -140,7 +140,7 @@ installed with setuptools." "Set the 'SOURCE_DATE_EPOCH' environment variable." ;; Use zero as the timestamp in .pyc files so that builds are deterministic. ;; TODO: Remove it when this variable is set in GNU:%STANDARD-PHASES. - (setenv "SOURCE_DATE_EPOCH" "0")) + (setenv "SOURCE_DATE_EPOCH" "1")) (define %standard-phases ;; 'configure' and 'build' phases are not needed. Everything is done during -- cgit v1.2.3