import: Update opam importer.

* guix/import/opam.scm: Update importer for opam 2.
* tests/opam.scm: Update tests for the opam 2 importer.

1 files changed, 168 insertions, 137 deletions

diff --git a/guix/import/opam.scm b/guix/import/opam.scm
index f252bdc31a..c42a5d767d 100644
--- a/guix/import/opam.scm
+++ b/guix/import/opam.scm
@@ -17,132 +17,108 @@
 ;;; along with GNU Guix.  If not, see <http://www.gnu.org/licenses/>.
 
 (define-module (guix import opam)
+  #:use-module (ice-9 ftw)
   #:use-module (ice-9 match)
-  #:use-module (ice-9 vlist)
+  #:use-module (ice-9 peg)
+  #:use-module (ice-9 receive)
   #:use-module ((ice-9 rdelim) #:select (read-line))
+  #:use-module (ice-9 textual-ports)
+  #:use-module (ice-9 vlist)
   #:use-module (srfi srfi-1)
+  #:use-module (srfi srfi-2)
   #:use-module (web uri)
   #:use-module (guix http-client)
+  #:use-module (guix git)
+  #:use-module (guix ui)
   #:use-module (guix utils)
   #:use-module (guix import utils)
   #:use-module ((guix licenses) #:prefix license:)
   #:export (opam->guix-package))
 
-(define (opam-urls)
-  "Fetch the urls.txt file from the opam repository and returns the list of
-URLs it contains."
-  (let ((port (http-fetch/cached (string->uri "https://opam.ocaml.org/urls.txt"))))
-    (let loop ((result '()))
-      (let ((line (read-line port)))
-        (if (eof-object? line)
-          (begin
-            (close port)
-            result)
-          (loop (cons line result)))))))
-
-(define (vhash-ref hashtable key default)
-  (match (vhash-assoc key hashtable)
-    (#f default)
-    ((_ . x) x)))
-
-(define (hashtable-update hashtable line)
-  "Parse @var{line} to get the name and version of the package and adds them
-to the hashtable."
-  (let* ((line (string-split line #\ )))
-    (match line
-      ((url foo ...)
-       (if (equal? url "repo")
-         hashtable
-         (match (string-split url #\/)
-           ((type name1 versionstr foo ...)
-            (if (equal? type "packages")
-              (match (string-split versionstr #\.)
-                ((name2 versions ...)
-                 (let ((version (string-join versions ".")))
-                   (if (equal? name1 name2)
-                     (let ((curr (vhash-ref hashtable name1 '())))
-                       (vhash-cons name1 (cons version curr) hashtable))
-                     hashtable)))
-                (_ hashtable))
-              hashtable))
-           (_ hashtable))))
-      (_ hashtable))))
-
-(define (urls->hashtable urls)
-  "Transform urls.txt in a hashtable whose keys are package names and values
-the list of available versions."
-  (let ((hashtable vlist-null))
-    (let loop ((urls urls) (hashtable hashtable))
-      (match urls
-        (() hashtable)
-        ((url rest ...) (loop rest (hashtable-update hashtable url)))))))
+;; Define a PEG parser for the opam format
+(define-peg-pattern SP none (or " " "\n"))
+(define-peg-pattern SP2 body (or " " "\n"))
+(define-peg-pattern QUOTE none "\"")
+(define-peg-pattern QUOTE2 body "\"")
+(define-peg-pattern COLON none ":")
+;; A string character is any character that is not a quote, or a quote preceded by a backslash.
+(define-peg-pattern STRCHR body
+                    (or " " "!" (and (ignore "\\") "\"")
+                        (and (ignore "\\") "\\") (range #\# #\頋)))
+(define-peg-pattern operator all (or "=" "!" "<" ">"))
+
+(define-peg-pattern records body (* (and (or record weird-record) (* SP))))
+(define-peg-pattern record all (and key COLON (* SP) value))
+(define-peg-pattern weird-record all (and key (* SP) dict))
+(define-peg-pattern key body (+ (or (range #\a #\z) "-")))
+(define-peg-pattern value body (and (or conditional-value ground-value operator) (* SP)))
+(define-peg-pattern ground-value body (and (or multiline-string string-pat list-pat var) (* SP)))
+(define-peg-pattern conditional-value all (and ground-value (* SP) condition))
+(define-peg-pattern string-pat all (and QUOTE (* STRCHR) QUOTE))
+(define-peg-pattern list-pat all (and (ignore "[") (* SP) (* (and value (* SP))) (ignore "]")))
+(define-peg-pattern var all (+ (or (range #\a #\z) "-")))
+(define-peg-pattern multiline-string all
+                    (and QUOTE QUOTE QUOTE (* SP)
+                         (* (or SP2 STRCHR (and QUOTE2 (not-followed-by QUOTE))
+                                (and QUOTE2 QUOTE2 (not-followed-by QUOTE))))
+                         QUOTE QUOTE QUOTE))
+(define-peg-pattern dict all (and (ignore "{") (* SP) records (* SP) (ignore "}")))
+
+(define-peg-pattern condition body (and (ignore "{") condition-form (ignore "}")))
+
+(define-peg-pattern condition-form body
+                    (and
+                      (* SP)
+                      (or condition-and condition-or condition-form2)
+                      (* SP)))
+(define-peg-pattern condition-form2 body
+                    (and (* SP) (or condition-greater-or-equal condition-greater
+                                    condition-lower-or-equal condition-lower
+                                    condition-neq condition-eq condition-content) (* SP)))
+
+;(define-peg-pattern condition-operator all (and (ignore operator) (* SP) condition-string))
+(define-peg-pattern condition-greater-or-equal all (and (ignore (and ">" "=")) (* SP) condition-string))
+(define-peg-pattern condition-greater all (and (ignore ">") (* SP) condition-string))
+(define-peg-pattern condition-lower-or-equal all (and (ignore (and "<" "=")) (* SP) condition-string))
+(define-peg-pattern condition-lower all (and (ignore "<") (* SP) condition-string))
+(define-peg-pattern condition-and all (and condition-form2 (* SP) (? (ignore "&")) (* SP) condition-form))
+(define-peg-pattern condition-or all (and condition-form2 (* SP) (ignore "|") (* SP) condition-form))
+(define-peg-pattern condition-eq all (and condition-content (* SP) (ignore "=") (* SP) condition-content))
+(define-peg-pattern condition-neq all (and condition-content (* SP) (ignore (and "!" "=")) (* SP) condition-content))
+(define-peg-pattern condition-content body (or condition-string condition-var))
+(define-peg-pattern condition-content2 body (and condition-content (* SP) (not-followed-by (or "&" "=" "!"))))
+(define-peg-pattern condition-string all (and QUOTE (* STRCHR) QUOTE))
+(define-peg-pattern condition-var all (+ (or (range #\a #\z) "-")))
+
+(define (get-opam-repository)
+  "Update or fetch the latest version of the opam repository and return the
+path to the repository."
+  (receive (location commit)
+    (update-cached-checkout "https://github.com/ocaml/opam-repository")
+    location))
 
 (define (latest-version versions)
   "Find the most recent version from a list of versions."
-  (match versions
-    ((first rest ...)
-     (let loop ((versions rest) (m first))
-       (match versions
-         (() m)
-         ((first rest ...)
-          (loop rest (if (version>? m first) m first))))))))
-
-(define (fetch-package-url uri)
-  "Fetch and parse the url file.  Return the URL the package can be downloaded
-from."
-  (let ((port (http-fetch uri)))
-    (let loop ((result #f))
-      (let ((line (read-line port)))
-        (if (eof-object? line)
-          (begin
-            (close port)
-            result)
-          (let* ((line (string-split line #\ )))
-            (match line
-              ((key value rest ...)
-               (if (member key '("archive:" "http:"))
-                 (loop (string-trim-both value #\"))
-                 (loop result))))))))))
-
-(define (fetch-package-metadata uri)
-  "Fetch and parse the opam file.  Return an association list containing the
-homepage, the license and the list of inputs."
-  (let ((port (http-fetch uri)))
-    (let loop ((result '()) (dependencies? #f))
-      (let ((line (read-line port)))
-        (if (eof-object? line)
-          (begin
-            (close port)
-            result)
-          (let* ((line (string-split line #\ )))
-            (match line
-               ((key value ...)
-                (let ((dependencies?
-                        (if dependencies?
-                          (not (equal? key "]"))
-                          (equal? key "depends:")))
-                      (val (string-trim-both (string-join value "") #\")))
-                  (cond
-                    ((equal? key "homepage:")
-                     (loop (cons `("homepage" . ,val) result) dependencies?))
-                    ((equal? key "license:")
-                     (loop (cons `("license" . ,val) result) dependencies?))
-                    ((and dependencies? (not (equal? val "[")))
-                     (match (string-split val #\{)
-                       ((val rest ...)
-                        (let ((curr (assoc-ref result "inputs"))
-                              (new (string-trim-both
-                                     val (list->char-set '(#\] #\[ #\")))))
-                          (loop (cons `("inputs" . ,(cons new (if curr curr '()))) result)
-                                (if (string-contains val "]") #f dependencies?))))))
-                    (else (loop result dependencies?))))))))))))
-
-(define (string->license str)
-  (cond
-    ((equal? str "MIT") '(license:expat))
-    ((equal? str "GPL2") '(license:gpl2))
-    ((equal? str "LGPLv2") '(license:lgpl2))
-    (else `())))
+  (fold (lambda (a b) (if (version>? a b) a b)) (car versions) versions))
+
+(define (find-latest-version package repository)
+  "Get the latest version of a package as described in the given repository."
+  (let* ((dir (string-append repository "/packages/" package))
+         (versions (scandir dir (lambda (name) (not (string-prefix? "." name))))))
+    (if versions
+      (let ((versions (map
+                        (lambda (dir)
+                          (string-join (cdr (string-split dir #\.)) "."))
+                        versions)))
+        (latest-version versions))
+      (begin
+        (format #t (G_ "Package not found in opam repository: ~a~%") package)
+        #f))))
+
+(define (get-metadata opam-file)
+  (with-input-from-file opam-file
+    (lambda _
+      (peg:tree (match-pattern records (get-string-all (current-input-port)))))))
 
 (define (ocaml-name->guix-name name)
   (cond
@@ -151,33 +127,85 @@ homepage, the license and the list of inputs."
     ((string-prefix? "conf-" name) (substring name 5))
     (else (string-append "ocaml-" name))))
 
-(define (dependencies->inputs dependencies)
-  "Transform the list of dependencies in a list of inputs."
-  (if (not dependencies)
-    '()
-    (map (lambda (input)
-           (list input (list 'unquote (string->symbol input))))
-         (map ocaml-name->guix-name dependencies))))
+(define (metadata-ref file lookup)
+  (pk 'file file 'lookup lookup)
+  (fold (lambda (record acc)
+          (match record
+            ((record key val)
+             (if (equal? key lookup)
+               (match val
+                 (('list-pat . stuff) stuff)
+                 (('string-pat stuff) stuff)
+                 (('multiline-string stuff) stuff)
+                 (('dict records ...) records))
+               acc))))
+        #f file))
+
+(define (native? condition)
+  (match condition
+    (('condition-var var)
+     (match var
+       ("with-test" #t)
+       ("test" #t)
+       ("build" #t)
+       (_ #f)))
+    ((or ('condition-or cond-left cond-right) ('condition-and cond-left cond-right))
+     (or (native? cond-left)
+         (native? cond-right)))
+    (_ #f)))
+
+(define (dependency->input dependency)
+  (match dependency
+    (('string-pat str) str)
+    (('conditional-value val condition)
+     (if (native? condition) "" (dependency->input val)))))
+
+(define (dependency->native-input dependency)
+  (match dependency
+    (('string-pat str) "")
+    (('conditional-value val condition)
+     (if (native? condition) (dependency->input val) ""))))
+
+(define (ocaml-names->guix-names names)
+  (map ocaml-name->guix-name
+       (remove (lambda (name)
+                 (or (equal? "" name))
+                     (equal? "ocaml" name))
+               names)))
+
+(define (depends->inputs depends)
+  (filter (lambda (name)
+            (and (not (equal? "" name))
+                 (not (equal? "ocaml" name))
+                 (not (equal? "ocamlfind" name))))
+    (map dependency->input depends)))
+
+(define (depends->native-inputs depends)
+  (filter (lambda (name) (not (equal? "" name)))
+    (map dependency->native-input depends)))
+
+(define (dependency-list->inputs lst)
+  (map
+    (lambda (dependency)
+      (list dependency (list 'unquote (string->symbol dependency))))
+    (ocaml-names->guix-names lst)))
 
 (define (opam->guix-package name)
-  (let* ((hashtable (urls->hashtable (opam-urls)))
-         (versions (vhash-ref hashtable name #f)))
-    (unless (eq? versions #f)
-      (let* ((version (latest-version versions))
-             (package-url (string-append "https://opam.ocaml.org/packages/" name
-                                         "/" name "." version "/"))
-             (url-url (string-append package-url "url"))
-             (opam-url (string-append package-url "opam"))
-             (source-url (fetch-package-url url-url))
-             (metadata (fetch-package-metadata opam-url))
-             (dependencies (assoc-ref metadata "inputs"))
-             (inputs (dependencies->inputs dependencies)))
+  (and-let* ((repository (get-opam-repository))
+             (version (find-latest-version name repository))
+             (file (string-append repository "/packages/" name "/" name "." (pk 'version version) "/opam"))
+             (opam-content (get-metadata file))
+             (url-dict (metadata-ref (pk 'metadata opam-content) "url"))
+             (source-url (metadata-ref url-dict "src"))
+             (requirements (metadata-ref opam-content "depends"))
+             (inputs (dependency-list->inputs (depends->inputs requirements)))
+             (native-inputs (dependency-list->inputs (depends->native-inputs requirements))))
         (call-with-temporary-output-file
           (lambda (temp port)
             (and (url-fetch source-url temp)
                  `(package
                     (name ,(ocaml-name->guix-name name))
-                    (version ,version)
+                    (version ,(metadata-ref opam-content "version"))
                     (source
                       (origin
                         (method url-fetch)
@@ -187,7 +215,10 @@ homepage, the license and the list of inputs."
                     ,@(if (null? inputs)
                         '()
                         `((inputs ,(list 'quasiquote inputs))))
-                    (home-page ,(assoc-ref metadata "homepage"))
-                    (synopsis "")
-                    (description "")
-                    (license ,@(string->license (assoc-ref metadata "license")))))))))))
+                    ,@(if (null? native-inputs)
+                        '()
+                        `((native-inputs ,(list 'quasiquote native-inputs))))
+                    (home-page ,(metadata-ref opam-content "homepage"))
+                    (synopsis ,(metadata-ref opam-content "synopsis"))
+                    (description ,(metadata-ref opam-content "description"))
+                    (license #f)))))))