diff --git a/3bmd-ext-footnotes.asd b/3bmd-ext-footnotes.asd new file mode 100644 index 0000000..b074d11 --- /dev/null +++ b/3bmd-ext-footnotes.asd @@ -0,0 +1,20 @@ +(defsystem 3bmd-ext-footnotes + :description "extension to 3bmd implementing github style ``` delimited code blocks, with support for syntax highlighting using colorize, pygments, or chroma" + :depends-on (3bmd alexandria) + :serial t + :components ((:file "footnotes")) + :in-order-to ((test-op (test-op 3bmd-ext-footnotes/tests)))) + + +(defsystem 3bmd-ext-footnotes/tests + :depends-on (#:3bmd-ext-footnotes #:3bmd-tests #:fiasco) + :serial t + :components ((:module "tests" + :components ((:module "extensions" + :components ((:file "footnotes")))))) + :perform (test-op (op c) + (declare (ignore op c)) + (or + (symbol-call "FIASCO" "RUN-PACKAGE-TESTS" + :package '#:3bmd-footnotes-tests) + (error "tests failed")))) diff --git a/README.md b/README.md index 94eeefb..d498b90 100644 --- a/README.md +++ b/README.md @@ -133,3 +133,14 @@ especially, without heading: $$ \frac{\partial E}{\partial y} = \frac{\partial }{\partial y} \frac{1}{n}\sum_{i=1}^{n} (y_i - a_i)^2 $$ + +* Loading `3bmd-footnotes.asd` adds support for footnotes. If `3bmd-footnotes:*footnotes*` is non-`NIL` while parsing, the syntax `[^id]` can be used to refer to a footnote, which should be defined by `[^id]: contents of the footnote`. For example: + + A footnote[^1] reference. + + [^1]: a footnote, which can have its own footnotes[^2]. + [^3]: almost anywhere: defining footnotes inside other block elements doesn't currently work. + [^2]: a footnote on a footnote. + [^x]: footnotes without any references in main document will not be shown. + + The footnote definitions can be anywhere[^3] in the document, and will be moved to the end of the generated HTML in order of reference. diff --git a/extensions.lisp b/extensions.lisp index c044edb..9709319 100644 --- a/extensions.lisp +++ b/extensions.lisp @@ -47,7 +47,8 @@ (escapes (cdr (assoc :escape-char-rule options))) (md-chars-to-escapes (cdr (assoc :md-chars-to-escape options))) (after (cdr (assoc :after options))) - (before (cdr (assoc :before options)))) + (before (cdr (assoc :before options))) + (bind (cdr (assoc :bind options)))) `(progn ;; define the flag to make the trivial case easier (defvar ,extension-flag nil) @@ -60,7 +61,7 @@ (add-expression-to-list ',(first characters) %extended-special-char-rules%)) (esrap:change-rule 'extended-special-char - (cons 'or %extended-special-char-rules%)))) + (cons 'or %extended-special-char-rules%)))) ;; define a rule for escaped chars if any ,@ (when escapes `((defrule ,(first escapes) @@ -79,8 +80,15 @@ (member (car a) '(:character-rule :escape-char-rule :md-chars-to-escape - :after :before))) - options)) + :after :before + :bind))) + options)) + ;; add additional bindings needed by extension during printing + ,@(when bind + (loop for (v i) on bind by 'cddr + collect `(setf (alexandria:assoc-value + 3bmd::*additional-bindings* ',v) + (lambda () ,i)))) (setf ,var (add-expression-to-list ',name ,var diff --git a/footnotes.lisp b/footnotes.lisp new file mode 100644 index 0000000..3c8864d --- /dev/null +++ b/footnotes.lisp @@ -0,0 +1,307 @@ +(defpackage #:3bmd-footnotes + (:use :cl :esrap :3bmd-ext) + (:export #:*footnotes*)) + +(in-package #:3bmd-footnotes + +(defvar *footnotes*) + +;; fixme: add an extension API for this +(pushnew '*footnotes* 3bmd::*footers*) + +(defrule footnote (and "[^" (+ (and (! (or #\[ #\])) + 3bmd-grammar::non-space-char)) + "]") + (:function second) + (:text t)) + +;;; ![^foo] parses as an image link, which is assumed to be incorrect +;;; when footnotes are enabled. Add a rule to parse the ! as text in +;;; that case so the [^foo] can be parsed as a footnote. +(define-extension-inline *footnotes* footnote-not-an-image + (and "!" (& footnote)) + (:before 3bmd-grammar::image) + (:function first) + (:text t)) + +(define-extension-inline *footnotes* footnote-ref + (and footnote (! #\:)) + (:function first) + (:lambda (id) + (list 'footnote-ref id))) + +;;; we parse footnote definitions basically like a loose list, but +;;; without a "next list item" +(defrule footnote-block (and (! 3bmd-grammar::blank-line) + 3bmd-grammar::line + (* footnote-block-line)) + (:function rest) + (:destructure (l block) + (text l block))) + +(defrule footnote-continuation-block (and (* 3bmd-grammar::blank-line) + (+ (and 3bmd-grammar::indent + footnote-block))) + (:destructure (b c) + (if b + (cons (text b) (mapcar 'second c)) + (cons :split (mapcar 'second c))))) + +(defrule footnote-block-line (and (! (or 3bmd-grammar::blank-line + 3bmd-grammar::horizontal-rule + (and footnote #\:))) + 3bmd-grammar::optionally-indented-line) + (:function second)) + + +;; hash table of ID -> (ref-name . backref-names) +(defvar *used-footnotes*) +(defvar *expanded-footnotes*) +(defvar *next-ref*) + +(define-extension-block *footnotes* footnote-def + (and (and footnote #\:) + footnote-block + (* footnote-continuation-block)) + (:before 3bmd-grammar::reference) + (:bind *used-footnotes* (make-hash-table :test 'equalp) + *expanded-footnotes* (make-hash-table :test 'equalp) + *next-ref* 1) + (:destructure + ((id c) block cont) + (declare (ignore c)) + (list* 'footnote-def + id + (loop for a in (split-sequence:split-sequence + :split (append (cons block (mapcan 'identity cont)) + (list " + +")) + :remove-empty-subseqs t) + for p = (3bmd::parse-doc (text a)) + ;; we append a node for the back links to last item if + ;; it is a paragraph, or after last item otherwise + if (typep (car (last p)) '(cons (eql :paragraph))) + do (push '(footnote-backlinks) + (cdr (last (car (last p))))) + else do (setf p (append p '((footnote-backlinks)))) + append p)))) + + +(defmethod 3bmd::extract-ref ((id (eql 'footnote-def)) cdr) + (when *footnotes* + (list* (list 'footnote-def (first cdr)) + cdr))) + +(defun walk-def (def) + ;; getting footnotes inside footnotes right is messy: + ;; + ;; we need to include backlinks, even from footnotes we haven't + ;; printed yet + ;; + ;; we don't want backlinks to footnotes that get dropped + ;; + ;; footnotes only used from unused footnotes should be considered + ;; unused + ;; + ;; not sure if otherwise unused recursive footnotes should be + ;; included. I think things are easier if we only consider roots + ;; outside defs, so going with that for now. + ;; + ;; if a footnote has a (new) footnote, we probably want that + ;; footnote to show up next rather than being added to the end? + ;; + ;; so when printing, recursively partially expand the referenced + ;; definition if it hasn't already been expanded. We can't fully + ;; print it since we don't have all the backlinks yet, so just + ;; replace any (footnote-ref x) in the body of the definition with + ;; the printed representation + (labels ((expand-def (def) + (typecase def + ((cons (eql footnote-ref)) + (list 'expanded-ref + (print-tagged-element 'footnote-ref nil (cdr def)))) + (cons + ;; assuming these are small enough that some extra + ;; consing won't matter, other option is to walk twice + ;; to see if we need to expand it? + (mapcar #'expand-def def)) + (t def)))) + (let ((orig (gethash (list 'footnote-def def) 3bmd::*references*))) + (when (and orig (not (gethash def *expanded-footnotes*))) + ;; store something before walking contents so we don't get + ;; stuck in a loop if there is a recursive footnote + (setf (gethash def *expanded-footnotes*) :processing) + (setf (gethash def *expanded-footnotes*) + (expand-def orig)))))) + +(defmethod print-tagged-element ((tag (eql 'footnote-ref)) stream rest) + (let* ((id (car rest)) + (defined (gethash (list 'footnote-def id) 3bmd::*references*))) + (cond + ((not defined) + (format stream "[^~a]" id)) + (t + (let* ((use (gethash id *used-footnotes*)) + (refno (or (first use) + (shiftf *next-ref* (1+ *next-ref*)))) + (backrefs (cddr use)) + (back (if backrefs + (format nil "fnref-~a.~a" refno (1+ (length backrefs))) + (format nil "fnref-~a" refno))) + (fn (or (second use) + (second + (setf (gethash id *used-footnotes*) + (list refno (format nil "fn-~a" refno))))))) + + (push back (cddr (gethash id *used-footnotes*))) + (walk-def id) + (format stream "~a" + fn back + refno)))))) + +(defmethod print-tagged-element ((tag (eql 'footnote-def)) stream rest) + ;; definitions will be printed in the footer, so ignore them here + ) + +(defmethod print-tagged-element ((tag (eql 'expanded-ref)) stream rest) + (format stream "~a" (car rest))) + +(defvar *backlinks* nil) +(defmethod print-tagged-element ((tag (eql 'footnote-backlinks)) stream rest) + (loop for b in *backlinks* + do (format stream " ↩︎" b))) + +(defmethod print-def (s refs (format (eql :html))) + (let ((def (gethash (first refs) *expanded-footnotes*)) + (*backlinks* (reverse (cdddr refs)))) + (format s "
  • " (third refs)) + (3bmd::padded (2 s) + (loop for i in (cdr def) + do (3bmd::print-element i s))) + (format s "
  • "))) + +(defmethod 3bmd::print-footer (stream (f (eql '*footnotes*)) format) + (when (plusp (hash-table-count *used-footnotes*)) + (format stream "
    ") + (3bmd::padded (2 stream) + (format stream "
    ") + (format stream "
      ") + (3bmd::padded (2 stream) + (loop for refs in (sort (alexandria:hash-table-alist *used-footnotes*) + '< :key 'second) + do (print-def stream refs format))) + (format stream "
    ")) + (format stream "
    "))) + + +(defmethod 3bmd::print-md-tagged-element ((tag (eql 'footnote-ref)) stream rest) + (format stream "[^~a]" (first rest))) + +(defmethod 3bmd::print-md-tagged-element ((tag (eql 'footnote-def)) stream rest) + (3bmd::ensure-block stream) + (format stream "~&[^~a]: " (first rest)) + (3bmd::with-md-indent (4) + (loop for i in (cdr rest) + do (3bmd::print-md-element i stream))) + (3bmd::end-block stream)) + +(defmethod 3bmd::print-md-tagged-element ((tag (eql 'footnote-backlinks)) stream rest) + ) + + +(3bmd::pprinter footnote-def (s o) + (format s "[^~a]: ~{~a ~}" (cadr o) (cddr o))) + +(3bmd::pprinter footnote-ref (s o) + (format s "[^~a]" (cadr o))) + +(3bmd::pprinter footnote-backlinks (s o) + (declare (ignore s o))) + +#++ +(let ((*footnotes* t) + (3bmd-code-blocks:*code-blocks* t) + (3bmd-tables:*tables* t) + (s t)) + (format s "~&----~%") + (with-open-file (s "/tmp/foo.html" :if-exists :supersede + :direction :output) + (3bmd:parse-string-and-print-to-stream "a footnote[^ref] ref + +[^ref]: the definition +[^a] multiline + +A footnote in a paragraph[^1] + +| Column1 | Column2[^1] | +| --------- | ------- | +| foot [^a] | note | + +[^a]: a footnote + +``` +some more body text + + +a[^a] + +b + +c + +d + +e + +f + +f +``` + +aHere's a simple footnote,[^1][^bignote] and[^1] here[^1]'s a longer one![^bignote] + + +[^1]: This is the first footnote[^indirect]. + +[^unused]: unused footnote[^double-unused] + +[^indirect]: indirect footnote + +[^double-unused]: unused footnote with a ref from unused footnote + +[^unused-recursive]: unused recursive footnote[^unused-recursive] + +[^umr1]: unused mutually recursive footnote1[^umr2] + +[^umr2]: unused mutually recursive footnote2[^umr1] + +[^mr1]: mutually recursive footnote1[^mr2] + +[^mr2]: mutually recursive footnote2[^mr1] + +[^c1]: footnotes without separating lines1 +[^c2]: footnotes without separating lines2 + +[url1]: http://example.com + +[^bignote]: Here's one with multiple paragraphs and code. missing def[^missing] + + Indent paragraphs to include them in the footnote[^mr2]. + + `{ my code }` [a [^c1] b](http://example.com) [c d](http://example.com) + + * Add as many paragraphs as you like[^c1]. + * list[^c2][^nested] + * [ref url][url1] + * [nested ref url][url2] + + [^nested]: probably doesn't work? + + [url1]: http://example.com + + more text + +end of the body text... + +" s :format :html))) diff --git a/printer.lisp b/printer.lisp index 7a1bfe0..07e88fb 100644 --- a/printer.lisp +++ b/printer.lisp @@ -308,14 +308,21 @@ (error "unknown cons? ~s" elem))) +;;; allow extensions to add other kinds of references, should return a +;;; list (key . value), where key is a list (ext-key ...), and value +;;; is whatever the extension needs +(defmethod extract-ref (id cdr)) +(defmethod extract-ref ((id (eql :reference)) cdr) + (list (print-label-to-string (getf cdr :label)) + (getf cdr :source) + (getf cdr :title))) (defun extract-refs (doc) (alexandria:alist-hash-table (loop for i in doc - when (and (consp i) (eq (car i) :reference)) - collect (list (print-label-to-string (getf (cdr i) :label)) - (getf (cdr i) :source) - (getf (cdr i) :title))) + when (and (consp i) + (extract-ref (car i) (cdr i))) + collect it) :test #'equalp)) (defun expand-tabs (doc &key add-newlines) @@ -336,6 +343,33 @@ (when add-newlines (format s "~%~%")))) +;;; todo: add extension API, and possibly some way to specify a default order? +;; +;; not really expecting multiple extensions with footers to be active +;; at once, so for now just rebind it in desired order if needed. (and +;; file an issue with use case so it can be improved at some point) +;; +;; list of symbols naming extension-flags +(defvar *footers* nil) + +;; extensions that print footers should define methods on this, with +;; extension-flag EQL specialized on the name of the extension flag +;; variable in *footers* +(defmethod print-footer (stream extension-flag format) + ) + +(defun print-footers (stream format) + (loop for flag in *footers* + when (symbol-value flag) + do (print-footer stream flag format))) + +;; alist of (var . initial-value-function) +(defvar *additional-bindings* nil) + +(defmethod print-doc-to-stream-using-format :around (doc stream format) + (progv (map 'list 'car *additional-bindings*) + (map 'list (alexandria:compose 'funcall 'cdr) *additional-bindings*) + (call-next-method))) (defmethod print-doc-to-stream-using-format (doc stream (format (eql :html))) (let ((*references* (extract-refs doc)) @@ -343,6 +377,7 @@ (*padding* *padding*)) (loop for i in doc do (print-element i stream)) + (print-footers stream format) (format stream "~&"))) (defun print-doc-to-stream (doc stream &key (format :html)) diff --git a/tests/extensions/footnotes.lisp b/tests/extensions/footnotes.lisp new file mode 100644 index 0000000..9eaaf42 --- /dev/null +++ b/tests/extensions/footnotes.lisp @@ -0,0 +1,257 @@ +(fiasco:define-test-package #:3bmd-footnotes-tests + (:use #:3bmd-tests) + (:import-from #:3bmd-footnotes + #:footnote + #:*footnotes* + #:footnote-ref + #:footnote-def + #:footnote-backlinks)) + +(in-package #:3bmd-footnotes-tests) + +(3bmd-tests::def-grammar-test footnote-ref1 + :enable-extensions *footnotes* + :rule 3bmd-grammar::%block + :text "a[^ref] reference +" + :expected '(:plain "a" (footnote-ref "ref") + " " "reference")) + +(3bmd-tests::def-grammar-test footnote-def1 + :enable-extensions *footnotes* + :rule 3bmd-grammar::%block + :text "[^def]: definition +" + :expected '(footnote-def "def" + (:paragraph "definition" + (footnote-backlinks)))) + +(3bmd-tests::def-grammar-test footnote-def2 + :enable-extensions *footnotes* + :rule 3bmd-grammar::%block + :text "[^2]: def + + with paragraphs + + ``` + and code + ``` +" + :expected '(footnote-def "2" + (:paragraph "def") + (:paragraph "with" " " "paragraphs") + (:paragraph (:code " +and code +") + (footnote-backlinks)))) + + +(3bmd-tests::def-grammar-test footnote-def3 + ;; backlinks go after last child if it isn't a :paragraph + :enable-extensions *footnotes* + :rule 3bmd-grammar::%block + :text "[^2]: def + + with paragraphs + + * 1 + * 2 +" + :expected '(footnote-def "2" + (:paragraph "def") + (:paragraph "with" " " "paragraphs") + (:bullet-list + (:list-item (:plain "1")) + (:list-item (:plain "2"))) + (footnote-backlinks))) + +(3bmd-tests::def-grammar-test footnote-def4 + :enable-extensions *footnotes* + :rule 3bmd-grammar::%block + :text "[^1 2]: not def +" + :expected '(:plain + (:reference-link :label ("^1" " " "2") + :tail NIL) + ":" " " "not" " " "def")) + + +(3bmd-tests::def-grammar-test footnote-def5 + :enable-extensions *footnotes* + :rule 3bmd-grammar::%block + :text "[^long-id]: def +" + :expected '(footnote-def "long-id" + (:paragraph "def" + (footnote-backlinks)))) + +(3bmd-tests::def-grammar-test footnote-def6 + ;; should parse as 2 definitions + :enable-extensions *footnotes* + :text "[^1]: def1 +[^2]: def2 +" + :expected '((footnote-def "1" + (:paragraph "def1" + (footnote-backlinks))) + (footnote-def "2" + (:paragraph "def2" + (footnote-backlinks))))) + + +(3bmd-tests::def-grammar-test footnote-recursive1 + :enable-extensions *footnotes* + :rule 3bmd-grammar::%block + :text "[^def]: an inline[^def] reference in a def +" + :expected '(footnote-def "def" + (:paragraph "an" " " "inline" + (footnote-ref "def") + " " "reference" " " "in" " " "a" " " + "def" + (footnote-backlinks)))) + +;; should generate a footnote #1, with superscript ¹ linking to +;; "#fn:ref" with id "fnref:ref", and ordered list of definitions +;; in a div with class "footnotes" or similar after all body text +;; +;; possibly add a "see footnote" title? +(def-print-test print-footnotes-1 + :enable-extensions *footnotes* + :text "a footnote[^ref] ref + +[^ref]: the definition + +some more body text +" + :expected "

    a footnote1 ref

    +some more body text
    + +
    1. the definition ↩︎

    +" +) + +(def-print-test print-footnotes-2 + :enable-extensions *footnotes* + :text "[^def]: a recursive[^def] footnote[^def2] + +[^def2]: a normal footnote + +body[^def] 2 +" + :expected "body1 2
    + +
    1. a recursive1 footnote2 ↩︎ ↩︎

    2. + +

      a normal footnote ↩︎

    +") + +;; for multiple references, include multiple backlinks (to distinguish +;; them if keeping original ID in names, add a number after fnref, so +;; "fnref:1" "fnref2:1" "fnref3:1". Other options are just generating +;; sequential numbers for the links and ignoring the supplied ID, or +(def-print-test print-footnotes-3 + :enable-extensions *footnotes* + :text "multiple[^1] references[^1] to same[^1] footnote + +[^1]: a footnote +" + :expected "

    multiple1 references1 to same1 footnote

    + +
    1. a footnote ↩︎ ↩︎ ↩︎

    +") + +;; not sure if this should error or merge the definitions or what? +;; GFM takes first definition, and looks like that's what we ended up +;; with too, so good enough. +(def-print-test print-footnotes-4 + :enable-extensions *footnotes* + :text " a reference[^def] + +[^def]: definition + +[^def]: a duplicate definition +" + :expected "

    a reference1

    + +
    1. definition ↩︎

    +") + +;; drop unused footnotes +(def-print-test print-footnotes-5 + :enable-extensions *footnotes* + :text " a reference[^1] + +[^1]: definition + +[^2]: an unused definition +" + :expected "

    a reference1

    + +
    1. definition ↩︎

    +") + + +;;; footnotes should be appended and numbered in order of use +(def-print-test print-footnotes-6 + :enable-extensions *footnotes* + :text " multiple[^3] unordered[^1] footnotes[^2] + +[^2]: first definition (id 2) third use + +[^3]: second definition (id 3) first use + +[^1]: third definition (id 1) second use + +more content + +" + :expected "

    multiple1 unordered2 footnotes3

    + +

    more content

    + +
    1. second definition (id 3) first use ↩︎

    2. + +

      third definition (id 1) second use ↩︎

    3. + +

      first definition (id 2) third use ↩︎

    +") + +;;; not sure about missing def: some drop completely, some print the +;;; original text of the ref ("[^1]") as if it were not parsed, some +;;; add a footnote with ID as contents +;; +;;; probably just convert back to [^1] in text +(def-print-test print-footnotes-7 + :enable-extensions *footnotes* + :text " a bad reference[^1] a normal reference[^2] + +[^2]: a definition + +body +" + :expected "

    a bad reference[^1] a normal reference1

    +body
    + +
    1. a definition ↩︎

    +") + +;;; make sure we don't generate nested tags. Possibly should parse +;;; as a link with "a [^1] b" as the text instead, or try to split the +;;; link so "a "," b" point to url, and "¹" points to footnote? +;; +;;; fixme: should probably auto-link the URL if not parsing whole +;;; thing as a link? +(def-print-test print-footnotes-8 + :enable-extensions *footnotes* + :text " [a [^1] b](http://example.com) + +[^1]: def + +body +" + :expected "

    [a 1 b](http://example.com)

    +body
    + +
    1. def ↩︎

    +")