Better YAML front matter parsing in commonmark-ext-yaml-front-matter

README.md

@@ -390,7 +390,8 @@ Use class `InsExtension` in artifact `commonmark-ext-ins`.
 
 ### YAML front matter
 
-Adds support for metadata through a YAML front matter block. This extension only supports a subset of YAML syntax. Here's an example of what's supported:
+Adds support for metadata through a YAML front matter block. The extension uses a built-in parser that supports a subset
+of YAML syntax, suitable for simple use cases. Here's an example of what's supported:
 
 ```markdown
 ---
@@ -407,7 +408,36 @@ literal: |
 document start here
 ```
 
-Use class `YamlFrontMatterExtension` in artifact `commonmark-ext-yaml-front-matter`. To fetch metadata, use `YamlFrontMatterVisitor`.
+Use class `YamlFrontMatterExtension` in artifact `commonmark-ext-yaml-front-matter`. To fetch metadata, use `YamlFrontMatterVisitor`:
+
+```java
+import org.commonmark.ext.front.matter.parser.RawContentParser;
+
+List<Extension> extensions = List.of(YamlFrontMatterExtension.create());
+Parser parser = Parser.builder()
+    .extensions(extensions)
+    .build();
+
+Node document = parser.parse(markdownDocument);
+Map<String, List<String>> frontMatter = YamlFrontMatterVisitor.readData(document);
+```
+
+Alternatively, you can use initialize the extension with `RawContentParser` to capture the entire front matter as a
+string for further processing with other tools:
+
+```java
+import org.commonmark.ext.front.matter.parser.RawContentParser;
+
+List<Extension> extensions = List.of(YamlFrontMatterExtension.create(new RawContentParser.Factory()));
+Parser parser = Parser.builder()
+    .extensions(extensions)
+    .build();
+
+Node document = parser.parse(markdownDocument);
+String frontMatter = YamlFrontMatterVisitor.readRawContent(document);
+```
+
+You can also write a custom front matter parser by implementing `FrontMatterParser` interface.
 
 ### Image Attributes
 

commonmark-ext-yaml-front-matter/src/main/java/org/commonmark/ext/front/matter/FrontMatterParser.java

@@ -0,0 +1,33 @@
+package org.commonmark.ext.front.matter;
+
+import org.commonmark.parser.SourceLine;
+
+/**
+ * Parses the content of the front matter block between `---` separators.
+ * The implementations should add at least 1 child node to
+ * {@link YamlFrontMatterBlock} to store the result of the parsing.
+ */
+public interface FrontMatterParser {
+    void onNextLine(YamlFrontMatterBlock block, SourceLine line);
+
+    /**
+     * Notifies about finding the line with the ending separator (`---` or `...`, without
+     * initial whitespace). Advanced parsers may be able to determine that the separator
+     * is a part of the single-/double-quoted multiline string and return {@link SeparatorRole#CONTENT}
+     * to include it in the front matter content. In this case, front matter parsing continues until
+     * finding another separator. To end the parsing, return {@link SeparatorRole#BLOCK_END}.
+     *
+     * @param block Main block node
+     * @param separator `---` or `...`
+     * @return Separator role: block end or the part of the front matter content
+     */
+    SeparatorRole onEndingSeparator(YamlFrontMatterBlock block, SourceLine separator);
+
+    interface Factory {
+        FrontMatterParser create();
+    }
+
+    enum SeparatorRole {
+        BLOCK_END, CONTENT
+    }
+}

commonmark-ext-yaml-front-matter/src/main/java/org/commonmark/ext/front/matter/YamlFrontMatterExtension.java

@@ -1,9 +1,13 @@
 package org.commonmark.ext.front.matter;
 
+import java.util.Objects;
 import java.util.Set;
 import org.commonmark.Extension;
+import org.commonmark.ext.front.matter.parser.RawContentParser;
+import org.commonmark.ext.front.matter.parser.YamlSubsetParser;
 import org.commonmark.ext.front.matter.internal.YamlFrontMatterBlockParser;
 import org.commonmark.ext.front.matter.internal.YamlFrontMatterMarkdownNodeRenderer;
+import org.commonmark.node.Node;
 import org.commonmark.parser.Parser;
 import org.commonmark.renderer.NodeRenderer;
 import org.commonmark.renderer.html.HtmlRenderer;
@@ -18,21 +22,38 @@
  * org.commonmark.parser.Parser.Builder#extensions(Iterable)}, {@link
  * HtmlRenderer.Builder#extensions(Iterable)}).
  *
- * <p>The parsed metadata is turned into {@link YamlFrontMatterNode}. You can access the metadata
- * using {@link YamlFrontMatterVisitor}.
+ * <p>By default, the extension parses the subset of YAML with a built-int {@link YamlSubsetParser}.
+ * The parsed metadata is turned into {@link YamlFrontMatterNode}. You can access
+ * the metadata using {@link YamlFrontMatterVisitor#readData(Node)}.
+ *
+ * <p>Alternatively, you can create the extension with {@link RawContentParser.Factory}.
+ * It turns the YAML front matter into {@link YamlFrontMatterRawContent} node, which stores
+ * the entire front matter as a string. You can access the content with
+ * {@link YamlFrontMatterVisitor#readRawContent(Node)} to process it with other tools.
+ *
+ * <p>Implement {@link FrontMatterParser} interface and the corresponding factory to
+ * parse the front matter with a custom parser.
  */
 public class YamlFrontMatterExtension
         implements Parser.ParserExtension, MarkdownRenderer.MarkdownRendererExtension {
 
-    private YamlFrontMatterExtension() {}
+    private final FrontMatterParser.Factory yamlExtractorFactory;
+
+    private YamlFrontMatterExtension(FrontMatterParser.Factory yamlExtractorFactory) {
+        this.yamlExtractorFactory = Objects.requireNonNull(yamlExtractorFactory);
+    }
 
     @Override
     public void extend(Parser.Builder parserBuilder) {
-        parserBuilder.customBlockParserFactory(new YamlFrontMatterBlockParser.Factory());
+        parserBuilder.customBlockParserFactory(new YamlFrontMatterBlockParser.Factory(yamlExtractorFactory));
     }
 
     public static Extension create() {
-        return new YamlFrontMatterExtension();
+        return create(new YamlSubsetParser.Factory());
+    }
+
+    public static Extension create(FrontMatterParser.Factory extractor) {
+        return new YamlFrontMatterExtension(extractor);
     }
 
     @Override

commonmark-ext-yaml-front-matter/src/main/java/org/commonmark/ext/front/matter/YamlFrontMatterRawContent.java

@@ -0,0 +1,19 @@
+package org.commonmark.ext.front.matter;
+
+import org.commonmark.node.CustomNode;
+
+public class YamlFrontMatterRawContent extends CustomNode {
+    private String content;
+
+    public YamlFrontMatterRawContent(String content) {
+        this.content = content;
+    }
+
+    public String getContent() {
+        return content;
+    }
+
+    public void setContent(String content) {
+        this.content = content;
+    }
+}

commonmark-ext-yaml-front-matter/src/main/java/org/commonmark/ext/front/matter/YamlFrontMatterVisitor.java

@@ -3,28 +3,88 @@
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+
+import org.commonmark.ext.front.matter.parser.YamlSubsetParser;
+import org.commonmark.ext.front.matter.parser.RawContentParser;
 import org.commonmark.node.AbstractVisitor;
 import org.commonmark.node.CustomNode;
+import org.commonmark.node.Node;
 
 public class YamlFrontMatterVisitor extends AbstractVisitor {
+    private boolean present;
     private Map<String, List<String>> data;
+    private String content;
 
     public YamlFrontMatterVisitor() {
         data = new LinkedHashMap<>();
+        content = "";
+        present = false;
+    }
+
+    /**
+     * Reads the YAML front matter metadata, if the Markdown
+     * document has the front matter and the extension
+     * uses {@link YamlSubsetParser} (default).
+     *
+     * @return The data stored in YAML front matter or empty map.
+     */
+    public static Map<String, List<String>> readData(Node document) {
+        YamlFrontMatterVisitor visitor = new YamlFrontMatterVisitor();
+        document.accept(visitor);
+        return visitor.getData();
+    }
+
+    /**
+     * Reads the raw content of the front matter, if the Markdown document has
+     * the front matter and the extension uses {@link RawContentParser}.
+     *
+     * @return Raw content of the front matter as string or empty string.
+     */
+    public static String readRawContent(Node document) {
+        YamlFrontMatterVisitor visitor = new YamlFrontMatterVisitor();
+        document.accept(visitor);
+        return visitor.getRawContent();
     }
 
     @Override
     public void visit(CustomNode customNode) {
         if (customNode instanceof YamlFrontMatterNode) {
             data.put(
                     ((YamlFrontMatterNode) customNode).getKey(),
-                    ((YamlFrontMatterNode) customNode).getValues());
+                    ((YamlFrontMatterNode) customNode).getValues()
+            );
+            present = true;
+        } else if (customNode instanceof YamlFrontMatterRawContent) {
+            content = ((YamlFrontMatterRawContent) customNode).getContent();
+            present = true;
         } else {
             super.visit(customNode);
         }
     }
 
+    /**
+     * Returns the YAML front matter metadata, if the Markdown document has
+     * the front matter and the extension uses {@link YamlSubsetParser}
+     * (default).
+     *
+     * @return The data stored in YAML front matter or empty map
+     */
     public Map<String, List<String>> getData() {
         return data;
     }
+
+    /**
+     * Returns the raw content of the front matter, if the Markdown
+     * document has the front matter and the extension uses
+     * {@link RawContentParser}.
+     *
+     * @return Raw content of the front matter as string or empty string.
+     */
+    public String getRawContent() {
+        return content;
+    }
+
+    public boolean isFrontMatterPresent() {
+        return present;
+    }
 }

commonmark-ext-yaml-front-matter/src/main/java/org/commonmark/ext/front/matter/internal/YamlFrontMatterBlockParser.java

@@ -1,32 +1,23 @@
 package org.commonmark.ext.front.matter.internal;
 
-import java.util.ArrayList;
-import java.util.List;
-import java.util.regex.Matcher;
 import java.util.regex.Pattern;
+
+import org.commonmark.ext.front.matter.FrontMatterParser;
 import org.commonmark.ext.front.matter.YamlFrontMatterBlock;
-import org.commonmark.ext.front.matter.YamlFrontMatterNode;
 import org.commonmark.node.Block;
 import org.commonmark.node.Document;
+import org.commonmark.parser.SourceLine;
 import org.commonmark.parser.block.*;
 
 public class YamlFrontMatterBlockParser extends AbstractBlockParser {
-    private static final Pattern REGEX_METADATA =
-            Pattern.compile("^[ ]{0,3}([A-Za-z0-9._-]+):\\s*(.*)");
-    private static final Pattern REGEX_METADATA_LIST = Pattern.compile("^[ ]+-\\s*(.*)");
-    private static final Pattern REGEX_METADATA_LITERAL = Pattern.compile("^\\s*(.*)");
     private static final Pattern REGEX_BEGIN = Pattern.compile("^-{3}(\\s.*)?");
     private static final Pattern REGEX_END = Pattern.compile("^(-{3}|\\.{3})(\\s.*)?");
 
-    private boolean inLiteral;
-    private String currentKey;
-    private List<String> currentValues;
     private YamlFrontMatterBlock block;
+    private FrontMatterParser frontMatterParser;
 
-    public YamlFrontMatterBlockParser() {
-        inLiteral = false;
-        currentKey = null;
-        currentValues = new ArrayList<>();
+    public YamlFrontMatterBlockParser(FrontMatterParser frontMatterParser) {
+        this.frontMatterParser = frontMatterParser;
         block = new YamlFrontMatterBlock();
     }
 
@@ -37,71 +28,28 @@ public Block getBlock() {
 
     @Override
     public BlockContinue tryContinue(ParserState parserState) {
-        final CharSequence line = parserState.getLine().getContent();
+        final SourceLine line = parserState.getLine();
 
-        if (REGEX_END.matcher(line).matches()) {
-            if (currentKey != null) {
-                block.appendChild(new YamlFrontMatterNode(currentKey, currentValues));
+        if (REGEX_END.matcher(line.getContent()).matches()) {
+            switch (frontMatterParser.onEndingSeparator(block, line)) {
+                case BLOCK_END:
+                    return BlockContinue.finished();
+                case CONTENT:
+                    return BlockContinue.atIndex(parserState.getIndex());
             }
-            return BlockContinue.finished();
         }
 
-        Matcher matcher = REGEX_METADATA.matcher(line);
-        if (matcher.matches()) {
-            if (currentKey != null) {
-                block.appendChild(new YamlFrontMatterNode(currentKey, currentValues));
-            }
-
-            inLiteral = false;
-            currentKey = matcher.group(1);
-            currentValues = new ArrayList<>();
-            String value = matcher.group(2);
-            if ("|".equals(value)) {
-                inLiteral = true;
-            } else if (!"".equals(value)) {
-                currentValues.add(parseString(value));
-            }
-
-            return BlockContinue.atIndex(parserState.getIndex());
-        } else {
-            if (inLiteral) {
-                matcher = REGEX_METADATA_LITERAL.matcher(line);
-                if (matcher.matches()) {
-                    if (currentValues.size() == 1) {
-                        currentValues.set(0, currentValues.get(0) + "\n" + matcher.group(1).trim());
-                    } else {
-                        currentValues.add(matcher.group(1).trim());
-                    }
-                }
-            } else {
-                matcher = REGEX_METADATA_LIST.matcher(line);
-                if (matcher.matches()) {
-                    String value = matcher.group(1);
-                    currentValues.add(parseString(value));
-                }
-            }
-
-            return BlockContinue.atIndex(parserState.getIndex());
-        }
+        frontMatterParser.onNextLine(block, line);
+        return BlockContinue.atIndex(parserState.getIndex());
     }
 
-    private static String parseString(String s) {
-        // Limited parsing of https://yaml.org/spec/1.2.2/#73-flow-scalar-styles
-        // We assume input is well-formed and otherwise treat it as a plain string. In a real
-        // parser, e.g. `'foo` would be invalid because it's missing a trailing `'`.
-        if (s.startsWith("'") && s.endsWith("'")) {
-            String inner = s.substring(1, s.length() - 1);
-            return inner.replace("''", "'");
-        } else if (s.startsWith("\"") && s.endsWith("\"")) {
-            String inner = s.substring(1, s.length() - 1);
-            // Only support escaped `\` and `"`, nothing else.
-            return inner.replace("\\\"", "\"").replace("\\\\", "\\");
-        } else {
-            return s;
+    public static class Factory extends AbstractBlockParserFactory {
+        private FrontMatterParser.Factory frontMatterParserFactory;
+
+        public Factory(FrontMatterParser.Factory factory) {
+            this.frontMatterParserFactory = factory;
         }
-    }
 
-    public static class Factory extends AbstractBlockParserFactory {
         @Override
         public BlockStart tryStart(ParserState state, MatchedBlockParser matchedBlockParser) {
             CharSequence line = state.getLine().getContent();
@@ -110,7 +58,7 @@ public BlockStart tryStart(ParserState state, MatchedBlockParser matchedBlockPar
             if (parentParser.getBlock() instanceof Document
                     && parentParser.getBlock().getFirstChild() == null
                     && REGEX_BEGIN.matcher(line).matches()) {
-                return BlockStart.of(new YamlFrontMatterBlockParser())
+                return BlockStart.of(new YamlFrontMatterBlockParser(frontMatterParserFactory.create()))
                         .atIndex(state.getNextNonSpaceIndex());
             }