Better YAML front matter parsing in commonmark-ext-yaml-front-matter

README.md

@@ -390,7 +390,8 @@ Use class `InsExtension` in artifact `commonmark-ext-ins`.
 
 ### YAML front matter
 
-Adds support for metadata through a YAML front matter block. This extension only supports a subset of YAML syntax. Here's an example of what's supported:
+Adds support for metadata through a YAML front matter block. The extension uses a built-in parser that supports a subset
+of YAML syntax, suitable for simple use cases. Here's an example of what's supported:
 
 ```markdown
 ---
@@ -407,7 +408,36 @@ literal: |
 document start here
 ```
 
-Use class `YamlFrontMatterExtension` in artifact `commonmark-ext-yaml-front-matter`. To fetch metadata, use `YamlFrontMatterVisitor`.
+Use class `YamlFrontMatterExtension` in artifact `commonmark-ext-yaml-front-matter`. To fetch metadata, use `YamlFrontMatterVisitor`:
+
+```java
+import org.commonmark.ext.front.matter.extractor.YamlContentExtractor;
+
+List<Extension> extensions = List.of(YamlFrontMatterExtension.create());
+Parser parser = Parser.builder()
+    .extensions(extensions)
+    .build();
+
+Node document = parser.parse(markdownDocument);
+Map<String, List<String>> frontMatter = YamlFrontMatterVisitor.readData(document);
+```
+
+Alternatively, you can use initialize the extension with `YamlContentExtractor` that saves the YAML front matter content
+as a string for further processing with other tools:
+
+```java
+import org.commonmark.ext.front.matter.extractor.YamlContentExtractor;
+
+List<Extension> extensions = List.of(YamlFrontMatterExtension.create(new YamlContentExtractor.Factory()));
+Parser parser = Parser.builder()
+    .extensions(extensions)
+    .build();
+
+Node document = parser.parse(markdownDocument);
+String frontMatter = YamlFrontMatterVisitor.readContent(document);
+```
+
+You can also write a custom extractor by implementing `YamlFrontMatterExtractor` interface.
 
 ### Image Attributes
 

commonmark-ext-yaml-front-matter/src/main/java/org/commonmark/ext/front/matter/YamlFrontMatterContent.java

@@ -0,0 +1,19 @@
+package org.commonmark.ext.front.matter;
+
+import org.commonmark.node.CustomNode;
+
+public class YamlFrontMatterContent extends CustomNode {
+    private String content;
+
+    public YamlFrontMatterContent(String content) {
+        this.content = content;
+    }
+
+    public String getContent() {
+        return content;
+    }
+
+    public void setContent(String content) {
+        this.content = content;
+    }
+}

commonmark-ext-yaml-front-matter/src/main/java/org/commonmark/ext/front/matter/YamlFrontMatterExtension.java

@@ -1,9 +1,13 @@
 package org.commonmark.ext.front.matter;
 
+import java.util.Objects;
 import java.util.Set;
 import org.commonmark.Extension;
+import org.commonmark.ext.front.matter.extractor.YamlContentExtractor;
+import org.commonmark.ext.front.matter.extractor.YamlDataExtractor;
 import org.commonmark.ext.front.matter.internal.YamlFrontMatterBlockParser;
 import org.commonmark.ext.front.matter.internal.YamlFrontMatterMarkdownNodeRenderer;
+import org.commonmark.node.Node;
 import org.commonmark.parser.Parser;
 import org.commonmark.renderer.NodeRenderer;
 import org.commonmark.renderer.html.HtmlRenderer;
@@ -18,21 +22,38 @@
  * org.commonmark.parser.Parser.Builder#extensions(Iterable)}, {@link
  * HtmlRenderer.Builder#extensions(Iterable)}).
  *
- * <p>The parsed metadata is turned into {@link YamlFrontMatterNode}. You can access the metadata
- * using {@link YamlFrontMatterVisitor}.
+ * <p>By default, the extension parses the subset of YAML with a built-int parser.
+ * The parsed metadata is turned into {@link YamlFrontMatterNode}. You can access
+ * the metadata using {@link YamlFrontMatterVisitor#readData(Node)}.
+ *
+ * <p>Alternatively, you can create the extension with {@link YamlContentExtractor.Factory}.
+ * It turns the YAML front matter into {@link YamlFrontMatterContent} node, which stores
+ * the front matter content as a simple string. You can access the content with
+ * {@link YamlFrontMatterVisitor#readContent(Node)} to process it with other tools.
+ *
+ * <p>To create a custom YAML front matter extractor, implement {@link YamlFrontMatterExtractor}
+ * interface and the corresponding factory.
  */
 public class YamlFrontMatterExtension
         implements Parser.ParserExtension, MarkdownRenderer.MarkdownRendererExtension {
 
-    private YamlFrontMatterExtension() {}
+    private final YamlFrontMatterExtractor.Factory yamlExtractorFactory;
+
+    private YamlFrontMatterExtension(YamlFrontMatterExtractor.Factory yamlExtractorFactory) {
+        this.yamlExtractorFactory = Objects.requireNonNull(yamlExtractorFactory);
+    }
 
     @Override
     public void extend(Parser.Builder parserBuilder) {
-        parserBuilder.customBlockParserFactory(new YamlFrontMatterBlockParser.Factory());
+        parserBuilder.customBlockParserFactory(new YamlFrontMatterBlockParser.Factory(yamlExtractorFactory));
     }
 
     public static Extension create() {
-        return new YamlFrontMatterExtension();
+        return create(new YamlDataExtractor.Factory());
+    }
+
+    public static Extension create(YamlFrontMatterExtractor.Factory extractor) {
+        return new YamlFrontMatterExtension(extractor);
     }
 
     @Override

commonmark-ext-yaml-front-matter/src/main/java/org/commonmark/ext/front/matter/YamlFrontMatterExtractor.java

@@ -0,0 +1,13 @@
+package org.commonmark.ext.front.matter;
+
+import org.commonmark.parser.block.BlockContinue;
+
+public interface YamlFrontMatterExtractor {
+    void onNextLine(YamlFrontMatterBlock block, CharSequence line);
+
+    BlockContinue onBlockEnd(YamlFrontMatterBlock block);
+
+    interface Factory {
+        YamlFrontMatterExtractor create();
+    }
+}

commonmark-ext-yaml-front-matter/src/main/java/org/commonmark/ext/front/matter/YamlFrontMatterVisitor.java

@@ -3,28 +3,89 @@
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+
+import org.commonmark.ext.front.matter.extractor.YamlDataExtractor;
+import org.commonmark.ext.front.matter.extractor.YamlContentExtractor;
 import org.commonmark.node.AbstractVisitor;
 import org.commonmark.node.CustomNode;
+import org.commonmark.node.Node;
 
 public class YamlFrontMatterVisitor extends AbstractVisitor {
+    private boolean present;
     private Map<String, List<String>> data;
+    private String content;
 
     public YamlFrontMatterVisitor() {
         data = new LinkedHashMap<>();
+        content = "";
+        present = false;
+    }
+
+    /**
+     * Reads the YAML front matter metadata, if the Markdown
+     * document has the YAML front matter and the extension
+     * uses {@link YamlDataExtractor} (default).
+     *
+     * @return The data stored in YAML front matter or empty map
+     */
+    public static Map<String, List<String>> readData(Node document) {
+        YamlFrontMatterVisitor visitor = new YamlFrontMatterVisitor();
+        document.accept(visitor);
+        return visitor.getData();
+    }
+
+    /**
+     * Reads the YAML Front Matter metadata as a string, if the Markdown
+     * document has the YAML Front Matter and the extension uses
+     * {@link YamlContentExtractor} (default).
+     *
+     * @return The content of YAML front matter as string or empty string.
+     */
+    public static String readContent(Node document) {
+        YamlFrontMatterVisitor visitor = new YamlFrontMatterVisitor();
+        document.accept(visitor);
+        return visitor.getContent();
     }
 
     @Override
     public void visit(CustomNode customNode) {
         if (customNode instanceof YamlFrontMatterNode) {
             data.put(
                     ((YamlFrontMatterNode) customNode).getKey(),
-                    ((YamlFrontMatterNode) customNode).getValues());
+                    ((YamlFrontMatterNode) customNode).getValues()
+            );
+            present = true;
+        } else if (customNode instanceof YamlFrontMatterContent) {
+            content = ((YamlFrontMatterContent) customNode).getContent();
+            present = true;
         } else {
             super.visit(customNode);
         }
     }
 
+    /**
+     * Returns the YAML front matter metadata, if the Markdown document has
+     * the YAML front matter and the extension uses {@link YamlDataExtractor}
+     * (default).
+     *
+     * @return The data stored in YAML front matter or empty map
+     */
     public Map<String, List<String>> getData() {
         return data;
     }
+
+    /**
+     * Returns the YAML Front Matter metadata as a string, if the Markdown
+     * document has the YAML Front Matter and the extension uses
+     * {@link YamlContentExtractor} (default).
+     *
+     * @return The content of YAML front matter as string or empty string.
+     */
+    public String getContent() {
+        return content;
+    }
+
+    public boolean isFrontMatterPresent() {
+        return present;
+    }
 }

commonmark-ext-yaml-front-matter/src/main/java/org/commonmark/ext/front/matter/extractor/YamlContentExtractor.java

@@ -0,0 +1,32 @@
+package org.commonmark.ext.front.matter.extractor;
+
+import org.commonmark.ext.front.matter.YamlFrontMatterExtractor;
+import org.commonmark.ext.front.matter.YamlFrontMatterBlock;
+import org.commonmark.ext.front.matter.YamlFrontMatterContent;
+import org.commonmark.parser.block.BlockContinue;
+
+public class YamlContentExtractor implements YamlFrontMatterExtractor {
+    private StringBuilder content;
+
+    public YamlContentExtractor() {
+        content = new StringBuilder();
+    }
+
+    @Override
+    public void onNextLine(YamlFrontMatterBlock block, CharSequence line) {
+        content.append(line).append('\n');
+    }
+
+    @Override
+    public BlockContinue onBlockEnd(YamlFrontMatterBlock block) {
+        block.appendChild(new YamlFrontMatterContent(content.toString()));
+        return BlockContinue.finished();
+    }
+
+    public static class Factory implements YamlFrontMatterExtractor.Factory {
+        @Override
+        public YamlFrontMatterExtractor create() {
+            return new YamlContentExtractor();
+        }
+    }
+}

commonmark-ext-yaml-front-matter/src/main/java/org/commonmark/ext/front/matter/extractor/YamlDataExtractor.java

@@ -0,0 +1,96 @@
+package org.commonmark.ext.front.matter.extractor;
+
+import org.commonmark.ext.front.matter.YamlFrontMatterExtractor;
+import org.commonmark.ext.front.matter.YamlFrontMatterBlock;
+import org.commonmark.ext.front.matter.YamlFrontMatterNode;
+import org.commonmark.parser.block.BlockContinue;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class YamlDataExtractor implements YamlFrontMatterExtractor {
+    private static final Pattern REGEX_METADATA =
+            Pattern.compile("^[ ]{0,3}([A-Za-z0-9._-]+):\\s*(.*)");
+    private static final Pattern REGEX_METADATA_LIST = Pattern.compile("^[ ]+-\\s*(.*)");
+    private static final Pattern REGEX_METADATA_LITERAL = Pattern.compile("^\\s*(.*)");
+
+    private boolean inLiteral;
+    private String currentKey;
+    private List<String> currentValues;
+
+    public YamlDataExtractor() {
+        inLiteral = false;
+        currentKey = null;
+        currentValues = new ArrayList<>();
+    }
+
+    @Override
+    public void onNextLine(YamlFrontMatterBlock block, CharSequence line) {
+        Matcher matcher = REGEX_METADATA.matcher(line);
+        if (matcher.matches()) {
+            if (currentKey != null) {
+                block.appendChild(new YamlFrontMatterNode(currentKey, currentValues));
+            }
+
+            inLiteral = false;
+            currentKey = matcher.group(1);
+            currentValues = new ArrayList<>();
+            String value = matcher.group(2);
+            if ("|".equals(value)) {
+                inLiteral = true;
+            } else if (!"".equals(value)) {
+                currentValues.add(parseString(value));
+            }
+        } else {
+            if (inLiteral) {
+                matcher = REGEX_METADATA_LITERAL.matcher(line);
+                if (matcher.matches()) {
+                    if (currentValues.size() == 1) {
+                        currentValues.set(0, currentValues.get(0) + "\n" + matcher.group(1).trim());
+                    } else {
+                        currentValues.add(matcher.group(1).trim());
+                    }
+                }
+            } else {
+                matcher = REGEX_METADATA_LIST.matcher(line);
+                if (matcher.matches()) {
+                    String value = matcher.group(1);
+                    currentValues.add(parseString(value));
+                }
+            }
+        }
+    }
+
+    @Override
+    public BlockContinue onBlockEnd(YamlFrontMatterBlock block) {
+        if (currentKey != null) {
+            block.appendChild(new YamlFrontMatterNode(currentKey, currentValues));
+        }
+        return BlockContinue.finished();
+    }
+
+    private static String parseString(String s) {
+        // Limited parsing of https://yaml.org/spec/1.2.2/#73-flow-scalar-styles
+        // We assume input is well-formed and otherwise treat it as a plain string. In a real
+        // parser, e.g. `'foo` would be invalid because it's missing a trailing `'`.
+        if (s.startsWith("'") && s.endsWith("'")) {
+            String inner = s.substring(1, s.length() - 1);
+            return inner.replace("''", "'");
+        } else if (s.startsWith("\"") && s.endsWith("\"")) {
+            String inner = s.substring(1, s.length() - 1);
+            // Only support escaped `\` and `"`, nothing else.
+            return inner.replace("\\\"", "\"").replace("\\\\", "\\");
+        } else {
+            return s;
+        }
+    }
+
+    public static class Factory implements YamlFrontMatterExtractor.Factory {
+        @Override
+        public YamlFrontMatterExtractor create() {
+            return new YamlDataExtractor();
+        }
+    }
+}