Added pelican-gfm plugin

pelican-gfm/__init__.py

@@ -1,3 +1,2 @@
 #!/usr/bin/environment python -B
-from __future__ import absolute_import
 from .gfm import *

pelican-gfm/gfm.py

@@ -20,7 +20,6 @@
 #
 # gfm_reader.py -- GitHub-Flavored Markdown reader for Pelican
 #
-from __future__ import absolute_import
 import sys
 import os
 import ctypes
@@ -30,8 +29,8 @@
 import pelican.signals
 import pelican.readers
 
-from . import gfmSetup
-from . import Settings
+import gfmSetup
+import Settings
 
 _LIBDIR = Settings.LIBCMARKLOCATION
 _LIBCMARK = 'libcmark-gfm.so'

pelican-gfm/gfmSetup.py

@@ -1,13 +1,11 @@
 #!/usr/bin/python3 -B
-from __future__ import absolute_import
-
 import sys
 import os
 import subprocess
 import shutil
 import tarfile
 import tempfile
-from . import Settings
+import Settings
 
 # Eventually, equivalents for
 # other operating systems / package

pelican-gfm/test.py

@@ -3,10 +3,8 @@
 import unittest
 
 # python2 and python3 differ on how to do this it seems
-try:
-    from .gfm import *
-except ImportError:
-    import gfm
+
+import gfm
 
 class gfmTest(unittest.TestCase):
 

pelican_gfm/Readme.md

@@ -0,0 +1,52 @@
+Pelican GFM
+===========
+
+Pelican GFM is a reader that leverages GitHub's C-based markdown library to translate GitHub Flavored Markdown to html.
+
+## Note
+
+It is important to note that this currently only works on debian systems
+
+Requirements
+============
+
+Pelican GFM has no requirements outside of the python standard library and pelican itself.
+
+How to Use
+=========
+
+Drop the entire pelican-gfm directory into the plugin path you have defined in your pelicanconf.py 
+Ensure that 'pelican-gfm' is on the list of plugins.
+
+## Settings and Configuration
+
+The Settings.py file contains the configurables for pelican-gfm.
+	- Location of the cmark lib files
+	- Version of the GitHub Cmark you wish to use
+	- cmark extensions you wish to use.
+
+## Pre-Flight Checks
+
+Before Pelican GFM run it checks:
+	- that the packages required to build the cmark files are present
+	- that any packages known to conflict with this reader are not present
+	- that the libcmark files required by the reader are present
+
+## Configuring Pelican GFM
+
+If Pelican GFM finds that there is a configuration issue it will recommend that you run the following:
+	`python gfmSetup.py`
+
+## Testing
+
+There is a unittest written for gfm. the test will register and spawn a new reader and return true if there were no issues.
+
+Syntax
+======
+This plugin leverages [GitHub Flavored Markdown](https://github.github.com/gfm/) in `md, markdown, mkd, mdown` files to generate html pages.
+
+
+Attribution
+===========
+`pelican-gfm` is based on [pelican_gfm_script](https://github.com/apache/infrastructure-website/blob/master/gfm_reader.py)
+Originally written by: Greg Stein

pelican_gfm/Settings.py

@@ -0,0 +1,25 @@
+#!/usr/bin/python -B
+
+# Eventual location of the libcmark files
+LIBCMARKLOCATION = "/usr/lib/x86_64-linux-gnu"
+
+# This is the GitHub Cmark version we're using
+VERSION = "0.28.3.gfm.12"
+
+# The GFM extensions that we want to use
+EXTENSIONS = (
+    'autolink',
+    'table',
+    'strikethrough',
+    'tagfilter',
+)
+OPTS = 0
+# Do not change these
+# Do not change these
+# Do not change these
+
+# This is the archive of GitHub's cmark-gfm files
+ARCHIVES = "https://github.com/github/cmark-gfm/archive"
+# The name of the local tarball that will be downloaded
+LOCAL = "cmark-gfm.VERSION.orig.tar.gz"
+

pelican_gfm/__init__.py

@@ -0,0 +1,3 @@
+#!/usr/bin/environment python -B
+from __future__ import absolute_import
+from .gfm import *

pelican_gfm/gfm.py

@@ -0,0 +1,221 @@
+#!/usr/bin/python -B
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#
+# gfm_reader.py -- GitHub-Flavored Markdown reader for Pelican
+#
+from __future__ import absolute_import
+import sys
+import os
+import ctypes
+import time
+import re
+import pelican.utils
+import pelican.signals
+import pelican.readers
+
+from . import gfmSetup
+from . import Settings
+
+_LIBDIR = Settings.LIBCMARKLOCATION
+_LIBCMARK = 'libcmark-gfm.so'
+_LIBEXT = 'libcmark-gfmextensions.so'
+try:
+    cmark = ctypes.CDLL(os.path.join(_LIBDIR, _LIBCMARK))
+    cmark_ext = ctypes.CDLL(os.path.join(_LIBDIR, _LIBEXT))
+except OSError:
+    raise ImportError(
+                      '%s not found. run python3 gfmSetup.py as a\
+ user with write permisions to %s or change LIBCMARKLOCATION\
+ in Settings.py'
+                      % (_LIBCMARK, _LIBDIR)
+                      )
+
+# Use ctypes to access the functions in libcmark-gfm
+F_cmark_parser_new = cmark.cmark_parser_new
+F_cmark_parser_new.restype = ctypes.c_void_p
+F_cmark_parser_new.argtypes = (ctypes.c_int,)
+
+F_cmark_parser_feed = cmark.cmark_parser_feed
+F_cmark_parser_feed.restype = None
+F_cmark_parser_feed.argtypes = (
+                                ctypes.c_void_p,
+                                ctypes.c_char_p,
+                                ctypes.c_size_t
+                                )
+
+F_cmark_parser_finish = cmark.cmark_parser_finish
+F_cmark_parser_finish.restype = ctypes.c_void_p
+F_cmark_parser_finish.argtypes = (ctypes.c_void_p,)
+
+F_cmark_parser_attach_syntax_extension = cmark.cmark_parser_attach_syntax_extension
+F_cmark_parser_attach_syntax_extension.restype = ctypes.c_int
+F_cmark_parser_attach_syntax_extension.argtypes = (
+                                                    ctypes.c_void_p,
+                                                    ctypes.c_void_p
+                                                    )
+
+F_cmark_parser_get_syntax_extensions = cmark.cmark_parser_get_syntax_extensions
+F_cmark_parser_get_syntax_extensions.restype = ctypes.c_void_p
+F_cmark_parser_get_syntax_extensions.argtypes = (ctypes.c_void_p,)
+
+F_cmark_parser_free = cmark.cmark_parser_free
+F_cmark_parser_free.restype = None
+F_cmark_parser_free.argtypes = (ctypes.c_void_p,)
+
+F_cmark_node_free = cmark.cmark_node_free
+F_cmark_node_free.restype = None
+F_cmark_node_free.argtypes = (ctypes.c_void_p,)
+
+F_cmark_find_syntax_extension = cmark.cmark_find_syntax_extension
+F_cmark_find_syntax_extension.restype = ctypes.c_void_p
+F_cmark_find_syntax_extension.argtypes = (ctypes.c_char_p,)
+
+F_cmark_render_html = cmark.cmark_render_html
+F_cmark_render_html.restype = ctypes.c_char_p
+F_cmark_render_html.argtypes = (ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p)
+
+
+# Set up the libcmark-gfm library and its extensions
+F_register = cmark_ext.core_extensions_ensure_registered
+F_register.restype = None
+F_register.argtypes = ()
+F_register()
+
+# technically, maybe install an atexit() to release the plugins
+
+
+class GFMReader(pelican.readers.BaseReader):
+    enabled = True
+    """GitHub Flavored Markdown Reader for the Pelican system.
+
+    Pelican looks for all subclasses of BaseReader, and automatically
+    registers them for the file extensions listed below. Thus, nothing
+    further is required by users of this Reader.
+    """
+
+    # NOTE: the builtin MarkdownReader must be disabled. Otherwise, it will be
+    #       non-deterministic which Reader will be used for these files.
+    file_extensions = ['md', 'markdown', 'mkd', 'mdown']
+
+    # Metadata is specified as a single, colon-separated line, such as:
+    #
+    # Title: this is the title
+    #
+    # Note: name starts in column 0, no whitespace before colon, will be
+    #       made lower-case, and value will be stripped
+    #
+    RE_METADATA = re.compile('^([A-za-z]+): (.*)$')
+
+    def read(self, source_path):
+        # Prepare the "slug", which is the target file name.
+        # It will be the same as the source file, minus the leading
+        # ".../content/(articles|pages)"
+        # and with the extension removed (Pelican will add .html)
+        relpath = os.path.relpath(source_path, self.settings['PATH'])
+        parts = relpath.split(os.sep)
+
+        # split off ext, keep base
+        parts[-1] = os.path.splitext(parts[-1])[0]
+        slug = os.sep.join(parts[1:])
+
+        metadata = {
+            'slug': slug,
+        }
+
+        # Fetch the source content, with a few appropriate tweaks
+        with pelican.utils.pelican_open(source_path) as text:
+
+            # Extract the metadata from the header of the text
+            lines = text.splitlines()
+            for i in range(len(lines)):
+                line = lines[i]
+                match = GFMReader.RE_METADATA.match(line)
+                if match:
+                    name = match.group(1).strip().lower()
+                    if name != 'slug':
+                        value = match.group(2).strip()
+                        if name == 'date':
+                            value = pelican.utils.get_date(value)
+                    metadata[name] = value
+                    # if name != 'title':
+                    #  print 'META:', name, value
+                elif not line.strip():
+                    # blank line
+                    continue
+                else:
+                    # reached actual content
+                    break
+
+            # Reassemble content, minus the metadata
+            text = '\n'.join(lines[i:])
+
+            # Render the markdown into HTML
+            if sys.version_info >= (3, 0):
+                text = text.encode('utf-8')
+                content = self.render(text).decode('utf-8')
+            else:
+                content = self.render(text)
+
+        # Redo the slug for articles.
+        if parts[0] == 'articles' and 'title' in metadata:
+            metadata['slug'] = pelican.utils.slugify(
+                                metadata['title'],
+                                self.settings.get(
+                                 'SLUG_SUBSTITUTIONS',
+                                 ()
+                                 )
+                                )
+
+        return content, metadata
+
+    def render(self, text):
+        "Use cmark-gfm to render the Markdown into an HTML fragment."
+
+        parser = F_cmark_parser_new(Settings.OPTS)
+        assert parser
+        for name in Settings.EXTENSIONS:
+            ext = F_cmark_find_syntax_extension(name.encode('utf-8'))
+            assert ext
+            rv = F_cmark_parser_attach_syntax_extension(parser, ext)
+            assert rv
+        exts = F_cmark_parser_get_syntax_extensions(parser)
+        F_cmark_parser_feed(parser, text, len(text))
+        doc = F_cmark_parser_finish(parser)
+        assert doc
+        output = F_cmark_render_html(doc, Settings.OPTS, exts)
+        F_cmark_parser_free(parser)
+        F_cmark_node_free(doc)
+        return output
+
+
+def add_readers(readers):
+    msg = "GFM plugin cannot find the required libcmark files.\
+ Please run python3 gfmSetup.py as a user with write permission \
+ to the directory into which the libcmark files will be placed."
+    if gfmSetup.test_configuration():
+        readers.reader_classes['md'] = GFMReader
+        return True
+    else:
+        raise Exception(msg)
+
+
+def register():
+    reader = pelican.signals.readers_init.connect(add_readers)
+    return(reader)