From 2f96d357c5ef2532d151b6ba1b098329b7d532e6 Mon Sep 17 00:00:00 2001 From: dfoulks Date: Fri, 11 Oct 2019 09:55:23 -0500 Subject: [PATCH 01/24] Added pelican-gfm plugin There were a few flake8 exceptions in pelican-gfm: - __init__.py imports gfm.* but does not call it. - a line in gfm.py is too long (left unshortened) The python unittest ran without error Created a Readme.md file for this plugin. 10k foot view Updated the Readme.rst in the pelican-plugins root. --- Readme.rst | 2 + pelican-gfm/Readme.md | 22 +++++ pelican-gfm/__init__.py | 3 + pelican-gfm/gfm.py | 208 ++++++++++++++++++++++++++++++++++++++++ pelican-gfm/gfmSetup.py | 111 +++++++++++++++++++++ 5 files changed, 346 insertions(+) create mode 100644 pelican-gfm/Readme.md create mode 100644 pelican-gfm/__init__.py create mode 100644 pelican-gfm/gfm.py create mode 100644 pelican-gfm/gfmSetup.py diff --git a/Readme.rst b/Readme.rst index 9195f24c1..82998c6a4 100644 --- a/Readme.rst +++ b/Readme.rst @@ -187,6 +187,8 @@ pelican-ert Allows you to add estimated reading time of an article Pelican-flickr Brings your Flickr photos & sets into your static website +pelican-gfm A reader that uses GitHub's C based cmark libraries to translate .md files + Pelican Genealogy Add surnames and people so metadata and context can be accessed from within a theme to provide surname and person pages Pelican Gist tag Easily embed GitHub Gists in your Pelican articles diff --git a/pelican-gfm/Readme.md b/pelican-gfm/Readme.md new file mode 100644 index 000000000..eb30a3c65 --- /dev/null +++ b/pelican-gfm/Readme.md @@ -0,0 +1,22 @@ +pelican-gfm +=========== +A reader that leverages GitHub's C-based markdown library to translate GitHub Flavored Markdown to html. + +Requirements +============ + +pelican-gfm has no requirements that are outside of the python standard library aside from pelican itself. + +How to Use +========= +Drop the entire pelican-gfm directory into the plugin path and invoke it from your pelicanconf.py to have pelican-gfm render contents/\*.md + +Syntax +====== +This plugin leverages [GitHub Flavored Markdown](https://github.github.com/gfm/) in `.md` files to generate html pages. + + +Attribution +=========== +`pelican-gfm` is based on [pelican_gfm_script](https://github.com/apache/infrastructure-website/blob/master/gfm_reader.py) +Originally written by: Greg Stein diff --git a/pelican-gfm/__init__.py b/pelican-gfm/__init__.py new file mode 100644 index 000000000..015c9028f --- /dev/null +++ b/pelican-gfm/__init__.py @@ -0,0 +1,3 @@ +#!/usr/bin/environment python -B + +from gfm import * diff --git a/pelican-gfm/gfm.py b/pelican-gfm/gfm.py new file mode 100644 index 000000000..e5480151b --- /dev/null +++ b/pelican-gfm/gfm.py @@ -0,0 +1,208 @@ +#!/usr/bin/python -B +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# +# gfm_reader.py -- GitHub-Flavored Markdown reader for Pelican +# + +import sys +import os +import ctypes +import re +import gfmSetup +import pelican.utils +import pelican.signals +import pelican.readers + +_LIBDIR = gfmSetup.LIBCMARKLOCATION +_LIBCMARK = 'libcmark-gfm.so' +_LIBEXT = 'libcmark-gfmextensions.so' +try: + cmark = ctypes.CDLL(os.path.join(_LIBDIR, _LIBCMARK)) + cmark_ext = ctypes.CDLL(os.path.join(_LIBDIR, _LIBEXT)) +except OSError: + raise ImportError('%s not found. see build-mark.sh and gfm_reader.py' + % _LIBCMARK) + +# Use ctypes to access the functions in libcmark-gfm +F_cmark_parser_new = cmark.cmark_parser_new +F_cmark_parser_new.restype = ctypes.c_void_p +F_cmark_parser_new.argtypes = (ctypes.c_int,) + +F_cmark_parser_feed = cmark.cmark_parser_feed +F_cmark_parser_feed.restype = None +F_cmark_parser_feed.argtypes = (ctypes.c_void_p, + ctypes.c_char_p, + ctypes.c_size_t,) + +F_cmark_parser_finish = cmark.cmark_parser_finish +F_cmark_parser_finish.restype = ctypes.c_void_p +F_cmark_parser_finish.argtypes = (ctypes.c_void_p,) + +F_cmark_parser_attach_syntax_extension = cmark.cmark_parser_attach_syntax_extension +F_cmark_parser_attach_syntax_extension.restype = ctypes.c_int +F_cmark_parser_attach_syntax_extension.argtypes = (ctypes.c_void_p, + ctypes.c_void_p,) + +F_cmark_parser_get_syntax_extensions = cmark.cmark_parser_get_syntax_extensions +F_cmark_parser_get_syntax_extensions.restype = ctypes.c_void_p +F_cmark_parser_get_syntax_extensions.argtypes = (ctypes.c_void_p,) + +F_cmark_parser_free = cmark.cmark_parser_free +F_cmark_parser_free.restype = None +F_cmark_parser_free.argtypes = (ctypes.c_void_p,) + +F_cmark_node_free = cmark.cmark_node_free +F_cmark_node_free.restype = None +F_cmark_node_free.argtypes = (ctypes.c_void_p,) + +F_cmark_find_syntax_extension = cmark.cmark_find_syntax_extension +F_cmark_find_syntax_extension.restype = ctypes.c_void_p +F_cmark_find_syntax_extension.argtypes = (ctypes.c_char_p,) + +F_cmark_render_html = cmark.cmark_render_html +F_cmark_render_html.restype = ctypes.c_char_p +F_cmark_render_html.argtypes = (ctypes.c_void_p, + ctypes.c_int, + ctypes.c_void_p,) + + +# Set up the libcmark-gfm library and its extensions +F_register = cmark_ext.core_extensions_ensure_registered +F_register.restype = None +F_register.argtypes = () +F_register() + +# technically, maybe install an atexit() to release the plugins + +# Options for the GFM rendering call +# this could be moved into SETTINGS or somesuch, but meh. not needed now. +OPTS = 0 + +# The GFM extensions that we want to use +EXTENSIONS = ( + 'autolink', + 'table', + 'strikethrough', + 'tagfilter', +) + + +class GFMReader(pelican.readers.BaseReader): + enabled = True + """GFM-flavored Reader for the Pelican system. + + Pelican looks for all subclasses of BaseReader, and automatically + registers them for the file extensions listed below. Thus, nothing + further is required by users of this Reader. + """ + + # NOTE: the builtin MarkdownReader must be disabled. Otherwise, it will be + # non-deterministic which Reader will be used for these files. + file_extensions = ['md', 'markdown', 'mkd', 'mdown'] + + # Metadata is specified as a single, colon-separated line, such as: + # + # Title: this is the title + # + # Note: name starts in column 0, no whitespace before colon, will be + # made lower-case, and value will be stripped + # + RE_METADATA = re.compile('^([A-za-z]+): (.*)$') + + def read(self, source_path): + # Prepare the "slug", which is the target file name. It will be the + # same as the source file minus the leading: + # ".../content/(articles|pages)" + # and with the extension removed (Pelican will add .html) + relpath = os.path.relpath(source_path, self.settings['PATH']) + parts = relpath.split(os.sep) + parts[-1] = os.path.splitext(parts[-1])[0] # split off ext, keep base + slug = os.sep.join(parts[1:]) + metadata = { + 'slug': slug, + } + + # Fetch the source content, with a few appropriate tweaks + with pelican.utils.pelican_open(source_path) as text: + # Extract the metadata from the header of the text + lines = text.splitlines() + for i in range(len(lines)): + line = lines[i] + match = GFMReader.RE_METADATA.match(line) + if match: + name = match.group(1).strip().lower() + if name != 'slug': + value = match.group(2).strip() + if name == 'date': + value = pelican.utils.get_date(value) + metadata[name] = value + # if name != 'title': + # print 'META:', name, value + elif not line.strip(): + # blank line + continue + else: + # reached actual content + break + # Reassemble content, minus the metadata + text = '\n'.join(lines[i:]) + content = self.render(text) + # Redo the slug for articles. + if parts[0] == 'articles' and 'title' in metadata: + metadata['slug'] = pelican.utils.slugify( + metadata['title'], + self.settings.get('SLUG_SUBSTITUTIONS', ())) + return content, metadata + + def render(self, text): + "Use cmark-gfm to render the Markdown into an HTML fragment." + + parser = F_cmark_parser_new(OPTS) + assert parser + for name in EXTENSIONS: + ext = F_cmark_find_syntax_extension(name) + assert ext + rv = F_cmark_parser_attach_syntax_extension(parser, ext) + assert rv + exts = F_cmark_parser_get_syntax_extensions(parser) + + F_cmark_parser_feed(parser, text, len(text)) + doc = F_cmark_parser_finish(parser) + assert doc + + output = F_cmark_render_html(doc, OPTS, exts) + + F_cmark_parser_free(parser) + F_cmark_node_free(doc) + + return output + + +def add_readers(readers): + if str(gfmSetup.test_configuration()) == "0": + readers.reader_classes['md'] = GFMReader + else: + # Optionally we can have the gfmSetup script run here + # as root and configure the system + # Probably just easier to run it independently tho. + sys.exit(1) + + +def register(): + pelican.signals.readers_init.connect(add_readers) diff --git a/pelican-gfm/gfmSetup.py b/pelican-gfm/gfmSetup.py new file mode 100644 index 000000000..ccc75fbd3 --- /dev/null +++ b/pelican-gfm/gfmSetup.py @@ -0,0 +1,111 @@ +#!/usr/bin/python -B + +import os +import subprocess + +# This gets used by the gfm plugin as well as the check_configure function +LIBCMARKLOCATION = "/usr/lib/x86_64-linux-gnu" + +ARCHIVES = "https://github.com/github/cmark-gfm/archive" +VERSION = "0.28.3.gfm.12" +LOCAL = "cmark-gfm.$VERSION.orig.tar.gz" +WORKSPACE = '/tmp/build-cmark' + + +def dpkg_installed(package): + t1 = subprocess.Popen(["dpkg", "-l"], stdout=subprocess.PIPE) + t2 = subprocess.Popen(["grep", "-q", package], + stdout=subprocess.PIPE, + stdin=t1.stdout,) + ec = t2.wait() + return ec + + +def test_setup(): + installed = ["cmake", "make", "wget"] + removed = ["libcmark-gfm-dev", + "libcmark-gfm-extensions-dev", + "libcmark-gfm0", + "libcmark-gfm-extensions0", ] + for package in installed: + if str(dpkg_installed(package)) == "1": + print(package + " not installed") + return 1 + + for package in removed: + if dpkg_installed == "0": + print(package + " needs removed") + return 1 + + +def apt_install(package): + # I need to be able to do this a better, in a less sudo + apt-y way + subprocess.call(["apt-get", "install", package, "-y"]) + + +def apt_remove(package): + # I need to be able to do this a better, in a less sudo + apt-y way + subprocess.call(["apt-get", "purge", package, "-y"]) + + +def cleanUp(): + subprocess.call(["rm", "-rf", WORKSPACE]) + + +def setup(): + test_setup() + # Configure the environment if it's not already configured + if not os.path.isdir(WORKSPACE): + os.mkdir(WORKSPACE) + subprocess.call(["wget", + "--quiet", + ARCHIVES + "/" + VERSION + ".tar.gz", WORKSPACE, + "-P", + WORKSPACE]) + subprocess.call(['tar', + 'zxf', + WORKSPACE + "/" + VERSION + ".tar.gz", + "-C", + WORKSPACE]) + BUILDSPACE = WORKSPACE + "/" + "cmark-gfm-" + VERSION + "/build" + if not os.path.isdir(BUILDSPACE): + os.mkdir(BUILDSPACE) + thing1 = subprocess.Popen(["cmake", + "-DCMARK_TESTS=OFF", + "-DCMARK_STATIC=OFF", + ".."], + cwd=BUILDSPACE) + thing1.wait() + + thing2 = subprocess.Popen(["make"], cwd=BUILDSPACE) + thing2.wait() + + # Move the libcmark.so artifacts in place + print("Moving files") + gfmfile = BUILDSPACE+"/src/libcmark-gfm.so."+VERSION + gfmextfile = BUILDSPACE+"/extensions/libcmark-gfmextensions.so."+VERSION + subprocess.call(["mv", + gfmfile, + LIBCMARKLOCATION + "libcmark-gfm.so"]) + subprocess.call(["mv", + gfmextfile, + LIBCMARKLOCATION + "libcmark-gfmextensions.so"]) + + +def test_configuration(): + gfmfile = LIBCMARKLOCATION + "/libcmark-gfm.so" + gfmextfile = LIBCMARKLOCATION + "/libcmark-gfmextensions.so" + if os.path.isfile(gfmfile) and os.path.isfile(gfmextfile): + return 0 + else: + return 1 + + +def configure(): + print("Configuring!!!") + setup() + cleanUp() + + +if __name__ == "__main__": + configure() From 140ab9fd0f12eeb3f81b88828bab399f5739609a Mon Sep 17 00:00:00 2001 From: dfoulks Date: Mon, 16 Dec 2019 12:18:47 -0600 Subject: [PATCH 02/24] Made some changes to the pelican gfm setup script trying to make it a little more terse and get rid of some of the junk that was in there. Also putting better documentation in the functions. --- pelican-gfm/gfm.py | 222 +++++++++++++++++++++------------------- pelican-gfm/gfmSetup.py | 127 +++++++++++------------ 2 files changed, 173 insertions(+), 176 deletions(-) diff --git a/pelican-gfm/gfm.py b/pelican-gfm/gfm.py index e5480151b..7bda6c9a0 100644 --- a/pelican-gfm/gfm.py +++ b/pelican-gfm/gfm.py @@ -23,6 +23,7 @@ import sys import os import ctypes +import time import re import gfmSetup import pelican.utils @@ -33,11 +34,11 @@ _LIBCMARK = 'libcmark-gfm.so' _LIBEXT = 'libcmark-gfmextensions.so' try: - cmark = ctypes.CDLL(os.path.join(_LIBDIR, _LIBCMARK)) - cmark_ext = ctypes.CDLL(os.path.join(_LIBDIR, _LIBEXT)) + cmark = ctypes.CDLL(os.path.join(_LIBDIR, _LIBCMARK)) + cmark_ext = ctypes.CDLL(os.path.join(_LIBDIR, _LIBEXT)) except OSError: - raise ImportError('%s not found. see build-mark.sh and gfm_reader.py' - % _LIBCMARK) + raise ImportError('%s not found. see build-mark.sh and gfm_reader.py' + % _LIBCMARK) # Use ctypes to access the functions in libcmark-gfm F_cmark_parser_new = cmark.cmark_parser_new @@ -46,9 +47,7 @@ F_cmark_parser_feed = cmark.cmark_parser_feed F_cmark_parser_feed.restype = None -F_cmark_parser_feed.argtypes = (ctypes.c_void_p, - ctypes.c_char_p, - ctypes.c_size_t,) +F_cmark_parser_feed.argtypes = (ctypes.c_void_p, ctypes.c_char_p, ctypes.c_size_t) F_cmark_parser_finish = cmark.cmark_parser_finish F_cmark_parser_finish.restype = ctypes.c_void_p @@ -56,8 +55,7 @@ F_cmark_parser_attach_syntax_extension = cmark.cmark_parser_attach_syntax_extension F_cmark_parser_attach_syntax_extension.restype = ctypes.c_int -F_cmark_parser_attach_syntax_extension.argtypes = (ctypes.c_void_p, - ctypes.c_void_p,) +F_cmark_parser_attach_syntax_extension.argtypes = (ctypes.c_void_p, ctypes.c_void_p) F_cmark_parser_get_syntax_extensions = cmark.cmark_parser_get_syntax_extensions F_cmark_parser_get_syntax_extensions.restype = ctypes.c_void_p @@ -77,21 +75,19 @@ F_cmark_render_html = cmark.cmark_render_html F_cmark_render_html.restype = ctypes.c_char_p -F_cmark_render_html.argtypes = (ctypes.c_void_p, - ctypes.c_int, - ctypes.c_void_p,) +F_cmark_render_html.argtypes = (ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p) # Set up the libcmark-gfm library and its extensions F_register = cmark_ext.core_extensions_ensure_registered F_register.restype = None -F_register.argtypes = () +F_register.argtypes = ( ) F_register() -# technically, maybe install an atexit() to release the plugins +### technically, maybe install an atexit() to release the plugins # Options for the GFM rendering call -# this could be moved into SETTINGS or somesuch, but meh. not needed now. +### this could be moved into SETTINGS or somesuch, but meh. not needed now. OPTS = 0 # The GFM extensions that we want to use @@ -104,105 +100,115 @@ class GFMReader(pelican.readers.BaseReader): - enabled = True - """GFM-flavored Reader for the Pelican system. - - Pelican looks for all subclasses of BaseReader, and automatically - registers them for the file extensions listed below. Thus, nothing - further is required by users of this Reader. - """ - - # NOTE: the builtin MarkdownReader must be disabled. Otherwise, it will be - # non-deterministic which Reader will be used for these files. - file_extensions = ['md', 'markdown', 'mkd', 'mdown'] - - # Metadata is specified as a single, colon-separated line, such as: - # - # Title: this is the title - # - # Note: name starts in column 0, no whitespace before colon, will be - # made lower-case, and value will be stripped - # - RE_METADATA = re.compile('^([A-za-z]+): (.*)$') - - def read(self, source_path): - # Prepare the "slug", which is the target file name. It will be the - # same as the source file minus the leading: - # ".../content/(articles|pages)" - # and with the extension removed (Pelican will add .html) - relpath = os.path.relpath(source_path, self.settings['PATH']) - parts = relpath.split(os.sep) - parts[-1] = os.path.splitext(parts[-1])[0] # split off ext, keep base - slug = os.sep.join(parts[1:]) - metadata = { - 'slug': slug, - } - - # Fetch the source content, with a few appropriate tweaks - with pelican.utils.pelican_open(source_path) as text: - # Extract the metadata from the header of the text - lines = text.splitlines() - for i in range(len(lines)): - line = lines[i] - match = GFMReader.RE_METADATA.match(line) - if match: - name = match.group(1).strip().lower() - if name != 'slug': - value = match.group(2).strip() - if name == 'date': - value = pelican.utils.get_date(value) - metadata[name] = value - # if name != 'title': - # print 'META:', name, value - elif not line.strip(): - # blank line - continue - else: - # reached actual content - break - # Reassemble content, minus the metadata - text = '\n'.join(lines[i:]) - content = self.render(text) - # Redo the slug for articles. - if parts[0] == 'articles' and 'title' in metadata: - metadata['slug'] = pelican.utils.slugify( - metadata['title'], - self.settings.get('SLUG_SUBSTITUTIONS', ())) - return content, metadata - - def render(self, text): - "Use cmark-gfm to render the Markdown into an HTML fragment." - - parser = F_cmark_parser_new(OPTS) - assert parser - for name in EXTENSIONS: - ext = F_cmark_find_syntax_extension(name) - assert ext - rv = F_cmark_parser_attach_syntax_extension(parser, ext) - assert rv - exts = F_cmark_parser_get_syntax_extensions(parser) - - F_cmark_parser_feed(parser, text, len(text)) - doc = F_cmark_parser_finish(parser) - assert doc - - output = F_cmark_render_html(doc, OPTS, exts) - - F_cmark_parser_free(parser) - F_cmark_node_free(doc) - - return output + enabled = True + """GFM-flavored Reader for the Pelican system. + + Pelican looks for all subclasses of BaseReader, and automatically + registers them for the file extensions listed below. Thus, nothing + further is required by users of this Reader. + """ + + # NOTE: the builtin MarkdownReader must be disabled. Otherwise, it will be + # non-deterministic which Reader will be used for these files. + file_extensions = ['md', 'markdown', 'mkd', 'mdown'] + + # Metadata is specified as a single, colon-separated line, such as: + # + # Title: this is the title + # + # Note: name starts in column 0, no whitespace before colon, will be + # made lower-case, and value will be stripped + # + RE_METADATA = re.compile('^([A-za-z]+): (.*)$') + + def read(self, source_path): + # Prepare the "slug", which is the target file name. It will be the + # same as the source file, minus the leading ".../content/(articles|pages)" + # and with the extension removed (Pelican will add .html) + relpath = os.path.relpath(source_path, self.settings['PATH']) + parts = relpath.split(os.sep) + parts[-1] = os.path.splitext(parts[-1])[0] # split off ext, keep base + slug = os.sep.join(parts[1:]) + + metadata = { + 'slug': slug, + } + + + + # Fetch the source content, with a few appropriate tweaks + with pelican.utils.pelican_open(source_path) as text: + + # Extract the metadata from the header of the text + lines = text.splitlines() + for i in range(len(lines)): + line = lines[i] + match = GFMReader.RE_METADATA.match(line) + if match: + name = match.group(1).strip().lower() + if name != 'slug': + value = match.group(2).strip() + if name == 'date': + value = pelican.utils.get_date(value) + metadata[name] = value + #if name != 'title': + # print 'META:', name, value + elif not line.strip(): + # blank line + continue + else: + # reached actual content + break + + # Reassemble content, minus the metadata + text = '\n'.join(lines[i:]) + + # Render the markdown into HTML +# if sys.version_info >= (3, 0): +# text = text.encode('utf-8') +# content = self.render(text).decode('utf-8') +# else: + content = self.render(text) + + # Redo the slug for articles. + if parts[0] == 'articles' and 'title' in metadata: + metadata['slug'] = pelican.utils.slugify( + metadata['title'], + self.settings.get('SLUG_SUBSTITUTIONS', ())) + + return content, metadata + + def render(self, text): + "Use cmark-gfm to render the Markdown into an HTML fragment." + + parser = F_cmark_parser_new(OPTS) + assert parser + for name in EXTENSIONS: + ext = F_cmark_find_syntax_extension(name) + assert ext + rv = F_cmark_parser_attach_syntax_extension(parser, ext) + assert rv + exts = F_cmark_parser_get_syntax_extensions(parser) + + F_cmark_parser_feed(parser, text, len(text)) + doc = F_cmark_parser_finish(parser) + assert doc + + output = F_cmark_render_html(doc, OPTS, exts) + + F_cmark_parser_free(parser) + F_cmark_node_free(doc) + + return output def add_readers(readers): if str(gfmSetup.test_configuration()) == "0": - readers.reader_classes['md'] = GFMReader + readers.reader_classes['md'] = GFMReader else: - # Optionally we can have the gfmSetup script run here - # as root and configure the system + # Optionally we can have the gfmSetup script run here as root and configure the system # Probably just easier to run it independently tho. sys.exit(1) - def register(): - pelican.signals.readers_init.connect(add_readers) + pelican.signals.readers_init.connect(add_readers) diff --git a/pelican-gfm/gfmSetup.py b/pelican-gfm/gfmSetup.py index ccc75fbd3..4c9f6568c 100644 --- a/pelican-gfm/gfmSetup.py +++ b/pelican-gfm/gfmSetup.py @@ -1,111 +1,102 @@ #!/usr/bin/python -B +import sys import os import subprocess # This gets used by the gfm plugin as well as the check_configure function -LIBCMARKLOCATION = "/usr/lib/x86_64-linux-gnu" - -ARCHIVES = "https://github.com/github/cmark-gfm/archive" -VERSION = "0.28.3.gfm.12" -LOCAL = "cmark-gfm.$VERSION.orig.tar.gz" -WORKSPACE = '/tmp/build-cmark' +LIBCMARKLOCATION="/usr/lib/x86_64-linux-gnu" +ARCHIVES="https://github.com/github/cmark-gfm/archive" +VERSION="0.28.3.gfm.12" +LOCAL="cmark-gfm.$VERSION.orig.tar.gz" +WORKSPACE='/tmp/build-cmark' def dpkg_installed(package): + """ Uses Dpkg to determine whether or not a package is installed + requires: """ t1 = subprocess.Popen(["dpkg", "-l"], stdout=subprocess.PIPE) - t2 = subprocess.Popen(["grep", "-q", package], - stdout=subprocess.PIPE, - stdin=t1.stdout,) + t2 = subprocess.Popen(["grep", "-q", package], stdout=subprocess.PIPE, stdin=t1.stdout) + data = t2.communicate() ec = t2.wait() return ec - -def test_setup(): +def dpkg_packages_installed(): + """ Checking to see if the appropriate packages are installed""" installed = ["cmake", "make", "wget"] - removed = ["libcmark-gfm-dev", - "libcmark-gfm-extensions-dev", - "libcmark-gfm0", - "libcmark-gfm-extensions0", ] - for package in installed: - if str(dpkg_installed(package)) == "1": - print(package + " not installed") - return 1 - - for package in removed: - if dpkg_installed == "0": - print(package + " needs removed") - return 1 - - -def apt_install(package): - # I need to be able to do this a better, in a less sudo + apt-y way - subprocess.call(["apt-get", "install", package, "-y"]) - - -def apt_remove(package): - # I need to be able to do this a better, in a less sudo + apt-y way - subprocess.call(["apt-get", "purge", package, "-y"]) - + removed = ["libcmark-gfm-dev", "libcmark-gfm-extensions-dev", "libcmark-gfm0", "libcmark-gfm-extensions0"] + need_to_be_removed = [package for package in removed + if dpkg_installed(package) == 0 + ] + + need_to_be_installed = [package for package in installed + if dpkg_installed(package) == 1 + ] + + + if len(need_to_be_removed) is not 0: + raise Exception("Found the following conflicting packages\ + which should be removed:\ + need_to_be_installed") + rem = False + else: rem = True + + if len(need_to_be_installed) is not 0: + raise Exception("Could not find the following required\ +packages: need_to_be_installed") + ins = False + else: ins = True + + if rem is True and ins is True: + print("Packages seem in order.") + return True def cleanUp(): subprocess.call(["rm", "-rf", WORKSPACE]) +def setup (): + + dpkg_packages_installed() + + if test_configuration() == 1: + print("System appears to be configured") + -def setup(): - test_setup() # Configure the environment if it's not already configured if not os.path.isdir(WORKSPACE): os.mkdir(WORKSPACE) - subprocess.call(["wget", - "--quiet", - ARCHIVES + "/" + VERSION + ".tar.gz", WORKSPACE, - "-P", - WORKSPACE]) - subprocess.call(['tar', - 'zxf', - WORKSPACE + "/" + VERSION + ".tar.gz", - "-C", - WORKSPACE]) + + subprocess.call(["wget", "--quiet", ARCHIVES + "/" + VERSION + ".tar.gz", WORKSPACE, "-P", WORKSPACE]) + subprocess.call(['tar', 'zxf', WORKSPACE + "/" + VERSION + ".tar.gz", "-C", WORKSPACE]) + BUILDSPACE = WORKSPACE + "/" + "cmark-gfm-" + VERSION + "/build" + if not os.path.isdir(BUILDSPACE): os.mkdir(BUILDSPACE) - thing1 = subprocess.Popen(["cmake", - "-DCMARK_TESTS=OFF", - "-DCMARK_STATIC=OFF", - ".."], - cwd=BUILDSPACE) + + thing1 = subprocess.Popen(["cmake", "-DCMARK_TESTS=OFF", "-DCMARK_STATIC=OFF", ".."], cwd=BUILDSPACE) thing1.wait() thing2 = subprocess.Popen(["make"], cwd=BUILDSPACE) thing2.wait() # Move the libcmark.so artifacts in place - print("Moving files") - gfmfile = BUILDSPACE+"/src/libcmark-gfm.so."+VERSION - gfmextfile = BUILDSPACE+"/extensions/libcmark-gfmextensions.so."+VERSION - subprocess.call(["mv", - gfmfile, - LIBCMARKLOCATION + "libcmark-gfm.so"]) - subprocess.call(["mv", - gfmextfile, - LIBCMARKLOCATION + "libcmark-gfmextensions.so"]) - + print "Moving files" + subprocess.call(["mv", BUILDSPACE + "/src/libcmark-gfm.so." + VERSION, LIBCMARKLOCATION + "libcmark-gfm.so"]) + subprocess.call(["mv", BUILDSPACE + "/extensions/libcmark-gfmextensions.so." + VERSION, LIBCMARKLOCATION + "libcmark-gfmextensions.so"]) def test_configuration(): - gfmfile = LIBCMARKLOCATION + "/libcmark-gfm.so" - gfmextfile = LIBCMARKLOCATION + "/libcmark-gfmextensions.so" - if os.path.isfile(gfmfile) and os.path.isfile(gfmextfile): + """ Tests to ensure that the files that the plugin needs are in place. """ + CMARKPATH = LIBCMARKLOCATION + "/libcmark-gfm.so." + VERSION + if os.path.isfile(LIBCMARKLOCATION + "/libcmark-gfm.so") and os.path.isfile(LIBCMARKLOCATION + "/libcmark-gfmextensions.so"): return 0 else: return 1 - def configure(): - print("Configuring!!!") + print("Checking out the configuration") setup() cleanUp() - if __name__ == "__main__": configure() From 9b31101ac3749e82a73c3279c7712e0011cbc36b Mon Sep 17 00:00:00 2001 From: dfoulks Date: Mon, 16 Dec 2019 13:58:11 -0600 Subject: [PATCH 03/24] PEP8 verified did PEP8 verification on the scripts via pycodestyle. they all pass. I also added a vars file so that we can set where the plugin will work. --- pelican-gfm/gfm.py | 236 +++++++++++++++++++++------------------- pelican-gfm/gfmSetup.py | 91 ++++++++++++---- pelican-gfm/gfmVars.py | 8 ++ 3 files changed, 200 insertions(+), 135 deletions(-) create mode 100644 pelican-gfm/gfmVars.py diff --git a/pelican-gfm/gfm.py b/pelican-gfm/gfm.py index 7bda6c9a0..8bca54003 100644 --- a/pelican-gfm/gfm.py +++ b/pelican-gfm/gfm.py @@ -26,6 +26,7 @@ import time import re import gfmSetup +import gfmVars import pelican.utils import pelican.signals import pelican.readers @@ -34,11 +35,13 @@ _LIBCMARK = 'libcmark-gfm.so' _LIBEXT = 'libcmark-gfmextensions.so' try: - cmark = ctypes.CDLL(os.path.join(_LIBDIR, _LIBCMARK)) - cmark_ext = ctypes.CDLL(os.path.join(_LIBDIR, _LIBEXT)) + cmark = ctypes.CDLL(os.path.join(_LIBDIR, _LIBCMARK)) + cmark_ext = ctypes.CDLL(os.path.join(_LIBDIR, _LIBEXT)) except OSError: - raise ImportError('%s not found. see build-mark.sh and gfm_reader.py' - % _LIBCMARK) + raise ImportError( + '%s not found. see build-mark.sh and gfm_reader.py' + % _LIBCMARK + ) # Use ctypes to access the functions in libcmark-gfm F_cmark_parser_new = cmark.cmark_parser_new @@ -47,15 +50,23 @@ F_cmark_parser_feed = cmark.cmark_parser_feed F_cmark_parser_feed.restype = None -F_cmark_parser_feed.argtypes = (ctypes.c_void_p, ctypes.c_char_p, ctypes.c_size_t) +F_cmark_parser_feed.argtypes = ( + ctypes.c_void_p, + ctypes.c_char_p, + ctypes.c_size_t + ) F_cmark_parser_finish = cmark.cmark_parser_finish F_cmark_parser_finish.restype = ctypes.c_void_p F_cmark_parser_finish.argtypes = (ctypes.c_void_p,) -F_cmark_parser_attach_syntax_extension = cmark.cmark_parser_attach_syntax_extension +F_cmark_parser_attach_syntax_extension = \ + cmark.cmark_parser_attach_syntax_extension F_cmark_parser_attach_syntax_extension.restype = ctypes.c_int -F_cmark_parser_attach_syntax_extension.argtypes = (ctypes.c_void_p, ctypes.c_void_p) +F_cmark_parser_attach_syntax_extension.argtypes = ( + ctypes.c_void_p, + ctypes.c_void_p + ) F_cmark_parser_get_syntax_extensions = cmark.cmark_parser_get_syntax_extensions F_cmark_parser_get_syntax_extensions.restype = ctypes.c_void_p @@ -81,13 +92,13 @@ # Set up the libcmark-gfm library and its extensions F_register = cmark_ext.core_extensions_ensure_registered F_register.restype = None -F_register.argtypes = ( ) +F_register.argtypes = () F_register() -### technically, maybe install an atexit() to release the plugins +# technically, maybe install an atexit() to release the plugins # Options for the GFM rendering call -### this could be moved into SETTINGS or somesuch, but meh. not needed now. +# this could be moved into SETTINGS or somesuch, but meh. not needed now. OPTS = 0 # The GFM extensions that we want to use @@ -100,115 +111,118 @@ class GFMReader(pelican.readers.BaseReader): - enabled = True - """GFM-flavored Reader for the Pelican system. - - Pelican looks for all subclasses of BaseReader, and automatically - registers them for the file extensions listed below. Thus, nothing - further is required by users of this Reader. - """ - - # NOTE: the builtin MarkdownReader must be disabled. Otherwise, it will be - # non-deterministic which Reader will be used for these files. - file_extensions = ['md', 'markdown', 'mkd', 'mdown'] - - # Metadata is specified as a single, colon-separated line, such as: - # - # Title: this is the title - # - # Note: name starts in column 0, no whitespace before colon, will be - # made lower-case, and value will be stripped - # - RE_METADATA = re.compile('^([A-za-z]+): (.*)$') - - def read(self, source_path): - # Prepare the "slug", which is the target file name. It will be the - # same as the source file, minus the leading ".../content/(articles|pages)" - # and with the extension removed (Pelican will add .html) - relpath = os.path.relpath(source_path, self.settings['PATH']) - parts = relpath.split(os.sep) - parts[-1] = os.path.splitext(parts[-1])[0] # split off ext, keep base - slug = os.sep.join(parts[1:]) - - metadata = { - 'slug': slug, - } - - - - # Fetch the source content, with a few appropriate tweaks - with pelican.utils.pelican_open(source_path) as text: - - # Extract the metadata from the header of the text - lines = text.splitlines() - for i in range(len(lines)): - line = lines[i] - match = GFMReader.RE_METADATA.match(line) - if match: - name = match.group(1).strip().lower() - if name != 'slug': - value = match.group(2).strip() - if name == 'date': - value = pelican.utils.get_date(value) - metadata[name] = value - #if name != 'title': - # print 'META:', name, value - elif not line.strip(): - # blank line - continue - else: - # reached actual content - break - - # Reassemble content, minus the metadata - text = '\n'.join(lines[i:]) - - # Render the markdown into HTML + enabled = True + """GFM-flavored Reader for the Pelican system. + + Pelican looks for all subclasses of BaseReader, and automatically + registers them for the file extensions listed below. Thus, nothing + further is required by users of this Reader. + """ + + # NOTE: the builtin MarkdownReader must be disabled. Otherwise, it will be + # non-deterministic which Reader will be used for these files. + file_extensions = ['md', 'markdown', 'mkd', 'mdown'] + + # Metadata is specified as a single, colon-separated line, such as: + # + # Title: this is the title + # + # Note: name starts in column 0, no whitespace before colon, will be + # made lower-case, and value will be stripped + # + RE_METADATA = re.compile('^([A-za-z]+): (.*)$') + + def read(self, source_path): + # Prepare the "slug", which is the target file name. + # It will be the same as the source file, minus the leading + # ".../content/(articles|pages)" + # and with the extension removed (Pelican will add .html) + relpath = os.path.relpath(source_path, self.settings['PATH']) + parts = relpath.split(os.sep) + + # split off ext, keep base + parts[-1] = os.path.splitext(parts[-1])[0] + slug = os.sep.join(parts[1:]) + + metadata = { + 'slug': slug, + } + + # Fetch the source content, with a few appropriate tweaks + with pelican.utils.pelican_open(source_path) as text: + + # Extract the metadata from the header of the text + lines = text.splitlines() + for i in range(len(lines)): + line = lines[i] + match = GFMReader.RE_METADATA.match(line) + if match: + name = match.group(1).strip().lower() + if name != 'slug': + value = match.group(2).strip() + if name == 'date': + value = pelican.utils.get_date(value) + metadata[name] = value + # if name != 'title': + # print 'META:', name, value + elif not line.strip(): + # blank line + continue + else: + # reached actual content + break + + # Reassemble content, minus the metadata + text = '\n'.join(lines[i:]) + + # Render the markdown into HTML # if sys.version_info >= (3, 0): # text = text.encode('utf-8') # content = self.render(text).decode('utf-8') # else: - content = self.render(text) - - # Redo the slug for articles. - if parts[0] == 'articles' and 'title' in metadata: - metadata['slug'] = pelican.utils.slugify( - metadata['title'], - self.settings.get('SLUG_SUBSTITUTIONS', ())) - - return content, metadata - - def render(self, text): - "Use cmark-gfm to render the Markdown into an HTML fragment." - - parser = F_cmark_parser_new(OPTS) - assert parser - for name in EXTENSIONS: - ext = F_cmark_find_syntax_extension(name) - assert ext - rv = F_cmark_parser_attach_syntax_extension(parser, ext) - assert rv - exts = F_cmark_parser_get_syntax_extensions(parser) - - F_cmark_parser_feed(parser, text, len(text)) - doc = F_cmark_parser_finish(parser) - assert doc - - output = F_cmark_render_html(doc, OPTS, exts) - - F_cmark_parser_free(parser) - F_cmark_node_free(doc) - - return output + content = self.render(text) + + # Redo the slug for articles. + if parts[0] == 'articles' and 'title' in metadata: + metadata['slug'] = pelican.utils.slugify( + metadata['title'], + self.settings.get( + 'SLUG_SUBSTITUTIONS', + () + ) + ) + + return content, metadata + + def render(self, text): + "Use cmark-gfm to render the Markdown into an HTML fragment." + + parser = F_cmark_parser_new(OPTS) + assert parser + for name in EXTENSIONS: + ext = F_cmark_find_syntax_extension(name) + assert ext + rv = F_cmark_parser_attach_syntax_extension(parser, ext) + assert rv + exts = F_cmark_parser_get_syntax_extensions(parser) + F_cmark_parser_feed(parser, text, len(text)) + doc = F_cmark_parser_finish(parser) + assert doc + output = F_cmark_render_html(doc, OPTS, exts) + F_cmark_parser_free(parser) + F_cmark_node_free(doc) + return output def add_readers(readers): if str(gfmSetup.test_configuration()) == "0": - readers.reader_classes['md'] = GFMReader + readers.reader_classes['md'] = GFMReader else: - # Optionally we can have the gfmSetup script run here as root and configure the system - # Probably just easier to run it independently tho. - sys.exit(1) + raise Exception('GFM plugin cannot find the required libcmark files + Please run gfmSetup.configure() to build and configure + the appropriate libcmark files' + ) + def register(): - pelican.signals.readers_init.connect(add_readers) + pelican.signals.readers_init.connect(add_readers) diff --git a/pelican-gfm/gfmSetup.py b/pelican-gfm/gfmSetup.py index 4c9f6568c..55471377d 100644 --- a/pelican-gfm/gfmSetup.py +++ b/pelican-gfm/gfmSetup.py @@ -3,78 +3,107 @@ import sys import os import subprocess +import gfmVars -# This gets used by the gfm plugin as well as the check_configure function -LIBCMARKLOCATION="/usr/lib/x86_64-linux-gnu" -ARCHIVES="https://github.com/github/cmark-gfm/archive" -VERSION="0.28.3.gfm.12" -LOCAL="cmark-gfm.$VERSION.orig.tar.gz" -WORKSPACE='/tmp/build-cmark' +# Eventually, equivalents for +# other operating systems / package +# managers could be written for this def dpkg_installed(package): """ Uses Dpkg to determine whether or not a package is installed requires: """ t1 = subprocess.Popen(["dpkg", "-l"], stdout=subprocess.PIPE) - t2 = subprocess.Popen(["grep", "-q", package], stdout=subprocess.PIPE, stdin=t1.stdout) + t2 = subprocess.Popen([ + "grep", + "-q", + package + ], stdout=subprocess.PIPE, stdin=t1.stdout) data = t2.communicate() ec = t2.wait() return ec + def dpkg_packages_installed(): """ Checking to see if the appropriate packages are installed""" installed = ["cmake", "make", "wget"] - removed = ["libcmark-gfm-dev", "libcmark-gfm-extensions-dev", "libcmark-gfm0", "libcmark-gfm-extensions0"] + removed = [ + "libcmark-gfm-dev", + "libcmark-gfm-extensions-dev", + "libcmark-gfm0", + "libcmark-gfm-extensions0" + ] + need_to_be_removed = [package for package in removed if dpkg_installed(package) == 0 ] - need_to_be_installed = [package for package in installed - if dpkg_installed(package) == 1 - ] - + need_to_be_installed = [ + package for package in installed + if dpkg_installed(package) == 1 + ] if len(need_to_be_removed) is not 0: raise Exception("Found the following conflicting packages\ which should be removed:\ need_to_be_installed") rem = False - else: rem = True - + else: + rem = True + if len(need_to_be_installed) is not 0: raise Exception("Could not find the following required\ packages: need_to_be_installed") ins = False - else: ins = True + else: + ins = True if rem is True and ins is True: print("Packages seem in order.") return True + def cleanUp(): subprocess.call(["rm", "-rf", WORKSPACE]) -def setup (): +def setup(): dpkg_packages_installed() if test_configuration() == 1: print("System appears to be configured") - - # Configure the environment if it's not already configured if not os.path.isdir(WORKSPACE): os.mkdir(WORKSPACE) - subprocess.call(["wget", "--quiet", ARCHIVES + "/" + VERSION + ".tar.gz", WORKSPACE, "-P", WORKSPACE]) - subprocess.call(['tar', 'zxf', WORKSPACE + "/" + VERSION + ".tar.gz", "-C", WORKSPACE]) + subprocess.call([ + "wget", + "--quiet", + ARCHIVES + "/" + VERSION + ".tar.gz", + WORKSPACE, + "-P", + WORKSPACE + ]) + subprocess.call([ + 'tar', + 'zxf', + WORKSPACE + "/" + VERSION + ".tar.gz", + "-C", + WORKSPACE + ] + ) BUILDSPACE = WORKSPACE + "/" + "cmark-gfm-" + VERSION + "/build" if not os.path.isdir(BUILDSPACE): os.mkdir(BUILDSPACE) - thing1 = subprocess.Popen(["cmake", "-DCMARK_TESTS=OFF", "-DCMARK_STATIC=OFF", ".."], cwd=BUILDSPACE) + thing1 = subprocess.Popen([ + "cmake", + "-DCMARK_TESTS=OFF", + "-DCMARK_STATIC=OFF", + ".." + ], cwd=BUILDSPACE) thing1.wait() thing2 = subprocess.Popen(["make"], cwd=BUILDSPACE) @@ -82,21 +111,35 @@ def setup (): # Move the libcmark.so artifacts in place print "Moving files" - subprocess.call(["mv", BUILDSPACE + "/src/libcmark-gfm.so." + VERSION, LIBCMARKLOCATION + "libcmark-gfm.so"]) - subprocess.call(["mv", BUILDSPACE + "/extensions/libcmark-gfmextensions.so." + VERSION, LIBCMARKLOCATION + "libcmark-gfmextensions.so"]) + subprocess.call([ + "mv", + BUILDSPACE + "/src/libcmark-gfm.so." + VERSION, + LIBCMARKLOCATION + "libcmark-gfm.so" + ] + ) + subprocess.call([ + "mv", + BUILDSPACE + "/extensions/libcmark-gfmextensions.so." + VERSION, + LIBCMARKLOCATION + "libcmark-gfmextensions.so" + ] + ) + def test_configuration(): """ Tests to ensure that the files that the plugin needs are in place. """ CMARKPATH = LIBCMARKLOCATION + "/libcmark-gfm.so." + VERSION - if os.path.isfile(LIBCMARKLOCATION + "/libcmark-gfm.so") and os.path.isfile(LIBCMARKLOCATION + "/libcmark-gfmextensions.so"): + if os.path.isfile(LIBCMARKLOCATION + "/libcmark-gfm.so") and + os.path.isfile(LIBCMARKLOCATION + "/libcmark-gfmextensions.so"): return 0 else: return 1 + def configure(): print("Checking out the configuration") setup() cleanUp() + if __name__ == "__main__": configure() diff --git a/pelican-gfm/gfmVars.py b/pelican-gfm/gfmVars.py new file mode 100644 index 000000000..a9a3b667c --- /dev/null +++ b/pelican-gfm/gfmVars.py @@ -0,0 +1,8 @@ +# This gets used by the gfm plugin as well as the check_configure function + +LIBCMARKLOCATION="/usr/lib/x86_64-linux-gnu" +ARCHIVES="https://github.com/github/cmark-gfm/archive" +VERSION="0.28.3.gfm.12" +LOCAL="cmark-gfm.$VERSION.orig.tar.gz" +WORKSPACE='/tmp/build-cmark' + From 44d5b95e63c7d083efeaca932e7096ff9c3f107a Mon Sep 17 00:00:00 2001 From: dfoulks Date: Mon, 16 Dec 2019 16:15:19 -0600 Subject: [PATCH 04/24] finished up pep8 compliance made sure that the plugin still runs. looks great and serves up just fine. I've pulled most of the configurables out of the script and put it into a vars file. hopefully this will suffice for the pelican folks. --- pelican-gfm/gfm.py | 10 +++++----- pelican-gfm/gfmSetup.py | 20 ++++++++++---------- pelican-gfm/gfmVars.py | 12 ++++++------ 3 files changed, 21 insertions(+), 21 deletions(-) diff --git a/pelican-gfm/gfm.py b/pelican-gfm/gfm.py index 8bca54003..5c9840e64 100644 --- a/pelican-gfm/gfm.py +++ b/pelican-gfm/gfm.py @@ -31,7 +31,7 @@ import pelican.signals import pelican.readers -_LIBDIR = gfmSetup.LIBCMARKLOCATION +_LIBDIR = gfmVars.LIBCMARKLOCATION _LIBCMARK = 'libcmark-gfm.so' _LIBEXT = 'libcmark-gfmextensions.so' try: @@ -215,13 +215,13 @@ def render(self, text): def add_readers(readers): + msg = "GFM plugin cannot find the required libcmark files.\ + Please run gfmSetup.configure() to build and\ + configure the appropriate libcmark files" if str(gfmSetup.test_configuration()) == "0": readers.reader_classes['md'] = GFMReader else: - raise Exception('GFM plugin cannot find the required libcmark files - Please run gfmSetup.configure() to build and configure - the appropriate libcmark files' - ) + raise Exception(msg) def register(): diff --git a/pelican-gfm/gfmSetup.py b/pelican-gfm/gfmSetup.py index 55471377d..c65e7237f 100644 --- a/pelican-gfm/gfmSetup.py +++ b/pelican-gfm/gfmSetup.py @@ -79,7 +79,7 @@ def setup(): subprocess.call([ "wget", "--quiet", - ARCHIVES + "/" + VERSION + ".tar.gz", + ARCHIVES + "/" + gfmVars.VERSION + ".tar.gz", WORKSPACE, "-P", WORKSPACE @@ -87,13 +87,13 @@ def setup(): subprocess.call([ 'tar', 'zxf', - WORKSPACE + "/" + VERSION + ".tar.gz", + WORKSPACE + "/" + gfmVars.VERSION + ".tar.gz", "-C", WORKSPACE ] ) - BUILDSPACE = WORKSPACE + "/" + "cmark-gfm-" + VERSION + "/build" + BUILDSPACE = gfmVars.WORKSPACE + "/" + "cmark-gfm-" + gfmVars.VERSION + "/build" if not os.path.isdir(BUILDSPACE): os.mkdir(BUILDSPACE) @@ -113,23 +113,23 @@ def setup(): print "Moving files" subprocess.call([ "mv", - BUILDSPACE + "/src/libcmark-gfm.so." + VERSION, - LIBCMARKLOCATION + "libcmark-gfm.so" + BUILDSPACE + "/src/libcmark-gfm.so." + gfmVars.VERSION, + gfmVars.LIBCMARKLOCATION + "libcmark-gfm.so" ] ) subprocess.call([ "mv", - BUILDSPACE + "/extensions/libcmark-gfmextensions.so." + VERSION, - LIBCMARKLOCATION + "libcmark-gfmextensions.so" + BUILDSPACE + "/extensions/libcmark-gfmextensions.so." + gfmVars.VERSION, + gfmVars.LIBCMARKLOCATION + "libcmark-gfmextensions.so" ] ) def test_configuration(): """ Tests to ensure that the files that the plugin needs are in place. """ - CMARKPATH = LIBCMARKLOCATION + "/libcmark-gfm.so." + VERSION - if os.path.isfile(LIBCMARKLOCATION + "/libcmark-gfm.so") and - os.path.isfile(LIBCMARKLOCATION + "/libcmark-gfmextensions.so"): + CMARKPATH = gfmVars.LIBCMARKLOCATION + "/libcmark-gfm.so." + gfmVars.VERSION + if os.path.isfile(gfmVars.LIBCMARKLOCATION + "/libcmark-gfm.so") and \ + os.path.isfile(gfmVars.LIBCMARKLOCATION + "/libcmark-gfmextensions.so"): return 0 else: return 1 diff --git a/pelican-gfm/gfmVars.py b/pelican-gfm/gfmVars.py index a9a3b667c..4d7e89ad2 100644 --- a/pelican-gfm/gfmVars.py +++ b/pelican-gfm/gfmVars.py @@ -1,8 +1,8 @@ +#!/usr/bin/python # This gets used by the gfm plugin as well as the check_configure function -LIBCMARKLOCATION="/usr/lib/x86_64-linux-gnu" -ARCHIVES="https://github.com/github/cmark-gfm/archive" -VERSION="0.28.3.gfm.12" -LOCAL="cmark-gfm.$VERSION.orig.tar.gz" -WORKSPACE='/tmp/build-cmark' - +LIBCMARKLOCATION = "/usr/lib/x86_64-linux-gnu" +ARCHIVES = "https://github.com/github/cmark-gfm/archive" +VERSION = "0.28.3.gfm.12" +LOCAL = "cmark-gfm.$VERSION.orig.tar.gz" +WORKSPACE = '/tmp/build-cmark' From 886009c4ff27d12d71521b7ed8969c27e226fe0a Mon Sep 17 00:00:00 2001 From: dfoulks Date: Mon, 16 Dec 2019 16:41:01 -0600 Subject: [PATCH 05/24] Updated Readme.md --- pelican-gfm/Readme.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pelican-gfm/Readme.md b/pelican-gfm/Readme.md index eb30a3c65..0ec2c77d2 100644 --- a/pelican-gfm/Readme.md +++ b/pelican-gfm/Readme.md @@ -4,8 +4,7 @@ A reader that leverages GitHub's C-based markdown library to translate GitHub Fl Requirements ============ - -pelican-gfm has no requirements that are outside of the python standard library aside from pelican itself. +pelican-gfm has no requirements outside of the python standard library and pelican pelican itself. How to Use ========= From 284b90efb668e2512cc2fe9fdc8b30a80359ef95 Mon Sep 17 00:00:00 2001 From: dfoulks Date: Mon, 16 Dec 2019 17:11:07 -0600 Subject: [PATCH 06/24] Python3-ized everything got tempfile.TemporaryDirectory to work out just fine. also checks the config like a champ. --- pelican-gfm/__init__.py | 2 +- pelican-gfm/gfm.py | 2 +- pelican-gfm/gfmSetup.py | 122 +++++++++++++++++++--------------------- pelican-gfm/gfmVars.py | 2 +- 4 files changed, 61 insertions(+), 67 deletions(-) diff --git a/pelican-gfm/__init__.py b/pelican-gfm/__init__.py index 015c9028f..513a4a685 100644 --- a/pelican-gfm/__init__.py +++ b/pelican-gfm/__init__.py @@ -1,3 +1,3 @@ -#!/usr/bin/environment python -B +#!/usr/bin/environment python333 -B from gfm import * diff --git a/pelican-gfm/gfm.py b/pelican-gfm/gfm.py index 5c9840e64..726d62dd7 100644 --- a/pelican-gfm/gfm.py +++ b/pelican-gfm/gfm.py @@ -1,4 +1,4 @@ -#!/usr/bin/python -B +#!/usr/bin/python3 -B # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/pelican-gfm/gfmSetup.py b/pelican-gfm/gfmSetup.py index c65e7237f..0177af3fa 100644 --- a/pelican-gfm/gfmSetup.py +++ b/pelican-gfm/gfmSetup.py @@ -1,10 +1,10 @@ -#!/usr/bin/python -B +#!/usr/bin/python3 -B import sys import os import subprocess import gfmVars - +from backports import tempfile # Eventually, equivalents for # other operating systems / package @@ -45,15 +45,14 @@ def dpkg_packages_installed(): if len(need_to_be_removed) is not 0: raise Exception("Found the following conflicting packages\ - which should be removed:\ - need_to_be_installed") + which should be removed:" + str(need_to_be_removed)) rem = False else: rem = True if len(need_to_be_installed) is not 0: raise Exception("Could not find the following required\ -packages: need_to_be_installed") + packages: " + need_to_be_installed ) ins = False else: ins = True @@ -63,82 +62,77 @@ def dpkg_packages_installed(): return True -def cleanUp(): - subprocess.call(["rm", "-rf", WORKSPACE]) - - def setup(): dpkg_packages_installed() if test_configuration() == 1: print("System appears to be configured") - # Configure the environment if it's not already configured - if not os.path.isdir(WORKSPACE): - os.mkdir(WORKSPACE) - - subprocess.call([ - "wget", - "--quiet", - ARCHIVES + "/" + gfmVars.VERSION + ".tar.gz", - WORKSPACE, - "-P", - WORKSPACE - ]) - subprocess.call([ - 'tar', - 'zxf', - WORKSPACE + "/" + gfmVars.VERSION + ".tar.gz", - "-C", - WORKSPACE - ] - ) - - BUILDSPACE = gfmVars.WORKSPACE + "/" + "cmark-gfm-" + gfmVars.VERSION + "/build" - - if not os.path.isdir(BUILDSPACE): - os.mkdir(BUILDSPACE) - - thing1 = subprocess.Popen([ - "cmake", - "-DCMARK_TESTS=OFF", - "-DCMARK_STATIC=OFF", - ".." - ], cwd=BUILDSPACE) - thing1.wait() - - thing2 = subprocess.Popen(["make"], cwd=BUILDSPACE) - thing2.wait() - - # Move the libcmark.so artifacts in place - print "Moving files" - subprocess.call([ - "mv", - BUILDSPACE + "/src/libcmark-gfm.so." + gfmVars.VERSION, - gfmVars.LIBCMARKLOCATION + "libcmark-gfm.so" - ] - ) - subprocess.call([ - "mv", - BUILDSPACE + "/extensions/libcmark-gfmextensions.so." + gfmVars.VERSION, - gfmVars.LIBCMARKLOCATION + "libcmark-gfmextensions.so" - ] - ) + else: + # Configure the environment if it's not already configured + + with tempfile.TemporaryDirectory() as WORKSPACE: + subprocess.call([ + "wget", + "--quiet", + gfmVars.ARCHIVES + "/" + gfmVars.VERSION + ".tar.gz", + WORKSPACE, + "-P", + WORKSPACE + ]) + subprocess.call([ + 'tar', + 'zxf', + WORKSPACE + "/" + gfmVars.VERSION + ".tar.gz", + "-C", + WORKSPACE + ] + ) + + BUILDSPACE = WORKSPACE + "/" + "cmark-gfm-" + gfmVars.VERSION + "/build" + + if not os.path.isdir(BUILDSPACE): + os.mkdir(BUILDSPACE) + + thing1 = subprocess.Popen([ + "cmake", + "-DCMARK_TESTS=OFF", + "-DCMARK_STATIC=OFF", + ".." + ], cwd=BUILDSPACE) + thing1.wait() + + thing2 = subprocess.Popen(["make"], cwd=BUILDSPACE) + thing2.wait() + + # Move the libcmark.so artifacts in place + print("Moving files") + subprocess.call([ + "mv", + BUILDSPACE + "/src/libcmark-gfm.so." + gfmVars.VERSION, + gfmVars.LIBCMARKLOCATION + "libcmark-gfm.so" + ] + ) + subprocess.call([ + "mv", + BUILDSPACE + "/extensions/libcmark-gfmextensions.so." + gfmVars.VERSION, + gfmVars.LIBCMARKLOCATION + "libcmark-gfmextensions.so" + ] + ) def test_configuration(): """ Tests to ensure that the files that the plugin needs are in place. """ CMARKPATH = gfmVars.LIBCMARKLOCATION + "/libcmark-gfm.so." + gfmVars.VERSION if os.path.isfile(gfmVars.LIBCMARKLOCATION + "/libcmark-gfm.so") and \ - os.path.isfile(gfmVars.LIBCMARKLOCATION + "/libcmark-gfmextensions.so"): - return 0 + os.path.isfile(gfmVars.LIBCMARKLOCATION + "/libcmark-gfmextensions.so"): + return True else: - return 1 + return False def configure(): print("Checking out the configuration") setup() - cleanUp() if __name__ == "__main__": diff --git a/pelican-gfm/gfmVars.py b/pelican-gfm/gfmVars.py index 4d7e89ad2..3df006dfe 100644 --- a/pelican-gfm/gfmVars.py +++ b/pelican-gfm/gfmVars.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 -B # This gets used by the gfm plugin as well as the check_configure function LIBCMARKLOCATION = "/usr/lib/x86_64-linux-gnu" From 03dfb9d566d6fbaa624abfa1b0b537b5efd9891d Mon Sep 17 00:00:00 2001 From: dfoulks Date: Mon, 16 Dec 2019 17:27:19 -0600 Subject: [PATCH 07/24] Moved Extensions definition to gfmVars --- pelican-gfm/gfm.py | 11 +---------- pelican-gfm/gfmVars.py | 8 ++++++++ 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/pelican-gfm/gfm.py b/pelican-gfm/gfm.py index 726d62dd7..72067d1fe 100644 --- a/pelican-gfm/gfm.py +++ b/pelican-gfm/gfm.py @@ -101,15 +101,6 @@ # this could be moved into SETTINGS or somesuch, but meh. not needed now. OPTS = 0 -# The GFM extensions that we want to use -EXTENSIONS = ( - 'autolink', - 'table', - 'strikethrough', - 'tagfilter', -) - - class GFMReader(pelican.readers.BaseReader): enabled = True """GFM-flavored Reader for the Pelican system. @@ -199,7 +190,7 @@ def render(self, text): parser = F_cmark_parser_new(OPTS) assert parser - for name in EXTENSIONS: + for name in gfmVars.EXTENSIONS: ext = F_cmark_find_syntax_extension(name) assert ext rv = F_cmark_parser_attach_syntax_extension(parser, ext) diff --git a/pelican-gfm/gfmVars.py b/pelican-gfm/gfmVars.py index 3df006dfe..d51ce3fc1 100644 --- a/pelican-gfm/gfmVars.py +++ b/pelican-gfm/gfmVars.py @@ -6,3 +6,11 @@ VERSION = "0.28.3.gfm.12" LOCAL = "cmark-gfm.$VERSION.orig.tar.gz" WORKSPACE = '/tmp/build-cmark' +# The GFM extensions that we want to use +EXTENSIONS = ( + 'autolink', + 'table', + 'strikethrough', + 'tagfilter', +) + From 963e121c29785bb180a421c684345e58959fa8b1 Mon Sep 17 00:00:00 2001 From: dfoulks Date: Tue, 17 Dec 2019 08:52:34 -0600 Subject: [PATCH 08/24] Clarified some of the names for the files. --- pelican-gfm/gfm.py | 6 +++--- pelican-gfm/gfmSetup.py | 22 +++++++++++----------- pelican-gfm/{gfmVars.py => setup.py} | 12 ++++++++---- 3 files changed, 22 insertions(+), 18 deletions(-) rename pelican-gfm/{gfmVars.py => setup.py} (60%) diff --git a/pelican-gfm/gfm.py b/pelican-gfm/gfm.py index 72067d1fe..98c9667a7 100644 --- a/pelican-gfm/gfm.py +++ b/pelican-gfm/gfm.py @@ -26,12 +26,12 @@ import time import re import gfmSetup -import gfmVars +import config import pelican.utils import pelican.signals import pelican.readers -_LIBDIR = gfmVars.LIBCMARKLOCATION +_LIBDIR = config.LIBCMARKLOCATION _LIBCMARK = 'libcmark-gfm.so' _LIBEXT = 'libcmark-gfmextensions.so' try: @@ -190,7 +190,7 @@ def render(self, text): parser = F_cmark_parser_new(OPTS) assert parser - for name in gfmVars.EXTENSIONS: + for name in config.EXTENSIONS: ext = F_cmark_find_syntax_extension(name) assert ext rv = F_cmark_parser_attach_syntax_extension(parser, ext) diff --git a/pelican-gfm/gfmSetup.py b/pelican-gfm/gfmSetup.py index 0177af3fa..8afba8cf4 100644 --- a/pelican-gfm/gfmSetup.py +++ b/pelican-gfm/gfmSetup.py @@ -3,7 +3,7 @@ import sys import os import subprocess -import gfmVars +import config from backports import tempfile # Eventually, equivalents for @@ -74,7 +74,7 @@ def setup(): subprocess.call([ "wget", "--quiet", - gfmVars.ARCHIVES + "/" + gfmVars.VERSION + ".tar.gz", + config.ARCHIVES + "/" + config.VERSION + ".tar.gz", WORKSPACE, "-P", WORKSPACE @@ -82,13 +82,13 @@ def setup(): subprocess.call([ 'tar', 'zxf', - WORKSPACE + "/" + gfmVars.VERSION + ".tar.gz", + WORKSPACE + "/" + config.VERSION + ".tar.gz", "-C", WORKSPACE ] ) - BUILDSPACE = WORKSPACE + "/" + "cmark-gfm-" + gfmVars.VERSION + "/build" + BUILDSPACE = WORKSPACE + "/" + "cmark-gfm-" + config.VERSION + "/build" if not os.path.isdir(BUILDSPACE): os.mkdir(BUILDSPACE) @@ -108,23 +108,23 @@ def setup(): print("Moving files") subprocess.call([ "mv", - BUILDSPACE + "/src/libcmark-gfm.so." + gfmVars.VERSION, - gfmVars.LIBCMARKLOCATION + "libcmark-gfm.so" + BUILDSPACE + "/src/libcmark-gfm.so." + config.VERSION, + config.LIBCMARKLOCATION + "libcmark-gfm.so" ] ) subprocess.call([ "mv", - BUILDSPACE + "/extensions/libcmark-gfmextensions.so." + gfmVars.VERSION, - gfmVars.LIBCMARKLOCATION + "libcmark-gfmextensions.so" + BUILDSPACE + "/extensions/libcmark-gfmextensions.so." + config.VERSION, + config.LIBCMARKLOCATION + "libcmark-gfmextensions.so" ] ) def test_configuration(): """ Tests to ensure that the files that the plugin needs are in place. """ - CMARKPATH = gfmVars.LIBCMARKLOCATION + "/libcmark-gfm.so." + gfmVars.VERSION - if os.path.isfile(gfmVars.LIBCMARKLOCATION + "/libcmark-gfm.so") and \ - os.path.isfile(gfmVars.LIBCMARKLOCATION + "/libcmark-gfmextensions.so"): + CMARKPATH = config.LIBCMARKLOCATION + "/libcmark-gfm.so." + config.VERSION + if os.path.isfile(config.LIBCMARKLOCATION + "/libcmark-gfm.so") and \ + os.path.isfile(config.LIBCMARKLOCATION + "/libcmark-gfmextensions.so"): return True else: return False diff --git a/pelican-gfm/gfmVars.py b/pelican-gfm/setup.py similarity index 60% rename from pelican-gfm/gfmVars.py rename to pelican-gfm/setup.py index d51ce3fc1..4a9bad4f1 100644 --- a/pelican-gfm/gfmVars.py +++ b/pelican-gfm/setup.py @@ -1,11 +1,15 @@ -#!/usr/bin/python3 -B -# This gets used by the gfm plugin as well as the check_configure function - +# Eventual location of the libcmark files LIBCMARKLOCATION = "/usr/lib/x86_64-linux-gnu" + +# This is the archive of GitHub's cmark-gfm files ARCHIVES = "https://github.com/github/cmark-gfm/archive" + +# This is the GitHub Cmark version we're using VERSION = "0.28.3.gfm.12" + +# The name of the local tarball that will be downloaded LOCAL = "cmark-gfm.$VERSION.orig.tar.gz" -WORKSPACE = '/tmp/build-cmark' + # The GFM extensions that we want to use EXTENSIONS = ( 'autolink', From 469c99a1025cee803152150e21b3c6b826809e15 Mon Sep 17 00:00:00 2001 From: dfoulks Date: Tue, 17 Dec 2019 08:56:06 -0600 Subject: [PATCH 09/24] updated name --- pelican-gfm/{setup.py => config.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename pelican-gfm/{setup.py => config.py} (100%) diff --git a/pelican-gfm/setup.py b/pelican-gfm/config.py similarity index 100% rename from pelican-gfm/setup.py rename to pelican-gfm/config.py From 922db0c352afba0b47509689b47bf84723c157bf Mon Sep 17 00:00:00 2001 From: dfoulks Date: Tue, 17 Dec 2019 09:51:07 -0600 Subject: [PATCH 10/24] pycodestyled the scripts after changes --- pelican-gfm/gfm.py | 1 + pelican-gfm/gfmSetup.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/pelican-gfm/gfm.py b/pelican-gfm/gfm.py index 98c9667a7..c29ecd314 100644 --- a/pelican-gfm/gfm.py +++ b/pelican-gfm/gfm.py @@ -101,6 +101,7 @@ # this could be moved into SETTINGS or somesuch, but meh. not needed now. OPTS = 0 + class GFMReader(pelican.readers.BaseReader): enabled = True """GFM-flavored Reader for the Pelican system. diff --git a/pelican-gfm/gfmSetup.py b/pelican-gfm/gfmSetup.py index 8afba8cf4..6bf5dcb44 100644 --- a/pelican-gfm/gfmSetup.py +++ b/pelican-gfm/gfmSetup.py @@ -10,6 +10,7 @@ # other operating systems / package # managers could be written for this + def dpkg_installed(package): """ Uses Dpkg to determine whether or not a package is installed requires: """ @@ -52,7 +53,7 @@ def dpkg_packages_installed(): if len(need_to_be_installed) is not 0: raise Exception("Could not find the following required\ - packages: " + need_to_be_installed ) + packages: " + need_to_be_installed) ins = False else: ins = True From 962682a4c7da769b709aa8080c36d46540016967 Mon Sep 17 00:00:00 2001 From: dfoulks Date: Tue, 17 Dec 2019 10:09:28 -0600 Subject: [PATCH 11/24] fixed python3 import --- pelican-gfm/__init__.py | 2 +- pelican-gfm/gfm.py | 4 ++-- pelican-gfm/gfmSetup.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pelican-gfm/__init__.py b/pelican-gfm/__init__.py index 513a4a685..7789b68dc 100644 --- a/pelican-gfm/__init__.py +++ b/pelican-gfm/__init__.py @@ -1,3 +1,3 @@ #!/usr/bin/environment python333 -B -from gfm import * +from .gfm import * diff --git a/pelican-gfm/gfm.py b/pelican-gfm/gfm.py index c29ecd314..373bab93d 100644 --- a/pelican-gfm/gfm.py +++ b/pelican-gfm/gfm.py @@ -25,8 +25,8 @@ import ctypes import time import re -import gfmSetup -import config +import .gfmSetup +import .config import pelican.utils import pelican.signals import pelican.readers diff --git a/pelican-gfm/gfmSetup.py b/pelican-gfm/gfmSetup.py index 6bf5dcb44..0af5b109f 100644 --- a/pelican-gfm/gfmSetup.py +++ b/pelican-gfm/gfmSetup.py @@ -3,7 +3,7 @@ import sys import os import subprocess -import config +import .config from backports import tempfile # Eventually, equivalents for From 7bb91158f1d2b8ba3578b56fa77929deff403932 Mon Sep 17 00:00:00 2001 From: dfoulks Date: Tue, 17 Dec 2019 11:03:43 -0600 Subject: [PATCH 12/24] changed some relative pathing import issues also removed an erroneous interpreter typo --- pelican-gfm/__init__.py | 5 +++-- pelican-gfm/gfm.py | 5 +++-- pelican-gfm/gfmSetup.py | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/pelican-gfm/__init__.py b/pelican-gfm/__init__.py index 7789b68dc..c522c9ea7 100644 --- a/pelican-gfm/__init__.py +++ b/pelican-gfm/__init__.py @@ -1,3 +1,4 @@ -#!/usr/bin/environment python333 -B +#!/usr/bin/environment python3 -B + +from gfm import * -from .gfm import * diff --git a/pelican-gfm/gfm.py b/pelican-gfm/gfm.py index 373bab93d..39f509b9f 100644 --- a/pelican-gfm/gfm.py +++ b/pelican-gfm/gfm.py @@ -1,4 +1,5 @@ #!/usr/bin/python3 -B + # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -25,11 +26,11 @@ import ctypes import time import re -import .gfmSetup -import .config import pelican.utils import pelican.signals import pelican.readers +import gfmSetup +import config _LIBDIR = config.LIBCMARKLOCATION _LIBCMARK = 'libcmark-gfm.so' diff --git a/pelican-gfm/gfmSetup.py b/pelican-gfm/gfmSetup.py index 0af5b109f..6bf5dcb44 100644 --- a/pelican-gfm/gfmSetup.py +++ b/pelican-gfm/gfmSetup.py @@ -3,7 +3,7 @@ import sys import os import subprocess -import .config +import config from backports import tempfile # Eventually, equivalents for From 6df3efdf2d92fdb3b9921b97a0b337a0a05e14e6 Mon Sep 17 00:00:00 2001 From: dfoulks Date: Tue, 17 Dec 2019 14:51:50 -0600 Subject: [PATCH 13/24] Ensured that the whole thing works --- pelican-gfm/Readme.md | 33 +++++++++++++++++---- pelican-gfm/{config.py => Settings.py} | 17 +++++++---- pelican-gfm/__init__.py | 7 +++-- pelican-gfm/gfm.py | 31 ++++++++++--------- pelican-gfm/gfmSetup.py | 41 +++++++++++++++++--------- 5 files changed, 85 insertions(+), 44 deletions(-) rename pelican-gfm/{config.py => Settings.py} (78%) diff --git a/pelican-gfm/Readme.md b/pelican-gfm/Readme.md index 0ec2c77d2..01595e761 100644 --- a/pelican-gfm/Readme.md +++ b/pelican-gfm/Readme.md @@ -1,18 +1,41 @@ -pelican-gfm +Pelican GFM =========== -A reader that leverages GitHub's C-based markdown library to translate GitHub Flavored Markdown to html. + +Pelican GFM is a reader that leverages GitHub's C-based markdown library to translate GitHub Flavored Markdown to html. Requirements ============ -pelican-gfm has no requirements outside of the python standard library and pelican pelican itself. + +Pelican GFM has no requirements outside of the python standard library and pelican itself. How to Use ========= -Drop the entire pelican-gfm directory into the plugin path and invoke it from your pelicanconf.py to have pelican-gfm render contents/\*.md + +Drop the entire pelican-gfm directory into the plugin path you have defined in your pelicanconf.py +Ensure that 'pelican-gfm' is on the list of plugins. + +## Settings and Configuration + +The Settings.py file contains the configurables for pelican-gfm. + - Location of the cmark lib files + - Version of the GitHub Cmark you wish to use + - cmark extensions you wish to use. + +## Pre-Flight Checks + +Before Pelican GFM run it checks: + - that the packages required to build the cmark files are present + - that any packages known to conflict with this reader are not present + - that the libcmark files required by the reader are present + +## Configuring Pelican GFM + +If Pelican GFM finds that there is a configuration issue it will recommend that you run the following: + `python gfmSetup.py` Syntax ====== -This plugin leverages [GitHub Flavored Markdown](https://github.github.com/gfm/) in `.md` files to generate html pages. +This plugin leverages [GitHub Flavored Markdown](https://github.github.com/gfm/) in `md, markdown, mkd, mdown` files to generate html pages. Attribution diff --git a/pelican-gfm/config.py b/pelican-gfm/Settings.py similarity index 78% rename from pelican-gfm/config.py rename to pelican-gfm/Settings.py index 4a9bad4f1..1a32c47fe 100644 --- a/pelican-gfm/config.py +++ b/pelican-gfm/Settings.py @@ -1,15 +1,11 @@ +#!/usr/bin/python -B + # Eventual location of the libcmark files LIBCMARKLOCATION = "/usr/lib/x86_64-linux-gnu" -# This is the archive of GitHub's cmark-gfm files -ARCHIVES = "https://github.com/github/cmark-gfm/archive" - # This is the GitHub Cmark version we're using VERSION = "0.28.3.gfm.12" -# The name of the local tarball that will be downloaded -LOCAL = "cmark-gfm.$VERSION.orig.tar.gz" - # The GFM extensions that we want to use EXTENSIONS = ( 'autolink', @@ -18,3 +14,12 @@ 'tagfilter', ) +# Do not change these +# Do not change these +# Do not change these + +# This is the archive of GitHub's cmark-gfm files +ARCHIVES = "https://github.com/github/cmark-gfm/archive" +# The name of the local tarball that will be downloaded +LOCAL = "cmark-gfm.VERSION.orig.tar.gz" + diff --git a/pelican-gfm/__init__.py b/pelican-gfm/__init__.py index c522c9ea7..1e9d106ce 100644 --- a/pelican-gfm/__init__.py +++ b/pelican-gfm/__init__.py @@ -1,4 +1,5 @@ -#!/usr/bin/environment python3 -B - -from gfm import * +#!/usr/bin/environment python -B +from .gfm import * +from .gfmSetup import * +from .Settings import * diff --git a/pelican-gfm/gfm.py b/pelican-gfm/gfm.py index 39f509b9f..a8d1bb7af 100644 --- a/pelican-gfm/gfm.py +++ b/pelican-gfm/gfm.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3 -B +#!/usr/bin/python -B # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file @@ -29,10 +29,10 @@ import pelican.utils import pelican.signals import pelican.readers -import gfmSetup -import config +from . import gfmSetup +from . import Settings -_LIBDIR = config.LIBCMARKLOCATION +_LIBDIR = Settings.LIBCMARKLOCATION _LIBCMARK = 'libcmark-gfm.so' _LIBEXT = 'libcmark-gfmextensions.so' try: @@ -61,8 +61,7 @@ F_cmark_parser_finish.restype = ctypes.c_void_p F_cmark_parser_finish.argtypes = (ctypes.c_void_p,) -F_cmark_parser_attach_syntax_extension = \ - cmark.cmark_parser_attach_syntax_extension +F_cmark_parser_attach_syntax_extension = cmark.cmark_parser_attach_syntax_extension F_cmark_parser_attach_syntax_extension.restype = ctypes.c_int F_cmark_parser_attach_syntax_extension.argtypes = ( ctypes.c_void_p, @@ -105,7 +104,7 @@ class GFMReader(pelican.readers.BaseReader): enabled = True - """GFM-flavored Reader for the Pelican system. + """GitHub Flavored Markdown Reader for the Pelican system. Pelican looks for all subclasses of BaseReader, and automatically registers them for the file extensions listed below. Thus, nothing @@ -169,11 +168,11 @@ def read(self, source_path): text = '\n'.join(lines[i:]) # Render the markdown into HTML -# if sys.version_info >= (3, 0): -# text = text.encode('utf-8') -# content = self.render(text).decode('utf-8') -# else: - content = self.render(text) + if sys.version_info >= (3, 0): + text = text.encode('utf-8') + content = self.render(text).decode('utf-8') + else: + content = self.render(text) # Redo the slug for articles. if parts[0] == 'articles' and 'title' in metadata: @@ -192,8 +191,8 @@ def render(self, text): parser = F_cmark_parser_new(OPTS) assert parser - for name in config.EXTENSIONS: - ext = F_cmark_find_syntax_extension(name) + for name in Settings.EXTENSIONS: + ext = F_cmark_find_syntax_extension(name.encode('utf-8')) assert ext rv = F_cmark_parser_attach_syntax_extension(parser, ext) assert rv @@ -209,9 +208,9 @@ def render(self, text): def add_readers(readers): msg = "GFM plugin cannot find the required libcmark files.\ - Please run gfmSetup.configure() to build and\ + Please run python gfmSetup.py to build and\ configure the appropriate libcmark files" - if str(gfmSetup.test_configuration()) == "0": + if gfmSetup.test_configuration(): readers.reader_classes['md'] = GFMReader else: raise Exception(msg) diff --git a/pelican-gfm/gfmSetup.py b/pelican-gfm/gfmSetup.py index 6bf5dcb44..7895da382 100644 --- a/pelican-gfm/gfmSetup.py +++ b/pelican-gfm/gfmSetup.py @@ -1,10 +1,22 @@ -#!/usr/bin/python3 -B +#!/usr/bin/python -B import sys import os import subprocess -import config -from backports import tempfile + +# Importing in python2 and python3 is different +# for these for some reason. this should catch +# the issues either way + +try: + import tempfile +except (ImportError, ValueError): + from backports import tempfile + +try: + from . import Settings +except (ImportError, ValueError): + import Settings # Eventually, equivalents for # other operating systems / package @@ -72,24 +84,26 @@ def setup(): # Configure the environment if it's not already configured with tempfile.TemporaryDirectory() as WORKSPACE: + # Pull into the workspace subprocess.call([ "wget", "--quiet", - config.ARCHIVES + "/" + config.VERSION + ".tar.gz", + Settings.ARCHIVES + "/" + Settings.VERSION + ".tar.gz", WORKSPACE, "-P", WORKSPACE ]) + # Untar the files subprocess.call([ 'tar', 'zxf', - WORKSPACE + "/" + config.VERSION + ".tar.gz", + WORKSPACE + "/" + Settings.VERSION + ".tar.gz", "-C", WORKSPACE ] ) - - BUILDSPACE = WORKSPACE + "/" + "cmark-gfm-" + config.VERSION + "/build" + # Create a buildspace for your cmake operation + BUILDSPACE = WORKSPACE + "/cmark-gfm-" + Settings.VERSION + "/build" if not os.path.isdir(BUILDSPACE): os.mkdir(BUILDSPACE) @@ -109,23 +123,22 @@ def setup(): print("Moving files") subprocess.call([ "mv", - BUILDSPACE + "/src/libcmark-gfm.so." + config.VERSION, - config.LIBCMARKLOCATION + "libcmark-gfm.so" + BUILDSPACE + "/src/libcmark-gfm.so." + Settings.VERSION, + Settings.LIBCMARKLOCATION + "/libcmark-gfm.so" ] ) subprocess.call([ "mv", - BUILDSPACE + "/extensions/libcmark-gfmextensions.so." + config.VERSION, - config.LIBCMARKLOCATION + "libcmark-gfmextensions.so" + BUILDSPACE + "/extensions/libcmark-gfmextensions.so." + Settings.VERSION, + Settings.LIBCMARKLOCATION + "/libcmark-gfmextensions.so" ] ) def test_configuration(): """ Tests to ensure that the files that the plugin needs are in place. """ - CMARKPATH = config.LIBCMARKLOCATION + "/libcmark-gfm.so." + config.VERSION - if os.path.isfile(config.LIBCMARKLOCATION + "/libcmark-gfm.so") and \ - os.path.isfile(config.LIBCMARKLOCATION + "/libcmark-gfmextensions.so"): + if os.path.isfile(Settings.LIBCMARKLOCATION + "/libcmark-gfm.so") and \ + os.path.isfile(Settings.LIBCMARKLOCATION + "/libcmark-gfmextensions.so"): return True else: return False From f9518da8de10f22d29713e9903aeff65b704e65b Mon Sep 17 00:00:00 2001 From: dfoulks Date: Tue, 17 Dec 2019 15:22:57 -0600 Subject: [PATCH 14/24] Added OPTS to settings --- pelican-gfm/Settings.py | 2 +- pelican-gfm/gfm.py | 9 ++------- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/pelican-gfm/Settings.py b/pelican-gfm/Settings.py index 1a32c47fe..b7520ecdf 100644 --- a/pelican-gfm/Settings.py +++ b/pelican-gfm/Settings.py @@ -13,7 +13,7 @@ 'strikethrough', 'tagfilter', ) - +OPTS = 0 # Do not change these # Do not change these # Do not change these diff --git a/pelican-gfm/gfm.py b/pelican-gfm/gfm.py index a8d1bb7af..ebe203792 100644 --- a/pelican-gfm/gfm.py +++ b/pelican-gfm/gfm.py @@ -97,11 +97,6 @@ # technically, maybe install an atexit() to release the plugins -# Options for the GFM rendering call -# this could be moved into SETTINGS or somesuch, but meh. not needed now. -OPTS = 0 - - class GFMReader(pelican.readers.BaseReader): enabled = True """GitHub Flavored Markdown Reader for the Pelican system. @@ -189,7 +184,7 @@ def read(self, source_path): def render(self, text): "Use cmark-gfm to render the Markdown into an HTML fragment." - parser = F_cmark_parser_new(OPTS) + parser = F_cmark_parser_new(Settings.OPTS) assert parser for name in Settings.EXTENSIONS: ext = F_cmark_find_syntax_extension(name.encode('utf-8')) @@ -200,7 +195,7 @@ def render(self, text): F_cmark_parser_feed(parser, text, len(text)) doc = F_cmark_parser_finish(parser) assert doc - output = F_cmark_render_html(doc, OPTS, exts) + output = F_cmark_render_html(doc, Settings.OPTS, exts) F_cmark_parser_free(parser) F_cmark_node_free(doc) return output From b897337b0b458fe1a65d555b9d8b25b75aa96495 Mon Sep 17 00:00:00 2001 From: dfoulks Date: Tue, 17 Dec 2019 16:09:22 -0600 Subject: [PATCH 15/24] pycodestyled it and ensured it works again. Will write some tests tomorrow --- pelican-gfm/gfm.py | 1 + pelican-gfm/gfmSetup.py | 16 ++++------------ 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/pelican-gfm/gfm.py b/pelican-gfm/gfm.py index ebe203792..9e9790f15 100644 --- a/pelican-gfm/gfm.py +++ b/pelican-gfm/gfm.py @@ -97,6 +97,7 @@ # technically, maybe install an atexit() to release the plugins + class GFMReader(pelican.readers.BaseReader): enabled = True """GitHub Flavored Markdown Reader for the Pelican system. diff --git a/pelican-gfm/gfmSetup.py b/pelican-gfm/gfmSetup.py index 7895da382..1927595d6 100644 --- a/pelican-gfm/gfmSetup.py +++ b/pelican-gfm/gfmSetup.py @@ -121,18 +121,10 @@ def setup(): # Move the libcmark.so artifacts in place print("Moving files") - subprocess.call([ - "mv", - BUILDSPACE + "/src/libcmark-gfm.so." + Settings.VERSION, - Settings.LIBCMARKLOCATION + "/libcmark-gfm.so" - ] - ) - subprocess.call([ - "mv", - BUILDSPACE + "/extensions/libcmark-gfmextensions.so." + Settings.VERSION, - Settings.LIBCMARKLOCATION + "/libcmark-gfmextensions.so" - ] - ) + shutil.move(BUILDSPACE + "/src/libcmark-gfm.so." + Settings.VERSION, + Settings.LIBCMARKLOCATION + "/libcmark-gfm.so") + shutil.move(BUILDSPACE + "/extensions/libcmark-gfmextensions.so." + Settings.VERSION, + Settings.LIBCMARKLOCATION + "/libcmark-gfmextensions.so") def test_configuration(): From 788e89b6daf762492b2605ce306f04902a13631f Mon Sep 17 00:00:00 2001 From: dfoulks Date: Tue, 17 Dec 2019 16:45:53 -0600 Subject: [PATCH 16/24] Updated readme file tried to get this to work when invoked with python or python3. currently works best with python3. something about the tempfile.TemporaryDirectory() call not working. I can handle that another day. --- pelican-gfm/Readme.md | 4 ++++ pelican-gfm/gfmSetup.py | 1 + 2 files changed, 5 insertions(+) diff --git a/pelican-gfm/Readme.md b/pelican-gfm/Readme.md index 01595e761..96621a932 100644 --- a/pelican-gfm/Readme.md +++ b/pelican-gfm/Readme.md @@ -3,6 +3,10 @@ Pelican GFM Pelican GFM is a reader that leverages GitHub's C-based markdown library to translate GitHub Flavored Markdown to html. +## Note + +It is important to note that this currently only works on debian systems + Requirements ============ diff --git a/pelican-gfm/gfmSetup.py b/pelican-gfm/gfmSetup.py index 1927595d6..077d4d88c 100644 --- a/pelican-gfm/gfmSetup.py +++ b/pelican-gfm/gfmSetup.py @@ -3,6 +3,7 @@ import sys import os import subprocess +import shutil # Importing in python2 and python3 is different # for these for some reason. this should catch From 8b6c56e0a4c4e625a337496600a29c796d650db1 Mon Sep 17 00:00:00 2001 From: dfoulks Date: Wed, 18 Dec 2019 12:26:35 -0600 Subject: [PATCH 17/24] Added a test for the register / add_readers function. Updated the readme --- pelican-gfm/Readme.md | 4 ++++ pelican-gfm/gfm.py | 20 +++++++++++++++----- pelican-gfm/gfmSetup.py | 15 ++++++--------- pelican-gfm/test.py | 17 +++++++++++++++++ 4 files changed, 42 insertions(+), 14 deletions(-) create mode 100644 pelican-gfm/test.py diff --git a/pelican-gfm/Readme.md b/pelican-gfm/Readme.md index 96621a932..4d826590a 100644 --- a/pelican-gfm/Readme.md +++ b/pelican-gfm/Readme.md @@ -37,6 +37,10 @@ Before Pelican GFM run it checks: If Pelican GFM finds that there is a configuration issue it will recommend that you run the following: `python gfmSetup.py` +## Testing + +There is a unittest written for gfm. the test will register and spawn a new reader and return true if there were no issues. + Syntax ====== This plugin leverages [GitHub Flavored Markdown](https://github.github.com/gfm/) in `md, markdown, mkd, mdown` files to generate html pages. diff --git a/pelican-gfm/gfm.py b/pelican-gfm/gfm.py index 9e9790f15..2a5c670f6 100644 --- a/pelican-gfm/gfm.py +++ b/pelican-gfm/gfm.py @@ -29,8 +29,16 @@ import pelican.utils import pelican.signals import pelican.readers -from . import gfmSetup -from . import Settings + +try: + from . import gfmSetup +except ImportError: + import gfmSetup + +try: + from . import Settings +except: + import Settings _LIBDIR = Settings.LIBCMARKLOCATION _LIBCMARK = 'libcmark-gfm.so' @@ -204,13 +212,15 @@ def render(self, text): def add_readers(readers): msg = "GFM plugin cannot find the required libcmark files.\ - Please run python gfmSetup.py to build and\ - configure the appropriate libcmark files" + Please run python3 gfmSetup.py as a user with write permission \ + to the directory into which the libcmark files will be placed." if gfmSetup.test_configuration(): readers.reader_classes['md'] = GFMReader + return True else: raise Exception(msg) def register(): - pelican.signals.readers_init.connect(add_readers) + reader = pelican.signals.readers_init.connect(add_readers) + return(reader) diff --git a/pelican-gfm/gfmSetup.py b/pelican-gfm/gfmSetup.py index 077d4d88c..da30e4668 100644 --- a/pelican-gfm/gfmSetup.py +++ b/pelican-gfm/gfmSetup.py @@ -4,6 +4,7 @@ import os import subprocess import shutil +import tarfile # Importing in python2 and python3 is different # for these for some reason. this should catch @@ -89,20 +90,16 @@ def setup(): subprocess.call([ "wget", "--quiet", - Settings.ARCHIVES + "/" + Settings.VERSION + ".tar.gz", + os.path.join(Settings.ARCHIVES, Settings.VERSION + ".tar.gz"), WORKSPACE, "-P", WORKSPACE ]) + # Untar the files - subprocess.call([ - 'tar', - 'zxf', - WORKSPACE + "/" + Settings.VERSION + ".tar.gz", - "-C", - WORKSPACE - ] - ) + tf = tarfile.open(os.path.join(WORKSPACE, Settings.VERSION + ".tar.gz")) + tf.extractall(path=WORKSPACE) + # Create a buildspace for your cmake operation BUILDSPACE = WORKSPACE + "/cmark-gfm-" + Settings.VERSION + "/build" diff --git a/pelican-gfm/test.py b/pelican-gfm/test.py new file mode 100644 index 000000000..ae71182b1 --- /dev/null +++ b/pelican-gfm/test.py @@ -0,0 +1,17 @@ +#!/usr/bin/python -B + +import unittest + +# python2 and python3 differ on how to do this it seems +try: + from .gfm import * +except ImportError: + import gfm + +class gfmTest(unittest.TestCase): + + def test_for_gfm(self): + self.assertTrue(gfm.register()) + +if __name__ == '__main__': + unittest.main() From d22316b1cf97ce37f609b0f436addf9ced2621c1 Mon Sep 17 00:00:00 2001 From: dfoulks Date: Wed, 18 Dec 2019 17:29:41 -0600 Subject: [PATCH 18/24] Updated error message unconfigured system will no instruct you as to how to configure the system --- pelican-gfm/gfm.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pelican-gfm/gfm.py b/pelican-gfm/gfm.py index 2a5c670f6..1d429c764 100644 --- a/pelican-gfm/gfm.py +++ b/pelican-gfm/gfm.py @@ -48,8 +48,10 @@ cmark_ext = ctypes.CDLL(os.path.join(_LIBDIR, _LIBEXT)) except OSError: raise ImportError( - '%s not found. see build-mark.sh and gfm_reader.py' - % _LIBCMARK + '%s not found. run python3 gfmSetup.py as a\ + user with write permisions to %s or change LIBCMARKLOCATION\ + in Settings.py' + % (_LIBCMARK, _LIBDIR) ) # Use ctypes to access the functions in libcmark-gfm From a056e7ed9aee1972f0669c9f029ae665a7709948 Mon Sep 17 00:00:00 2001 From: dfoulks Date: Thu, 19 Dec 2019 16:59:12 -0600 Subject: [PATCH 19/24] Used __future__ to avoid the try / except for importing --- pelican-gfm/__init__.py | 4 +--- pelican-gfm/gfm.py | 13 +++---------- pelican-gfm/gfmSetup.py | 19 ++++--------------- 3 files changed, 8 insertions(+), 28 deletions(-) diff --git a/pelican-gfm/__init__.py b/pelican-gfm/__init__.py index 1e9d106ce..d2719baa5 100644 --- a/pelican-gfm/__init__.py +++ b/pelican-gfm/__init__.py @@ -1,5 +1,3 @@ #!/usr/bin/environment python -B - +from __future__ import absolute_import from .gfm import * -from .gfmSetup import * -from .Settings import * diff --git a/pelican-gfm/gfm.py b/pelican-gfm/gfm.py index 1d429c764..13fb624ec 100644 --- a/pelican-gfm/gfm.py +++ b/pelican-gfm/gfm.py @@ -20,7 +20,7 @@ # # gfm_reader.py -- GitHub-Flavored Markdown reader for Pelican # - +from __future__ import absolute_import import sys import os import ctypes @@ -30,15 +30,8 @@ import pelican.signals import pelican.readers -try: - from . import gfmSetup -except ImportError: - import gfmSetup - -try: - from . import Settings -except: - import Settings +from . import gfmSetup +from . import Settings _LIBDIR = Settings.LIBCMARKLOCATION _LIBCMARK = 'libcmark-gfm.so' diff --git a/pelican-gfm/gfmSetup.py b/pelican-gfm/gfmSetup.py index da30e4668..3771bf16e 100644 --- a/pelican-gfm/gfmSetup.py +++ b/pelican-gfm/gfmSetup.py @@ -1,24 +1,13 @@ -#!/usr/bin/python -B +#!/usr/bin/python3 -B +from __future__ import absolute_import import sys import os import subprocess import shutil import tarfile - -# Importing in python2 and python3 is different -# for these for some reason. this should catch -# the issues either way - -try: - import tempfile -except (ImportError, ValueError): - from backports import tempfile - -try: - from . import Settings -except (ImportError, ValueError): - import Settings +import tempfile +from . import Settings # Eventually, equivalents for # other operating systems / package From 76574fb889ef9da9df3b51eadd874325e34134f0 Mon Sep 17 00:00:00 2001 From: dfoulks Date: Fri, 20 Dec 2019 12:29:52 -0600 Subject: [PATCH 20/24] Removed absolute imports Couldn't get thee imports to work consistently. --- pelican-gfm/__init__.py | 1 - pelican-gfm/gfm.py | 5 +- pelican-gfm/gfmSetup.py | 4 +- pelican-gfm/test.py | 6 +- pelican_gfm/Readme.md | 52 ++++++++++ pelican_gfm/Settings.py | 25 +++++ pelican_gfm/__init__.py | 3 + pelican_gfm/gfm.py | 221 ++++++++++++++++++++++++++++++++++++++++ pelican_gfm/gfmSetup.py | 132 ++++++++++++++++++++++++ 9 files changed, 438 insertions(+), 11 deletions(-) create mode 100644 pelican_gfm/Readme.md create mode 100644 pelican_gfm/Settings.py create mode 100644 pelican_gfm/__init__.py create mode 100644 pelican_gfm/gfm.py create mode 100644 pelican_gfm/gfmSetup.py diff --git a/pelican-gfm/__init__.py b/pelican-gfm/__init__.py index d2719baa5..924eaf145 100644 --- a/pelican-gfm/__init__.py +++ b/pelican-gfm/__init__.py @@ -1,3 +1,2 @@ #!/usr/bin/environment python -B -from __future__ import absolute_import from .gfm import * diff --git a/pelican-gfm/gfm.py b/pelican-gfm/gfm.py index 13fb624ec..eebce8e7b 100644 --- a/pelican-gfm/gfm.py +++ b/pelican-gfm/gfm.py @@ -20,7 +20,6 @@ # # gfm_reader.py -- GitHub-Flavored Markdown reader for Pelican # -from __future__ import absolute_import import sys import os import ctypes @@ -30,8 +29,8 @@ import pelican.signals import pelican.readers -from . import gfmSetup -from . import Settings +import gfmSetup +import Settings _LIBDIR = Settings.LIBCMARKLOCATION _LIBCMARK = 'libcmark-gfm.so' diff --git a/pelican-gfm/gfmSetup.py b/pelican-gfm/gfmSetup.py index 3771bf16e..e2e543e05 100644 --- a/pelican-gfm/gfmSetup.py +++ b/pelican-gfm/gfmSetup.py @@ -1,13 +1,11 @@ #!/usr/bin/python3 -B -from __future__ import absolute_import - import sys import os import subprocess import shutil import tarfile import tempfile -from . import Settings +import Settings # Eventually, equivalents for # other operating systems / package diff --git a/pelican-gfm/test.py b/pelican-gfm/test.py index ae71182b1..75ce56424 100644 --- a/pelican-gfm/test.py +++ b/pelican-gfm/test.py @@ -3,10 +3,8 @@ import unittest # python2 and python3 differ on how to do this it seems -try: - from .gfm import * -except ImportError: - import gfm + +import gfm class gfmTest(unittest.TestCase): diff --git a/pelican_gfm/Readme.md b/pelican_gfm/Readme.md new file mode 100644 index 000000000..4d826590a --- /dev/null +++ b/pelican_gfm/Readme.md @@ -0,0 +1,52 @@ +Pelican GFM +=========== + +Pelican GFM is a reader that leverages GitHub's C-based markdown library to translate GitHub Flavored Markdown to html. + +## Note + +It is important to note that this currently only works on debian systems + +Requirements +============ + +Pelican GFM has no requirements outside of the python standard library and pelican itself. + +How to Use +========= + +Drop the entire pelican-gfm directory into the plugin path you have defined in your pelicanconf.py +Ensure that 'pelican-gfm' is on the list of plugins. + +## Settings and Configuration + +The Settings.py file contains the configurables for pelican-gfm. + - Location of the cmark lib files + - Version of the GitHub Cmark you wish to use + - cmark extensions you wish to use. + +## Pre-Flight Checks + +Before Pelican GFM run it checks: + - that the packages required to build the cmark files are present + - that any packages known to conflict with this reader are not present + - that the libcmark files required by the reader are present + +## Configuring Pelican GFM + +If Pelican GFM finds that there is a configuration issue it will recommend that you run the following: + `python gfmSetup.py` + +## Testing + +There is a unittest written for gfm. the test will register and spawn a new reader and return true if there were no issues. + +Syntax +====== +This plugin leverages [GitHub Flavored Markdown](https://github.github.com/gfm/) in `md, markdown, mkd, mdown` files to generate html pages. + + +Attribution +=========== +`pelican-gfm` is based on [pelican_gfm_script](https://github.com/apache/infrastructure-website/blob/master/gfm_reader.py) +Originally written by: Greg Stein diff --git a/pelican_gfm/Settings.py b/pelican_gfm/Settings.py new file mode 100644 index 000000000..b7520ecdf --- /dev/null +++ b/pelican_gfm/Settings.py @@ -0,0 +1,25 @@ +#!/usr/bin/python -B + +# Eventual location of the libcmark files +LIBCMARKLOCATION = "/usr/lib/x86_64-linux-gnu" + +# This is the GitHub Cmark version we're using +VERSION = "0.28.3.gfm.12" + +# The GFM extensions that we want to use +EXTENSIONS = ( + 'autolink', + 'table', + 'strikethrough', + 'tagfilter', +) +OPTS = 0 +# Do not change these +# Do not change these +# Do not change these + +# This is the archive of GitHub's cmark-gfm files +ARCHIVES = "https://github.com/github/cmark-gfm/archive" +# The name of the local tarball that will be downloaded +LOCAL = "cmark-gfm.VERSION.orig.tar.gz" + diff --git a/pelican_gfm/__init__.py b/pelican_gfm/__init__.py new file mode 100644 index 000000000..d2719baa5 --- /dev/null +++ b/pelican_gfm/__init__.py @@ -0,0 +1,3 @@ +#!/usr/bin/environment python -B +from __future__ import absolute_import +from .gfm import * diff --git a/pelican_gfm/gfm.py b/pelican_gfm/gfm.py new file mode 100644 index 000000000..13fb624ec --- /dev/null +++ b/pelican_gfm/gfm.py @@ -0,0 +1,221 @@ +#!/usr/bin/python -B + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# +# gfm_reader.py -- GitHub-Flavored Markdown reader for Pelican +# +from __future__ import absolute_import +import sys +import os +import ctypes +import time +import re +import pelican.utils +import pelican.signals +import pelican.readers + +from . import gfmSetup +from . import Settings + +_LIBDIR = Settings.LIBCMARKLOCATION +_LIBCMARK = 'libcmark-gfm.so' +_LIBEXT = 'libcmark-gfmextensions.so' +try: + cmark = ctypes.CDLL(os.path.join(_LIBDIR, _LIBCMARK)) + cmark_ext = ctypes.CDLL(os.path.join(_LIBDIR, _LIBEXT)) +except OSError: + raise ImportError( + '%s not found. run python3 gfmSetup.py as a\ + user with write permisions to %s or change LIBCMARKLOCATION\ + in Settings.py' + % (_LIBCMARK, _LIBDIR) + ) + +# Use ctypes to access the functions in libcmark-gfm +F_cmark_parser_new = cmark.cmark_parser_new +F_cmark_parser_new.restype = ctypes.c_void_p +F_cmark_parser_new.argtypes = (ctypes.c_int,) + +F_cmark_parser_feed = cmark.cmark_parser_feed +F_cmark_parser_feed.restype = None +F_cmark_parser_feed.argtypes = ( + ctypes.c_void_p, + ctypes.c_char_p, + ctypes.c_size_t + ) + +F_cmark_parser_finish = cmark.cmark_parser_finish +F_cmark_parser_finish.restype = ctypes.c_void_p +F_cmark_parser_finish.argtypes = (ctypes.c_void_p,) + +F_cmark_parser_attach_syntax_extension = cmark.cmark_parser_attach_syntax_extension +F_cmark_parser_attach_syntax_extension.restype = ctypes.c_int +F_cmark_parser_attach_syntax_extension.argtypes = ( + ctypes.c_void_p, + ctypes.c_void_p + ) + +F_cmark_parser_get_syntax_extensions = cmark.cmark_parser_get_syntax_extensions +F_cmark_parser_get_syntax_extensions.restype = ctypes.c_void_p +F_cmark_parser_get_syntax_extensions.argtypes = (ctypes.c_void_p,) + +F_cmark_parser_free = cmark.cmark_parser_free +F_cmark_parser_free.restype = None +F_cmark_parser_free.argtypes = (ctypes.c_void_p,) + +F_cmark_node_free = cmark.cmark_node_free +F_cmark_node_free.restype = None +F_cmark_node_free.argtypes = (ctypes.c_void_p,) + +F_cmark_find_syntax_extension = cmark.cmark_find_syntax_extension +F_cmark_find_syntax_extension.restype = ctypes.c_void_p +F_cmark_find_syntax_extension.argtypes = (ctypes.c_char_p,) + +F_cmark_render_html = cmark.cmark_render_html +F_cmark_render_html.restype = ctypes.c_char_p +F_cmark_render_html.argtypes = (ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p) + + +# Set up the libcmark-gfm library and its extensions +F_register = cmark_ext.core_extensions_ensure_registered +F_register.restype = None +F_register.argtypes = () +F_register() + +# technically, maybe install an atexit() to release the plugins + + +class GFMReader(pelican.readers.BaseReader): + enabled = True + """GitHub Flavored Markdown Reader for the Pelican system. + + Pelican looks for all subclasses of BaseReader, and automatically + registers them for the file extensions listed below. Thus, nothing + further is required by users of this Reader. + """ + + # NOTE: the builtin MarkdownReader must be disabled. Otherwise, it will be + # non-deterministic which Reader will be used for these files. + file_extensions = ['md', 'markdown', 'mkd', 'mdown'] + + # Metadata is specified as a single, colon-separated line, such as: + # + # Title: this is the title + # + # Note: name starts in column 0, no whitespace before colon, will be + # made lower-case, and value will be stripped + # + RE_METADATA = re.compile('^([A-za-z]+): (.*)$') + + def read(self, source_path): + # Prepare the "slug", which is the target file name. + # It will be the same as the source file, minus the leading + # ".../content/(articles|pages)" + # and with the extension removed (Pelican will add .html) + relpath = os.path.relpath(source_path, self.settings['PATH']) + parts = relpath.split(os.sep) + + # split off ext, keep base + parts[-1] = os.path.splitext(parts[-1])[0] + slug = os.sep.join(parts[1:]) + + metadata = { + 'slug': slug, + } + + # Fetch the source content, with a few appropriate tweaks + with pelican.utils.pelican_open(source_path) as text: + + # Extract the metadata from the header of the text + lines = text.splitlines() + for i in range(len(lines)): + line = lines[i] + match = GFMReader.RE_METADATA.match(line) + if match: + name = match.group(1).strip().lower() + if name != 'slug': + value = match.group(2).strip() + if name == 'date': + value = pelican.utils.get_date(value) + metadata[name] = value + # if name != 'title': + # print 'META:', name, value + elif not line.strip(): + # blank line + continue + else: + # reached actual content + break + + # Reassemble content, minus the metadata + text = '\n'.join(lines[i:]) + + # Render the markdown into HTML + if sys.version_info >= (3, 0): + text = text.encode('utf-8') + content = self.render(text).decode('utf-8') + else: + content = self.render(text) + + # Redo the slug for articles. + if parts[0] == 'articles' and 'title' in metadata: + metadata['slug'] = pelican.utils.slugify( + metadata['title'], + self.settings.get( + 'SLUG_SUBSTITUTIONS', + () + ) + ) + + return content, metadata + + def render(self, text): + "Use cmark-gfm to render the Markdown into an HTML fragment." + + parser = F_cmark_parser_new(Settings.OPTS) + assert parser + for name in Settings.EXTENSIONS: + ext = F_cmark_find_syntax_extension(name.encode('utf-8')) + assert ext + rv = F_cmark_parser_attach_syntax_extension(parser, ext) + assert rv + exts = F_cmark_parser_get_syntax_extensions(parser) + F_cmark_parser_feed(parser, text, len(text)) + doc = F_cmark_parser_finish(parser) + assert doc + output = F_cmark_render_html(doc, Settings.OPTS, exts) + F_cmark_parser_free(parser) + F_cmark_node_free(doc) + return output + + +def add_readers(readers): + msg = "GFM plugin cannot find the required libcmark files.\ + Please run python3 gfmSetup.py as a user with write permission \ + to the directory into which the libcmark files will be placed." + if gfmSetup.test_configuration(): + readers.reader_classes['md'] = GFMReader + return True + else: + raise Exception(msg) + + +def register(): + reader = pelican.signals.readers_init.connect(add_readers) + return(reader) diff --git a/pelican_gfm/gfmSetup.py b/pelican_gfm/gfmSetup.py new file mode 100644 index 000000000..3771bf16e --- /dev/null +++ b/pelican_gfm/gfmSetup.py @@ -0,0 +1,132 @@ +#!/usr/bin/python3 -B +from __future__ import absolute_import + +import sys +import os +import subprocess +import shutil +import tarfile +import tempfile +from . import Settings + +# Eventually, equivalents for +# other operating systems / package +# managers could be written for this + + +def dpkg_installed(package): + """ Uses Dpkg to determine whether or not a package is installed + requires: """ + t1 = subprocess.Popen(["dpkg", "-l"], stdout=subprocess.PIPE) + t2 = subprocess.Popen([ + "grep", + "-q", + package + ], stdout=subprocess.PIPE, stdin=t1.stdout) + data = t2.communicate() + ec = t2.wait() + return ec + + +def dpkg_packages_installed(): + """ Checking to see if the appropriate packages are installed""" + installed = ["cmake", "make", "wget"] + removed = [ + "libcmark-gfm-dev", + "libcmark-gfm-extensions-dev", + "libcmark-gfm0", + "libcmark-gfm-extensions0" + ] + + need_to_be_removed = [package for package in removed + if dpkg_installed(package) == 0 + ] + + need_to_be_installed = [ + package for package in installed + if dpkg_installed(package) == 1 + ] + + if len(need_to_be_removed) is not 0: + raise Exception("Found the following conflicting packages\ + which should be removed:" + str(need_to_be_removed)) + rem = False + else: + rem = True + + if len(need_to_be_installed) is not 0: + raise Exception("Could not find the following required\ + packages: " + need_to_be_installed) + ins = False + else: + ins = True + + if rem is True and ins is True: + print("Packages seem in order.") + return True + + +def setup(): + dpkg_packages_installed() + + if test_configuration() == 1: + print("System appears to be configured") + else: + # Configure the environment if it's not already configured + + with tempfile.TemporaryDirectory() as WORKSPACE: + # Pull into the workspace + subprocess.call([ + "wget", + "--quiet", + os.path.join(Settings.ARCHIVES, Settings.VERSION + ".tar.gz"), + WORKSPACE, + "-P", + WORKSPACE + ]) + + # Untar the files + tf = tarfile.open(os.path.join(WORKSPACE, Settings.VERSION + ".tar.gz")) + tf.extractall(path=WORKSPACE) + + # Create a buildspace for your cmake operation + BUILDSPACE = WORKSPACE + "/cmark-gfm-" + Settings.VERSION + "/build" + + if not os.path.isdir(BUILDSPACE): + os.mkdir(BUILDSPACE) + + thing1 = subprocess.Popen([ + "cmake", + "-DCMARK_TESTS=OFF", + "-DCMARK_STATIC=OFF", + ".." + ], cwd=BUILDSPACE) + thing1.wait() + + thing2 = subprocess.Popen(["make"], cwd=BUILDSPACE) + thing2.wait() + + # Move the libcmark.so artifacts in place + print("Moving files") + shutil.move(BUILDSPACE + "/src/libcmark-gfm.so." + Settings.VERSION, + Settings.LIBCMARKLOCATION + "/libcmark-gfm.so") + shutil.move(BUILDSPACE + "/extensions/libcmark-gfmextensions.so." + Settings.VERSION, + Settings.LIBCMARKLOCATION + "/libcmark-gfmextensions.so") + + +def test_configuration(): + """ Tests to ensure that the files that the plugin needs are in place. """ + if os.path.isfile(Settings.LIBCMARKLOCATION + "/libcmark-gfm.so") and \ + os.path.isfile(Settings.LIBCMARKLOCATION + "/libcmark-gfmextensions.so"): + return True + else: + return False + + +def configure(): + print("Checking out the configuration") + setup() + + +if __name__ == "__main__": + configure() From 6439c433cd25a3e2abedcc7a09a3f8ecbdd88608 Mon Sep 17 00:00:00 2001 From: dfoulks Date: Fri, 20 Dec 2019 15:06:12 -0600 Subject: [PATCH 21/24] got absolute imports working the format doesn't seem to gel with your contributing a plugin documentation. https://github.com/getpelican/pelican-plugins/blob/master/Contributing.rst --- pelican-gfm/Readme.md | 52 ----- pelican-gfm/__init__.py | 2 - pelican-gfm/gfm.py | 220 ------------------- pelican-gfm/gfmSetup.py | 130 ----------- pelican_gfm/Settings.py | 25 --- {pelican-gfm => pelican_gfm/gfm}/Settings.py | 0 pelican_gfm/{ => gfm}/__init__.py | 2 +- pelican_gfm/{ => gfm}/gfm.py | 4 +- pelican_gfm/{ => gfm}/gfmSetup.py | 2 +- {pelican-gfm => pelican_gfm}/test.py | 7 +- 10 files changed, 6 insertions(+), 438 deletions(-) delete mode 100644 pelican-gfm/Readme.md delete mode 100644 pelican-gfm/__init__.py delete mode 100644 pelican-gfm/gfm.py delete mode 100644 pelican-gfm/gfmSetup.py delete mode 100644 pelican_gfm/Settings.py rename {pelican-gfm => pelican_gfm/gfm}/Settings.py (100%) rename pelican_gfm/{ => gfm}/__init__.py (76%) rename pelican_gfm/{ => gfm}/gfm.py (99%) rename pelican_gfm/{ => gfm}/gfmSetup.py (99%) rename {pelican-gfm => pelican_gfm}/test.py (72%) diff --git a/pelican-gfm/Readme.md b/pelican-gfm/Readme.md deleted file mode 100644 index 4d826590a..000000000 --- a/pelican-gfm/Readme.md +++ /dev/null @@ -1,52 +0,0 @@ -Pelican GFM -=========== - -Pelican GFM is a reader that leverages GitHub's C-based markdown library to translate GitHub Flavored Markdown to html. - -## Note - -It is important to note that this currently only works on debian systems - -Requirements -============ - -Pelican GFM has no requirements outside of the python standard library and pelican itself. - -How to Use -========= - -Drop the entire pelican-gfm directory into the plugin path you have defined in your pelicanconf.py -Ensure that 'pelican-gfm' is on the list of plugins. - -## Settings and Configuration - -The Settings.py file contains the configurables for pelican-gfm. - - Location of the cmark lib files - - Version of the GitHub Cmark you wish to use - - cmark extensions you wish to use. - -## Pre-Flight Checks - -Before Pelican GFM run it checks: - - that the packages required to build the cmark files are present - - that any packages known to conflict with this reader are not present - - that the libcmark files required by the reader are present - -## Configuring Pelican GFM - -If Pelican GFM finds that there is a configuration issue it will recommend that you run the following: - `python gfmSetup.py` - -## Testing - -There is a unittest written for gfm. the test will register and spawn a new reader and return true if there were no issues. - -Syntax -====== -This plugin leverages [GitHub Flavored Markdown](https://github.github.com/gfm/) in `md, markdown, mkd, mdown` files to generate html pages. - - -Attribution -=========== -`pelican-gfm` is based on [pelican_gfm_script](https://github.com/apache/infrastructure-website/blob/master/gfm_reader.py) -Originally written by: Greg Stein diff --git a/pelican-gfm/__init__.py b/pelican-gfm/__init__.py deleted file mode 100644 index 924eaf145..000000000 --- a/pelican-gfm/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/environment python -B -from .gfm import * diff --git a/pelican-gfm/gfm.py b/pelican-gfm/gfm.py deleted file mode 100644 index eebce8e7b..000000000 --- a/pelican-gfm/gfm.py +++ /dev/null @@ -1,220 +0,0 @@ -#!/usr/bin/python -B - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -# -# gfm_reader.py -- GitHub-Flavored Markdown reader for Pelican -# -import sys -import os -import ctypes -import time -import re -import pelican.utils -import pelican.signals -import pelican.readers - -import gfmSetup -import Settings - -_LIBDIR = Settings.LIBCMARKLOCATION -_LIBCMARK = 'libcmark-gfm.so' -_LIBEXT = 'libcmark-gfmextensions.so' -try: - cmark = ctypes.CDLL(os.path.join(_LIBDIR, _LIBCMARK)) - cmark_ext = ctypes.CDLL(os.path.join(_LIBDIR, _LIBEXT)) -except OSError: - raise ImportError( - '%s not found. run python3 gfmSetup.py as a\ - user with write permisions to %s or change LIBCMARKLOCATION\ - in Settings.py' - % (_LIBCMARK, _LIBDIR) - ) - -# Use ctypes to access the functions in libcmark-gfm -F_cmark_parser_new = cmark.cmark_parser_new -F_cmark_parser_new.restype = ctypes.c_void_p -F_cmark_parser_new.argtypes = (ctypes.c_int,) - -F_cmark_parser_feed = cmark.cmark_parser_feed -F_cmark_parser_feed.restype = None -F_cmark_parser_feed.argtypes = ( - ctypes.c_void_p, - ctypes.c_char_p, - ctypes.c_size_t - ) - -F_cmark_parser_finish = cmark.cmark_parser_finish -F_cmark_parser_finish.restype = ctypes.c_void_p -F_cmark_parser_finish.argtypes = (ctypes.c_void_p,) - -F_cmark_parser_attach_syntax_extension = cmark.cmark_parser_attach_syntax_extension -F_cmark_parser_attach_syntax_extension.restype = ctypes.c_int -F_cmark_parser_attach_syntax_extension.argtypes = ( - ctypes.c_void_p, - ctypes.c_void_p - ) - -F_cmark_parser_get_syntax_extensions = cmark.cmark_parser_get_syntax_extensions -F_cmark_parser_get_syntax_extensions.restype = ctypes.c_void_p -F_cmark_parser_get_syntax_extensions.argtypes = (ctypes.c_void_p,) - -F_cmark_parser_free = cmark.cmark_parser_free -F_cmark_parser_free.restype = None -F_cmark_parser_free.argtypes = (ctypes.c_void_p,) - -F_cmark_node_free = cmark.cmark_node_free -F_cmark_node_free.restype = None -F_cmark_node_free.argtypes = (ctypes.c_void_p,) - -F_cmark_find_syntax_extension = cmark.cmark_find_syntax_extension -F_cmark_find_syntax_extension.restype = ctypes.c_void_p -F_cmark_find_syntax_extension.argtypes = (ctypes.c_char_p,) - -F_cmark_render_html = cmark.cmark_render_html -F_cmark_render_html.restype = ctypes.c_char_p -F_cmark_render_html.argtypes = (ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p) - - -# Set up the libcmark-gfm library and its extensions -F_register = cmark_ext.core_extensions_ensure_registered -F_register.restype = None -F_register.argtypes = () -F_register() - -# technically, maybe install an atexit() to release the plugins - - -class GFMReader(pelican.readers.BaseReader): - enabled = True - """GitHub Flavored Markdown Reader for the Pelican system. - - Pelican looks for all subclasses of BaseReader, and automatically - registers them for the file extensions listed below. Thus, nothing - further is required by users of this Reader. - """ - - # NOTE: the builtin MarkdownReader must be disabled. Otherwise, it will be - # non-deterministic which Reader will be used for these files. - file_extensions = ['md', 'markdown', 'mkd', 'mdown'] - - # Metadata is specified as a single, colon-separated line, such as: - # - # Title: this is the title - # - # Note: name starts in column 0, no whitespace before colon, will be - # made lower-case, and value will be stripped - # - RE_METADATA = re.compile('^([A-za-z]+): (.*)$') - - def read(self, source_path): - # Prepare the "slug", which is the target file name. - # It will be the same as the source file, minus the leading - # ".../content/(articles|pages)" - # and with the extension removed (Pelican will add .html) - relpath = os.path.relpath(source_path, self.settings['PATH']) - parts = relpath.split(os.sep) - - # split off ext, keep base - parts[-1] = os.path.splitext(parts[-1])[0] - slug = os.sep.join(parts[1:]) - - metadata = { - 'slug': slug, - } - - # Fetch the source content, with a few appropriate tweaks - with pelican.utils.pelican_open(source_path) as text: - - # Extract the metadata from the header of the text - lines = text.splitlines() - for i in range(len(lines)): - line = lines[i] - match = GFMReader.RE_METADATA.match(line) - if match: - name = match.group(1).strip().lower() - if name != 'slug': - value = match.group(2).strip() - if name == 'date': - value = pelican.utils.get_date(value) - metadata[name] = value - # if name != 'title': - # print 'META:', name, value - elif not line.strip(): - # blank line - continue - else: - # reached actual content - break - - # Reassemble content, minus the metadata - text = '\n'.join(lines[i:]) - - # Render the markdown into HTML - if sys.version_info >= (3, 0): - text = text.encode('utf-8') - content = self.render(text).decode('utf-8') - else: - content = self.render(text) - - # Redo the slug for articles. - if parts[0] == 'articles' and 'title' in metadata: - metadata['slug'] = pelican.utils.slugify( - metadata['title'], - self.settings.get( - 'SLUG_SUBSTITUTIONS', - () - ) - ) - - return content, metadata - - def render(self, text): - "Use cmark-gfm to render the Markdown into an HTML fragment." - - parser = F_cmark_parser_new(Settings.OPTS) - assert parser - for name in Settings.EXTENSIONS: - ext = F_cmark_find_syntax_extension(name.encode('utf-8')) - assert ext - rv = F_cmark_parser_attach_syntax_extension(parser, ext) - assert rv - exts = F_cmark_parser_get_syntax_extensions(parser) - F_cmark_parser_feed(parser, text, len(text)) - doc = F_cmark_parser_finish(parser) - assert doc - output = F_cmark_render_html(doc, Settings.OPTS, exts) - F_cmark_parser_free(parser) - F_cmark_node_free(doc) - return output - - -def add_readers(readers): - msg = "GFM plugin cannot find the required libcmark files.\ - Please run python3 gfmSetup.py as a user with write permission \ - to the directory into which the libcmark files will be placed." - if gfmSetup.test_configuration(): - readers.reader_classes['md'] = GFMReader - return True - else: - raise Exception(msg) - - -def register(): - reader = pelican.signals.readers_init.connect(add_readers) - return(reader) diff --git a/pelican-gfm/gfmSetup.py b/pelican-gfm/gfmSetup.py deleted file mode 100644 index e2e543e05..000000000 --- a/pelican-gfm/gfmSetup.py +++ /dev/null @@ -1,130 +0,0 @@ -#!/usr/bin/python3 -B -import sys -import os -import subprocess -import shutil -import tarfile -import tempfile -import Settings - -# Eventually, equivalents for -# other operating systems / package -# managers could be written for this - - -def dpkg_installed(package): - """ Uses Dpkg to determine whether or not a package is installed - requires: """ - t1 = subprocess.Popen(["dpkg", "-l"], stdout=subprocess.PIPE) - t2 = subprocess.Popen([ - "grep", - "-q", - package - ], stdout=subprocess.PIPE, stdin=t1.stdout) - data = t2.communicate() - ec = t2.wait() - return ec - - -def dpkg_packages_installed(): - """ Checking to see if the appropriate packages are installed""" - installed = ["cmake", "make", "wget"] - removed = [ - "libcmark-gfm-dev", - "libcmark-gfm-extensions-dev", - "libcmark-gfm0", - "libcmark-gfm-extensions0" - ] - - need_to_be_removed = [package for package in removed - if dpkg_installed(package) == 0 - ] - - need_to_be_installed = [ - package for package in installed - if dpkg_installed(package) == 1 - ] - - if len(need_to_be_removed) is not 0: - raise Exception("Found the following conflicting packages\ - which should be removed:" + str(need_to_be_removed)) - rem = False - else: - rem = True - - if len(need_to_be_installed) is not 0: - raise Exception("Could not find the following required\ - packages: " + need_to_be_installed) - ins = False - else: - ins = True - - if rem is True and ins is True: - print("Packages seem in order.") - return True - - -def setup(): - dpkg_packages_installed() - - if test_configuration() == 1: - print("System appears to be configured") - else: - # Configure the environment if it's not already configured - - with tempfile.TemporaryDirectory() as WORKSPACE: - # Pull into the workspace - subprocess.call([ - "wget", - "--quiet", - os.path.join(Settings.ARCHIVES, Settings.VERSION + ".tar.gz"), - WORKSPACE, - "-P", - WORKSPACE - ]) - - # Untar the files - tf = tarfile.open(os.path.join(WORKSPACE, Settings.VERSION + ".tar.gz")) - tf.extractall(path=WORKSPACE) - - # Create a buildspace for your cmake operation - BUILDSPACE = WORKSPACE + "/cmark-gfm-" + Settings.VERSION + "/build" - - if not os.path.isdir(BUILDSPACE): - os.mkdir(BUILDSPACE) - - thing1 = subprocess.Popen([ - "cmake", - "-DCMARK_TESTS=OFF", - "-DCMARK_STATIC=OFF", - ".." - ], cwd=BUILDSPACE) - thing1.wait() - - thing2 = subprocess.Popen(["make"], cwd=BUILDSPACE) - thing2.wait() - - # Move the libcmark.so artifacts in place - print("Moving files") - shutil.move(BUILDSPACE + "/src/libcmark-gfm.so." + Settings.VERSION, - Settings.LIBCMARKLOCATION + "/libcmark-gfm.so") - shutil.move(BUILDSPACE + "/extensions/libcmark-gfmextensions.so." + Settings.VERSION, - Settings.LIBCMARKLOCATION + "/libcmark-gfmextensions.so") - - -def test_configuration(): - """ Tests to ensure that the files that the plugin needs are in place. """ - if os.path.isfile(Settings.LIBCMARKLOCATION + "/libcmark-gfm.so") and \ - os.path.isfile(Settings.LIBCMARKLOCATION + "/libcmark-gfmextensions.so"): - return True - else: - return False - - -def configure(): - print("Checking out the configuration") - setup() - - -if __name__ == "__main__": - configure() diff --git a/pelican_gfm/Settings.py b/pelican_gfm/Settings.py deleted file mode 100644 index b7520ecdf..000000000 --- a/pelican_gfm/Settings.py +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/python -B - -# Eventual location of the libcmark files -LIBCMARKLOCATION = "/usr/lib/x86_64-linux-gnu" - -# This is the GitHub Cmark version we're using -VERSION = "0.28.3.gfm.12" - -# The GFM extensions that we want to use -EXTENSIONS = ( - 'autolink', - 'table', - 'strikethrough', - 'tagfilter', -) -OPTS = 0 -# Do not change these -# Do not change these -# Do not change these - -# This is the archive of GitHub's cmark-gfm files -ARCHIVES = "https://github.com/github/cmark-gfm/archive" -# The name of the local tarball that will be downloaded -LOCAL = "cmark-gfm.VERSION.orig.tar.gz" - diff --git a/pelican-gfm/Settings.py b/pelican_gfm/gfm/Settings.py similarity index 100% rename from pelican-gfm/Settings.py rename to pelican_gfm/gfm/Settings.py diff --git a/pelican_gfm/__init__.py b/pelican_gfm/gfm/__init__.py similarity index 76% rename from pelican_gfm/__init__.py rename to pelican_gfm/gfm/__init__.py index d2719baa5..e2d67d4a8 100644 --- a/pelican_gfm/__init__.py +++ b/pelican_gfm/gfm/__init__.py @@ -1,3 +1,3 @@ #!/usr/bin/environment python -B from __future__ import absolute_import -from .gfm import * +from gfm.gfm import * diff --git a/pelican_gfm/gfm.py b/pelican_gfm/gfm/gfm.py similarity index 99% rename from pelican_gfm/gfm.py rename to pelican_gfm/gfm/gfm.py index 13fb624ec..6dbadbaf4 100644 --- a/pelican_gfm/gfm.py +++ b/pelican_gfm/gfm/gfm.py @@ -30,8 +30,8 @@ import pelican.signals import pelican.readers -from . import gfmSetup -from . import Settings +from gfm import gfmSetup +from gfm import Settings _LIBDIR = Settings.LIBCMARKLOCATION _LIBCMARK = 'libcmark-gfm.so' diff --git a/pelican_gfm/gfmSetup.py b/pelican_gfm/gfm/gfmSetup.py similarity index 99% rename from pelican_gfm/gfmSetup.py rename to pelican_gfm/gfm/gfmSetup.py index 3771bf16e..e5f174d48 100644 --- a/pelican_gfm/gfmSetup.py +++ b/pelican_gfm/gfm/gfmSetup.py @@ -7,7 +7,7 @@ import shutil import tarfile import tempfile -from . import Settings +from gfm import Settings # Eventually, equivalents for # other operating systems / package diff --git a/pelican-gfm/test.py b/pelican_gfm/test.py similarity index 72% rename from pelican-gfm/test.py rename to pelican_gfm/test.py index 75ce56424..406dafc2c 100644 --- a/pelican-gfm/test.py +++ b/pelican_gfm/test.py @@ -1,10 +1,7 @@ #!/usr/bin/python -B - +from __future__ import absolute_import import unittest - -# python2 and python3 differ on how to do this it seems - -import gfm +from gfm import gfm class gfmTest(unittest.TestCase): From 4a932bee64ca2aceb3484c971951669548f0aa91 Mon Sep 17 00:00:00 2001 From: dfoulks Date: Fri, 20 Dec 2019 15:50:58 -0600 Subject: [PATCH 22/24] updated some string checking. --- pelican_gfm/gfm/gfmSetup.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/pelican_gfm/gfm/gfmSetup.py b/pelican_gfm/gfm/gfmSetup.py index e5f174d48..71f873baa 100644 --- a/pelican_gfm/gfm/gfmSetup.py +++ b/pelican_gfm/gfm/gfmSetup.py @@ -17,15 +17,12 @@ def dpkg_installed(package): """ Uses Dpkg to determine whether or not a package is installed requires: """ - t1 = subprocess.Popen(["dpkg", "-l"], stdout=subprocess.PIPE) - t2 = subprocess.Popen([ - "grep", - "-q", - package - ], stdout=subprocess.PIPE, stdin=t1.stdout) - data = t2.communicate() - ec = t2.wait() - return ec + + packages = subprocess.check_output(["dpkg", "-l"]) + if re.search(package, packages): + return 0 + else: + return 1 def dpkg_packages_installed(): @@ -68,6 +65,7 @@ def dpkg_packages_installed(): def setup(): dpkg_packages_installed() + sys.exit(0) if test_configuration() == 1: print("System appears to be configured") From f57cf1db1cf1d5f7fa5a7ee4f48d1b1f6fc4a931 Mon Sep 17 00:00:00 2001 From: dfoulks Date: Fri, 20 Dec 2019 17:40:32 -0600 Subject: [PATCH 23/24] Updated Settings and Docs and some trimmed fat --- pelican_gfm/Readme.md | 67 ++++++++++++++++++++++++++++--------- pelican_gfm/gfm/Settings.py | 14 ++++---- pelican_gfm/gfm/gfm.py | 3 +- pelican_gfm/gfm/gfmSetup.py | 29 ++++++++-------- 4 files changed, 75 insertions(+), 38 deletions(-) diff --git a/pelican_gfm/Readme.md b/pelican_gfm/Readme.md index 4d826590a..9bf0a4f64 100644 --- a/pelican_gfm/Readme.md +++ b/pelican_gfm/Readme.md @@ -3,41 +3,76 @@ Pelican GFM Pelican GFM is a reader that leverages GitHub's C-based markdown library to translate GitHub Flavored Markdown to html. -## Note +### Note -It is important to note that this currently only works on debian systems +It is important to note that this currently only works on Debian systems Requirements ============ -Pelican GFM has no requirements outside of the python standard library and pelican itself. +### No Conflicting cmark parsers + +no libcmark-gfm packages are installed as they will interfere with the processing of certain file extensions. + +### Expected .so files + +Pelican GFM requires that the following files: + * libcmark-dev.so + * libcmark-devextension.so + +exist in LIBCMARKLOCATION + +### If the .so files are not found: + +#### On debian systems + +gfmSetup.setup will check for the following package requirements: + + Installed: + * cmake + * make + * wget + + Removed: + * libcmark-gfm-dev + * libcmark-gfm-extensions-dev + * libcmark-gfm0 + * libcmark-gfm-extensions0 + + +### Building the GFM .so files + +Package requirements met, running the following: + +`python3 -B -m gfmSetup.setup()` + +will download and Make the appropriate version of GitHub's cmark library. + +Pelican GFM has no python requirements outside of the python standard library and pelican itself. + How to Use ========= -Drop the entire pelican-gfm directory into the plugin path you have defined in your pelicanconf.py -Ensure that 'pelican-gfm' is on the list of plugins. +Place the `gfm` directory into the plugin path defined in pelicanconf.py +Ensure that `gfm` is on the list of plugins. -## Settings and Configuration +### Settings and Configuration -The Settings.py file contains the configurables for pelican-gfm. +The Settings.py file contains the configurables for pelican-gfm, namely: - Location of the cmark lib files - Version of the GitHub Cmark you wish to use - - cmark extensions you wish to use. + - CMark extensions you wish to use. + - File extensions to be evaluated by gfm -## Pre-Flight Checks +### Pre-Flight Checks -Before Pelican GFM run it checks: +Before GFM runs it checks: - that the packages required to build the cmark files are present - that any packages known to conflict with this reader are not present - that the libcmark files required by the reader are present -## Configuring Pelican GFM - -If Pelican GFM finds that there is a configuration issue it will recommend that you run the following: - `python gfmSetup.py` - -## Testing +### Testing There is a unittest written for gfm. the test will register and spawn a new reader and return true if there were no issues. diff --git a/pelican_gfm/gfm/Settings.py b/pelican_gfm/gfm/Settings.py index b7520ecdf..3f1d7202a 100644 --- a/pelican_gfm/gfm/Settings.py +++ b/pelican_gfm/gfm/Settings.py @@ -1,22 +1,24 @@ #!/usr/bin/python -B -# Eventual location of the libcmark files +# Location of the libcmark files LIBCMARKLOCATION = "/usr/lib/x86_64-linux-gnu" -# This is the GitHub Cmark version we're using +# GitHub Cmark version we're using VERSION = "0.28.3.gfm.12" -# The GFM extensions that we want to use +# The CMARK extensions that we want to use EXTENSIONS = ( 'autolink', 'table', 'strikethrough', 'tagfilter', ) + +# The File extensions GFM should evaluate. +FILE_EXTENSIONS = ['md', 'markdown', 'mkd', 'mdown'] + + OPTS = 0 -# Do not change these -# Do not change these -# Do not change these # This is the archive of GitHub's cmark-gfm files ARCHIVES = "https://github.com/github/cmark-gfm/archive" diff --git a/pelican_gfm/gfm/gfm.py b/pelican_gfm/gfm/gfm.py index 6dbadbaf4..ccaa1cf95 100644 --- a/pelican_gfm/gfm/gfm.py +++ b/pelican_gfm/gfm/gfm.py @@ -112,7 +112,8 @@ class GFMReader(pelican.readers.BaseReader): # NOTE: the builtin MarkdownReader must be disabled. Otherwise, it will be # non-deterministic which Reader will be used for these files. - file_extensions = ['md', 'markdown', 'mkd', 'mdown'] + # See settings for the list of extensions. + file_extensions = Settings.FILE_EXTENSIONS # Metadata is specified as a single, colon-separated line, such as: # diff --git a/pelican_gfm/gfm/gfmSetup.py b/pelican_gfm/gfm/gfmSetup.py index 71f873baa..b3df64568 100644 --- a/pelican_gfm/gfm/gfmSetup.py +++ b/pelican_gfm/gfm/gfmSetup.py @@ -13,20 +13,28 @@ # other operating systems / package # managers could be written for this +# VARIABLES + +# Archive of GitHub's cmark-gfm files +ARCHIVES = "https://github.com/github/cmark-gfm/archive" + +# Name of the tarball that will be downloaded +LOCAL = "cmark-gfm.VERSION.orig.tar.gz" + def dpkg_installed(package): """ Uses Dpkg to determine whether or not a package is installed requires: """ - packages = subprocess.check_output(["dpkg", "-l"]) if re.search(package, packages): return 0 else: return 1 - def dpkg_packages_installed(): - """ Checking to see if the appropriate packages are installed""" + """ Check to see if the packages required to download and + make the cmark files are installed""" + installed = ["cmake", "make", "wget"] removed = [ "libcmark-gfm-dev", @@ -64,14 +72,11 @@ def dpkg_packages_installed(): def setup(): - dpkg_packages_installed() - sys.exit(0) - if test_configuration() == 1: print("System appears to be configured") else: - # Configure the environment if it's not already configured - + # Download, make, and place the libcmark files for gfm. + dpkg_packages_installed() with tempfile.TemporaryDirectory() as WORKSPACE: # Pull into the workspace subprocess.call([ @@ -120,11 +125,5 @@ def test_configuration(): else: return False - -def configure(): - print("Checking out the configuration") - setup() - - if __name__ == "__main__": - configure() + setup() From 7e8b15c151c743ed9b0d46af9ad6bab0e25b9059 Mon Sep 17 00:00:00 2001 From: dfoulks Date: Fri, 20 Dec 2019 17:43:36 -0600 Subject: [PATCH 24/24] OF: Fixed a typo --- pelican_gfm/Readme.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pelican_gfm/Readme.md b/pelican_gfm/Readme.md index 9bf0a4f64..b993f2419 100644 --- a/pelican_gfm/Readme.md +++ b/pelican_gfm/Readme.md @@ -59,11 +59,11 @@ Ensure that `gfm` is on the list of plugins. ### Settings and Configuration -The Settings.py file contains the configurables for pelican-gfm, namely: - - Location of the cmark lib files - - Version of the GitHub Cmark you wish to use - - CMark extensions you wish to use. - - File extensions to be evaluated by gfm +The Settings.py file contains the configurables for Pelican GFM, namely: + * Location of the cmark lib files + * Version of the GitHub Cmark you wish to use + * CMark extensions you wish to use. + * File extensions to be evaluated by gfm ### Pre-Flight Checks