added fgs

fgs/Makefile

+
+.PHONY: run
+run:
+	python3 __main__.py

fgs/__main__.py

+#!/usr/bin/env python3
+
+import json
+import os
+import sys
+
+
+import reader
+import generator
+import writer
+
+CONTENT_DIR = '../content'
+OUTPUT_DIR = '../public'
+THEME_DIR = '../theme'
+
+
+def main():
+    print("Hello World")
+    config = {}
+    with open('../config.json') as f:
+        config.update(json.loads(f.read()))
+    with open('../lang.json') as f:
+        config['lang'] = json.loads(f.read())
+    with open(CONTENT_DIR + '/config.json') as f:
+        config.update(json.loads(f.read()))
+    print(config)
+
+    mdreader = reader.MarkdownReader(config)
+    pages = []
+    directory = os.path.join(CONTENT_DIR, ".")
+    parse_dir(directory, pages, mdreader)
+
+    static_dirs = []
+    for sdir in config['static_directories']:
+        if "." in sdir:
+            raise Exception("Illegal static directory name: ", sdir)
+        static_dirs.append(CONTENT_DIR + "/"+ sdir)
+    static_files = {}
+    for sdir in static_dirs:
+        read_static_dir(sdir, static_files)
+
+    read_static_dir(THEME_DIR + '/static', static_files, [config['theme']['static_dir']])
+
+
+    context = {}
+    gen = generator.Generator(config, context)
+    gen.generate_context(pages, static_files)
+
+    wrt = writer.Writer(config, context, OUTPUT_DIR, THEME_DIR)
+
+    gen.generate_output(wrt)
+
+
+
+def read_static_dir(directory, static_files, subpath = []):
+    print("static_dir: " + directory);
+    for filename in os.listdir(directory):
+        fp = os.path.join(directory, filename)
+        if os.path.isfile(fp) and not filename.startswith("."):
+            lpath = '/'.join(subpath + [filename])
+            with open(fp, "rb") as f:
+                static_files[lpath] = f.read()
+            print("read: ", fp, " as: ", lpath)
+        elif os.path.isdir(fp) and not filename.startswith("."):
+            read_static_dir(fp, static_files, subpath + [filename])
+
+
+def parse_dir(directory, pages, mdreader, subpath = []):
+    print("parse_dir: " + directory);
+    for filename in os.listdir(directory):
+        f = os.path.join(directory, filename)
+        if os.path.isfile(f) and filename.endswith(".md"):
+            pages.append(mdreader.read_and_parse_file(f, subpath))
+        elif os.path.isdir(f) and not filename.startswith("."):
+            parse_dir(f, pages, mdreader, subpath + [filename])
+
+
+
+
+
+
+    
+
+if __name__ == '__main__':
+    main()
+

fgs/generator.py

+
+class Generator:
+    def __init__(self, config, context):
+        self.config = config
+        self.context = context
+
+    def generate_context(self, pages, static_files):
+        # static_files
+        self.context['static_files'] = static_files
+
+        published_pages = []
+        for page in pages:
+            if page.status == "published":
+                published_pages.append(page)
+
+
+        # pages
+        all_pages = {}
+        for page in published_pages:
+            if page.lang not in all_pages:
+                all_pages[page.lang] = {}
+            if page.slug in all_pages[page.lang]:
+                raise Exception("duplicate language (",lang,") for slug '", slug ,"'")
+            all_pages[page.lang][page.slug] = page
+        self.context['pages'] = all_pages
+
+        # pages_modified
+        pages_modified = {}
+        for lang in self.config['lang']['supported']:
+            lang_pages = []
+            for page in published_pages:
+                if page.lang == lang:
+                    lang_pages.append(page)
+            lang_pages.sort()
+            pages_modified[lang] = lang_pages
+        self.context['pages_modified'] = pages_modified
+
+        # TODO hidden pages
+        # TODO draft pages
+        # TODO authors
+
+        # categories
+        categories = {}
+        for page in published_pages:
+            if page.category not in categories:
+                categories[page.category] = {}
+            if page.lang not in categories[page.category]:
+                categories[page.category][page.lang] = []
+            categories[page.category][page.lang].append(page)
+        self.context['categories'] = categories
+
+        # tags
+        tags = {}
+        for page in published_pages:
+            for tag in page.tags:
+                if tag not in tags:
+                    tags[tag] = {}
+                if page.lang not in tags[tag]:
+                    tags[tag][page.lang] = []
+                tags[tag][page.lang].append(page)
+        self.context['tags'] = tags
+
+
+
+    def generate_homepage(self, writer, lang, path):
+            page = self.context['pages'][lang][self.config['theme']['homepage_slug']]
+            writer.write_template(page.template, path , lang, {'page': page})
+
+
+    def generate_output(self, writer):
+        for sf, raw in self.context['static_files'].items():
+                print ("writing binary file: ", sf)
+                writer.write_file(sf, raw, mode="wb")
+
+        for lang in self.config['lang']['supported']:
+            # all pages
+            for page in self.context['pages'][lang].values():
+                writer.write_template(page.template, page.url, lang, {'page': page})
+
+            # homepages for languages
+            self.generate_homepage(writer, lang, lang + "/index.html")
+        
+            
+        # homepage
+        self.generate_homepage(writer, self.config['lang']['default'], "index.html")
+            
+        
+

fgs/page.py

+class Page:
+
+    #filename = None
+    #subpath = None
+    #raw = None
+    #metadata = None
+    #content = None
+    #toc = None
+    #title = None
+    #category = None
+    #slug = None
+    #lang = None
+    #date_created = None
+    #date_modified = None
+    #status = None
+    #authors = None
+    #tags = None
+    #template = None
+    
+
+    def __init__(self, filename, subpath, raw, metadata, content, toc, title, category, slug, lang, date_created, date_modified, authors, last_modification_author, status, config):
+        self.filename =  filename
+        self.subpath =  subpath
+        self.raw =  raw
+        self.metadata =  metadata
+        self.content =  content
+        self.toc =  toc
+        self.title =  title
+        self.category =  category
+        self.slug =  slug
+        self.lang =  lang
+        self.date_created =  Date(date_created, config)
+        self.date_modified =  Date(date_modified, config)
+        self.status =  status
+
+        #self.config =  config
+
+        # authors
+        self.authors = []
+        for local_part, domain, name in authors:
+            self.authors.append(Author(local_part, domain, name, config))
+        if last_modification_author:
+            self.last_modification_author = Author(last_modification_author[0], last_modification_author[1], last_modification_author[2], config)
+        else:
+            self.last_modification_author = None
+        #print(authors)
+        #print(last_modification_author)
+
+
+        # tags
+        self.tags = []
+        if 'tags' in metadata:
+            for t in metadata['tags']:
+                if (t != category):
+                    self.tags.append(t)
+        self.tags.append(self.category) # the category is also a default tag
+
+        # template
+        if 'template' in metadata:
+            self.template = metadata['template']
+        else:
+            self.template = config['theme']['default_template']
+
+        # url
+        self.url = self.lang + '/' + self.category + '/' + self.slug + ".html"
+
+    def __lt__(self, other):
+        return self.date_modified < other.date_modified
+
+
+class Date:
+    def __init__(self, dt, config):
+        self.dt = dt
+        #self.config = config
+        # TODO various formats
+    def isoformat(self):
+        return self.dt.isoformat()
+
+    def __lt__(self, other):
+        return self.dt.timestamp() < other.dt.timestamp()
+
+class Author:
+    def __init__(self, email_local_part, email_domain, name, config):
+        self.email_local_part = email_local_part
+        self.email_domain = email_domain
+        self.email = email_local_part + '@' + email_domain
+        self.name = name
+        #self.config = config
+        # TODO md5 hash etc.

fgs/pandoc_toc.html

+$table-of-contents$

fgs/reader.py

+import page
+
+import frontmatter
+
+from datetime import datetime
+from dateutil import parser as dtparser
+
+import subprocess
+import os
+
+class MarkdownReader:
+
+    def __init__(self, config):
+        self.config = config
+
+
+    def read_and_parse_file(self, path, subpath):
+        if not path.endswith(".md"):
+            raise Exception("can only parse markdown files: ", path)
+        elif len(subpath) > 1:
+            raise Exception("markdown file is too deep in directory structure: ", path, subpath)
+
+
+        print("parsing file: ", path, subpath)
+
+        f = open(path)
+        rawcontent = f.read()
+        f.close()
+        metadata, _ = frontmatter.parse(rawcontent)
+
+        #print(metadata)
+
+        category_name = self.get_category_name(metadata, subpath)
+
+        # content
+        content = self.run_pandoc(rawcontent, self.config['pandoc']['base'], self.config['pandoc']['extensions'], "html5")
+        #print(content)
+
+        # TOC
+        toc = self.run_pandoc(rawcontent, self.config['pandoc']['base'], self.config['pandoc']['extensions'], "html5", ["--template", "./pandoc_toc.html", "--toc", "--toc-depth", str(self.config['toc_depth'])])
+        #print((toc))
+    
+        # title
+        if 'title' not in metadata:
+            raise Exception("File is missing title in metadata: ", path, subpath)
+        title = metadata['title']
+
+        # slug and lang
+        pathlist = path.split('/')
+        filename = pathlist[-1]
+        filenamelist = filename.split('.')
+        filenamelist = filenamelist[:-1] # remove .md
+        slug = None
+        lang = None
+        if len(filenamelist) < 1:
+            raise Exception("filename is empty?", path, subpath)
+        elif len(filenamelist) == 1:
+            slug = filenamelist[0]
+        elif len(filenamelist) == 2:
+            slug = filenamelist[0]
+            lang = filenamelist[1]
+        if 'slug' in metadata:
+            slug = metadata['slug']
+        if 'lang' in metadata:
+            lang = metadata['lang']
+
+        if lang == None:
+            lang = self.config['lang']['default']
+
+        if not self.is_supported_lang(lang):
+            raise Exception("language is not supported: ", lang)
+        slug = self.secure_slug(slug)
+        #print("slug: ", slug)
+        #print("lang: ", lang)
+
+        # date_created and date_modified
+        date_modified = datetime.now()
+        date_created = datetime.now()
+        date_changes = self.run_git(path, "log", ["--follow", "--format=%ad", "--date", "iso-strict"]).splitlines()
+        #print("date_changes: ", date_changes)
+        if (len(date_changes) > 0):
+            date_modified = datetime.fromisoformat(date_changes[0])
+            date_created = datetime.fromisoformat(date_changes[-1])
+        if 'date' in metadata:
+            date_created = dtparser.parse(metadata['date'])
+        if 'modified' in metadata:
+            date_modified = dtparser.parse(metadata['modified'])
+        #print("created: ", date_created)
+        #print("last changed: ", date_modified)
+
+        # author
+        # TODO author from metadata
+        authors_raw = self.run_git(path, "log", ["--follow", "--format=%aE@%aN", "--use-mailmap"]).splitlines()
+        authors = []
+        known_author_raws = []
+        for author_raw in authors_raw:
+            if author_raw not in known_author_raws:
+                authors.append(self.extract_author(author_raw))
+                known_author_raws.append(author_raw)
+        if len(authors_raw) > 0:
+            last_modification_author = self.extract_author(authors_raw[0])
+        else:
+            last_modification_author = None
+
+
+
+        # status
+        status = self.config['default_status']
+        if 'status' in metadata:
+            status = metadata['status']
+        valid_status = ["published", "draft", "hidden"]
+        if status not in valid_status:
+            raise Exception("invalid status '", status, "' must be one of ", valid_status)
+
+        # TODO summary
+
+        p = page.Page(
+                filename,
+                subpath,
+                rawcontent,
+                metadata,
+                content,
+                toc,
+                title,
+                category_name,
+                slug,
+                lang,
+                date_created,
+                date_modified,
+                authors,
+                last_modification_author,
+                status,
+                self.config)
+        return p
+
+    def extract_author(self, raw):
+        author_split = raw.split('@')
+        author_local_part = author_split[0]
+        author_domain = author_split[1]
+        author_name = '@'.join(author_split[2:])
+        return (author_local_part, author_domain, author_name)
+
+    def is_supported_lang(self, lang):
+        if not isinstance(lang, str):
+            return False
+        return (lang in self.config['lang']['supported'])
+
+
+    def secure_slug(self, slug):
+        if not isinstance(slug, str):
+            raise Exception("slug is not a string: '", slug, "'")
+        slug = slug.lower()
+        whitelist = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
+        res = ""
+        for c in slug:
+            if c in whitelist:
+                #print ("c: '", c,"'")
+                res += c
+        #print("res: '", res, "'")
+        if len(res) == 0:
+            raise Exception("slug is empty")
+        return res
+
+
+    def get_category_name(self, metadata, subpath):
+        if 'category' in metadata:
+            return metadata['category']
+        elif len(subpath) == 1:
+            return subpath[0]
+        else:
+            return 'misc'
+
+    def run_git(self, path, subcmd, extra_args):
+        real_path = os.path.realpath(path)
+        filename = os.path.basename(real_path)
+        dir_path = os.path.dirname(real_path)
+        git_bin = "git"
+        args = [git_bin, subcmd] + extra_args + ["--", filename]
+        p = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, cwd=dir_path)
+        out, _ = p.communicate("".encode('utf-8', errors='strict'))
+        out_str = out.decode('utf-8')
+        return out_str
+
+
+    def run_pandoc(self, source, base="markdown", extensions=[], to="json", extra_args=[]):
+        ext_str = ""
+        if isinstance(extensions, list):
+            for ext in extensions:
+                if ext.startswith('#'):
+                    continue
+                if ext.startswith('+') or ext.startswith('-'):
+                    ext_str = ext_str + ext
+                elif len(ext) > 0:
+                    ext_str = ext_str + '+' + ext
+        elif isinstance(extensions, dict):
+            for ext_key in extensions:
+                # TODO catch 'illegal' ext_keys (containing spaces for example)
+                ext = extensions[ext_key]
+                if "ignore" in ext and ext["ignore"]:
+                    continue
+                flag='+'
+                if "enabled" in ext and not ext["enabled"]:
+                    flag='-'
+                ext_str = ext_str + flag + ext_key
+    
+        #print(ext_str)
+        pandoc_bin = "pandoc"
+        args = [pandoc_bin, "-f", base + ext_str, "-t", to] + extra_args
+        p = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+        out, _ = p.communicate(source.encode('utf-8', errors='strict'))
+        out_str = out.decode('utf-8')
+        #print("----------------------")
+        #print(out_str)
+        #print("----------------------")
+        #json_dict = json.loads(out.decode('utf-8'))
+        return out_str
+
+

fgs/writer.py

+from jinja2 import Environment, FileSystemLoader
+
+import os
+
+class Writer:
+
+    def __init__(self, config, context, output_dir, theme_dir):
+        self.config = config
+        self.context = context
+        self.output_dir = output_dir
+        self.theme_dir = theme_dir
+
+        self.env = Environment(
+            loader=FileSystemLoader(theme_dir + "/templates"),
+            autoescape=False
+        )
+
+        print("templates: ", self.env.list_templates())
+
+    def write_template(self, template, path, lang, extra_context):
+        tmpl = self.env.get_template(template)
+
+        pathsplit = path.split('/')
+        #pathsplit.remove('.') # TODO remove all not just one
+        #pathsplit.remove('..') # TODO remove all not just one
+
+        siteurl = "."
+        if self.config['relative_urls']:
+            count = len(pathsplit) - 1
+            for i in range(count):
+                siteurl = siteurl + '/..'
+        else:
+            siteurl = config['siteurl']
+
+        # render template
+        context = {}
+        context.update(self.context)
+        context["config"] = self.config
+        context["theme"] = self.config['theme']
+        context["template"] = template
+        context["t"] = self.config["lang"] # translate
+        context["l"] = lang # current language
+        context["path"] = path
+        context["siteurl"] = siteurl
+        context.update(extra_context)
+        out = tmpl.render(context)
+
+        # write file
+        self.write_file(path, out)
+
+    def write_file(self, path, out, mode="w"):
+        # write to file
+        fullpath = self.output_dir + '/' + path
+        directory = os.path.dirname(fullpath)
+        print("fullpath: ", fullpath)
+        print("dir: ", directory)
+        os.makedirs(directory, exist_ok=True)
+        with open(fullpath, mode) as f:
+            f.write(out)
+
+