bandolier-website/lib/docgen.js

/*
  Utilities mostly for generating the documentation and static pages in
  most sites.  It's targetted at the kind of content I make, which is
  programming docs, educational lessons, and other Markdown documents
  with code in them.

  The main feature of this module is the `class Ork` which load code from
  external code files for inclusion in the output HTML.
 */
import { Remarkable } from "remarkable";
import assert from "assert";
// Node is absolutely stupid. The remarkable project has an esm setting which
// specifies the dist/esm directory, but it completely ignores it and complains.
import { linkify } from "remarkable/dist/cjs/linkify.js";
import Prism from "../static/js/prism.cjs";
import _ from "lodash";
import slugify from "slugify";
import fs from "fs";
import PATH from "path";
import child_process from "child_process";

_.templateSettings.interpolate = /{{([\s\S]+?)}}/g;

/*
  This is a little class I use to read in an external code
  file, then wrap that code with the markdown code blocks.

  Look in `render_with_code` to see how it is implemented:

  1. Creates a `lodash` template that has a variable named `ork`.
  2. When `render_with_code` runs it then first runs your `.md` file through this template.
  3. That template can then call functions on `Ork`, as in: `{{ ork.js() }}`.
  4. `Ork` then returns whatever Markdown code format is needed for the output.
 */
export class Ork {

  /*
    Given a `base` directory and metadata object it will load code as needed.  Keep
    in mind this is tied to my setup where I have a private directory in a separate
    repository that contains the code for the course.  This let's you do this:

    ```javascript
    let ork = new Ork("javascript_level_1", { code: "exercise_1"});
    let md_out = ork.js("code.js");
    ```

    This will load the file in `javascript_level_1/exercise_1/code.js`, wrap it with
    Markdown code blocks.

    + `base string` -- The base directory where all of the code examples are.
    + `metadata Object` -- When a `.md` file is loaded it has metadata at the top, and one is `code` which says where code for that `.md` file should be loaded.

    ___BUG___: `base` needs to go away, but for now it's where the root of the db is.
   */
  constructor(base, metadata) {
    if(metadata.code) {
      this.code_path = `${base}${metadata.code}`;
    } else {
      this.code_path = false;
    }
  }

  /*
    Loads a JavaScript file and returns it wrapped in a Markdown code block tagged
    with `javascript`.

    + `file string` -- The file to load relative to `base/{metadata.code}` from the constructor.
   */
  js(file='code.js') {
    assert(this.code_path, "You are using ork but no code path set in metadata.");
    return `\`\`\`javascript\n${fs.readFileSync(`${this.code_path}/${file}`).toString().trim()}\n\`\`\``
  }

  /*
    Actually __executes__ the code file using the local `node` interpreter, and returns
    the output, instead of the code.  This lets you automatically generate the results
    of code for your documents from one source.  Otherwise you have to do this manually
    then include it.

    + `file string` -- The file to _run_ relative to `base/{metadata.code}` from the constructor.
    + `args string` -- Arguments to the `node` command line.
   */
  node(file='code.js', args='') {
    assert(this.code_path, "You are using ork but no code path set.");
    let cmd = `node "${file}" ${args}`;
    let result = child_process.execSync(cmd, {cwd: this.code_path});
    return `\`\`\`shell-session\n$ ${cmd}\n${result.toString().trim()}\n\`\`\``
  }
}

/*
  To make CSS line numbers work you have to do this janky line counting
  thing.  Definitely not an efficient way to do it but it is a simple way.
 */
export const count_lines = (str) => {
  let match = str.match(/\n(?!$)/g);
  return match ? match.length + 1 : 1;
}

/*
  Using `count_lines` this will generate a bunch of `<span></span>` tags
  to match the count.  This is the easiest way to create a strip of line
  numbers to the left of a code block.
 */
export const line_number_spans = (str) => {
  let count = count_lines(str);
  let spans = new Array(count + 1).join('<span></span>');
  return `<span aria-hidden="true" class="line-numbers-rows">${spans}</span>`;
}

/*
  Runs Prism on the string, defaulting to the "markup" language which will
  encode most code correctly even if you don't specify a language after the
  Markdown code block header.

  + `str string` -- Code to parse with Prism.
  + `lang string` -- Defaults to "markup" but can be anything Prism supports.
 */
export const run_prism = (str, lang="markup") => {
  assert(lang !== undefined, "Language can't be undefined.");

  // do the check again since the language might not exist
  if(Prism.languages[lang]) {
    return Prism.highlight(str, Prism.languages[lang], lang);
  } else {
    console.error("Unknown language", lang);
    return Prism.highlight(str, Prism.languages["markup"], "markup");
  }
}

/*
  Does a complete highlight of a block of code for the specified
  `lang`.  It does the combination of `run_prism` + `line_number_spans`
  so that the output is immediately useable as code output with line numbers.

  + `str string` -- Code to parse with Prism.
  + `lang string` -- Can be anything Prism supports, like "javascript" or "shell".
 */
export const highlight = (str, lang) => {
  return run_prism(str, lang) + line_number_spans(str);
}

/*
  A separate renderer just for the titles that doesn't need anything else.
  The main thing this does is turn of paragraphs by assigning:

  ```javascript
  rem.renderer.rules.paragraph_open = pass;
  rem.renderer.rules.paragraph_close = pass;
  ```

  The `pass` function is an empty function that removes the paragraphs from the
  Remarkable output.  That's all you need for title rendering.
 */
export const title_render = new Remarkable('full').use(rem => {
  let pass = (tokens, idx) => '';
  rem.renderer.rules.paragraph_open = pass;
  rem.renderer.rules.paragraph_close = pass;
});

/*
  My Markdown files `.md` files with a format of:

  ```markup
  {
    metadata
  }
  ------
  Markdown
  ```

  This function splits the input at the 6 `-` characters, runs
  `JSON.parse` on the top part, then returns that metadata and body.

  + `raw_md string` -- The markdown to split.
  + ___return `Array[Object, string]` -- An Array with the metadata object and the string body.
 */
export const split = (raw_md) => {
  let [metadata, ...body] = raw_md.split('------');
  metadata = JSON.parse(metadata);
  body = body.join('------');
  return [metadata, body];
}

const null_cb = (metadata, body) => body;

/*
  Constructs the Remarkable render function that will do:

  1. `linkify` any hostnames it finds.
  2. Update the `toc` Array with new headings for a TOC output.
  3. Fix up links, but I'm not sure why I'm doing this.

  ___BUG___: Why am I changing the link output.
 */
export const create_renderer = (toc) => {
  const renderer = new Remarkable('full', {
    html: true,
    highlight
  });

  renderer.use(linkify);

  renderer.use(rem => {
    rem.renderer.rules.heading_open = (tokens, idx) => {
      let level = tokens[idx].hLevel;
      let content = tokens[idx + 1].content;
      let slug = slugify(content, {lower: true, strict: true});
      if(toc) toc.push({level, content, slug});
      return `<h${level} id="${slug}">`;
    }

    rem.renderer.rules.heading_close = (tokens, idx) => {
      return `</h${tokens[idx].hLevel}>\n`;
    }
  });

  renderer.use(rem => {
    const orig_open = rem.renderer.rules.link_open;
    const orig_close = rem.renderer.rules.link_close;

    rem.renderer.rules.link_open = (tokens, idx, options) => {
      return orig_open(tokens, idx, options);
    }

    rem.renderer.rules.link_close = (tokens, idx, options) => {
      return orig_close(tokens, idx, options);
    }
  });

  return renderer;
}

/*
  Performs a render of the `raw_md` that will:

  1. Split the `raw_md` with `split`.
  2. Render with a Remarkable renderer from `create_renderer`.
  3. Go through each element of the TOC and run `title_render.render` on them.


  + `raw_md string` -- The Markdown string to parse.
  + `cb function(metadata, body)` -- An optional callback that gets the `metadata, body` from `split`.
  + ___return___ `{toc: Array, content: string, metadata: Object}`
 */
export const render = (raw_md, cb=null_cb) => {
  let toc = [];
  let [metadata, body] = split(raw_md);

  const renderer = create_renderer(toc);
  let content =  renderer.render(cb(metadata, body));

  // now we can use the TOC to figure out a title
  metadata.title = toc[0].content;

  try {
    // finally, run the renderer on all of the TOC
    toc.forEach(t => t.html = title_render.render(t.content));
  } catch(error) {
    console.error(error);
  }

  metadata.original_md = body;

  return {toc, content, metadata};
}

/*
  The tip of the iceberg, this does the actual render of a markdown file
  with source code in it.  It combines all of the functions so far into
  one cohesive rendering function:

  1. It uses `Ork` to load code into the markdown.
  2. It uses `render` to create a Remarkable renderer.
  3. Finally uses `slugify` to generate a slug for the metadata output.

  + `source_dir string` -- Source code directory for the `Ork(source_dir, metadata)` call.
  + `md_file string` -- Path to the markdown file to load.
  + ___return___ `{content: string, toc: Array, metadata: Object}`
 */
export const render_with_code = (source_dir, md_file) => {
  // get the file without path or extension
  const tail_no_ext = PATH.basename(md_file, PATH.extname(md_file));

  // this is a closure that is passed to render to run Ork for the code wrapping
  const load_code = (metadata, body) => {
    return _.template(body)({ fs, ork: new Ork(source_dir, metadata)});
  }

  // load the .md file contents
  let raw_md = fs.readFileSync(md_file);

  let { toc, content, metadata } = render(raw_md.toString(), load_code);

  metadata.slug = slugify(tail_no_ext, {lower: true, strict: true});

  return {content, toc, metadata};
}

export default {
  render,
  split,
  render_with_code,
  create_renderer
}