Correcting a bug in notebook-render

While attempting to get a jupyter notebook running in gatsby this bug was encountered

mkdir ~/projects/notebook-repair
cd ~/projects/notebook-repair
yarn
yarn add react react-dom @nteract/notebook-render

// repro.js
const React = require("react")
const ReactDOMServer = require(`react-dom/server`)
const NotebookRender = require(`@nteract/notebook-render`).default
const notebook = {
  cells: [
    {
      cell_type: "markdown",
      metadata: {},
      source: ["Hello, world!"],
    },
  ],
  metadata: {
    kernelspec: {
      display_name: "Python 3",
      language: "python",
      name: "python3",
    },
    language_info: {
      codemirror_mode: {
        name: "ipython",
        version: 3,
      },
      file_extension: ".py",
      mimetype: "text/x-python",
      name: "python",
      nbconvert_exporter: "python",
      pygments_lexer: "ipython3",
      version: "3.7.3",
    },
  },
  nbformat: 4,
  nbformat_minor: 2,
}

const reactComponent = React.createElement(NotebookRender, { notebook }, null)
const html = ReactDOMServer.renderToStaticMarkup(reactComponent)

console.log(html)

node repro.js
/home/me/projects/notebook-render/node_modules/react-dom/cjs/react-dom-server.node.development.js:3555
            throw err;
            ^

Error: Renderer for type `element` not defined or is not renderable
    at astToReact (/home/me/projects/notebook-render/node_modules/react-markdown/lib/ast-to-react.js:37:11)
    at /home/me/projects/notebook-render/node_modules/react-markdown/lib/ast-to-react.js:45:14
    at Array.map (<anonymous>)
    at resolveChildren (/home/me/projects/notebook-render/node_modules/react-markdown/lib/ast-to-react.js:44:43)
    at astToReact (/home/me/projects/notebook-render/node_modules/react-markdown/lib/ast-to-react.js:41:73)
    at ReactMarkdown (/home/me/projects/notebook-render/node_modules/react-markdown/lib/react-markdown.js:61:10)
    at processChild (/home/me/projects/notebook-render/node_modules/react-dom/cjs/react-dom-server.node.development.js:3204:14)
    at resolve (/home/me/projects/notebook-render/node_modules/react-dom/cjs/react-dom-server.node.development.js:3124:5)
    at ReactDOMServerRenderer.render (/home/me/projects/notebook-render/node_modules/react-dom/cjs/react-dom-server.node.development.js:3598:22)
    at ReactDOMServerRenderer.read (/home/me/projects/notebook-render/node_modules/react-dom/cjs/react-dom-server.node.development.js:3536:29)

react-markdown appears prominently

// repro-markdown.js

const React = require("react")
const ReactDOMServer = require(`react-dom/server`)
const ReactMarkdown = require("react-markdown/with-html")

const source = `
This is markdown linking to [this post](.)
`

const reactComponent = React.createElement(ReactMarkdown, { source }, null)
const html = ReactDOMServer.renderToStaticMarkup(reactComponent)

console.log(html)

// node_modules/react-markdown/lib/ast-to-react.js:22
function astToReact(node, options) {
  var parent = arguments.length > 2 && arguments[2] !== undefined ? arguments[2] : {};
  var index = arguments.length > 3 && arguments[3] !== undefined ? arguments[3] : 0;
  var renderer = options.renderers[node.type]; // nodes generated by plugins may not have position data
  // much of the code after this point will attempt to access properties of the node.position
  // this will set the node position to the parent node's position to prevent errors

  // adding debugging...
  console.log(node)

{ type: 'root',
  children:
   [ { type: 'element',
       tagName: 'p',
       properties: {},
       children: [Array],
       position: [Object] } ],
  position:
   { start: { line: 1, column: 1, offset: 0 },
     end: { line: 1, column: 14, offset: 13 } } }
{ type: 'element',
  tagName: 'p',
  properties: {},
  children:
   [ { type: 'text', value: 'Hello, world!', position: [Object] } ],
  position:
   { start: { line: 1, column: 1, offset: 0 },
     end: { line: 1, column: 14, offset: 13 } } }

Let's see what node should look like for the same source:

// repro-markdown.js
const React = require("react")
const ReactDOMServer = require(`react-dom/server`)
const ReactMarkdown = require("react-markdown/with-html")

const source = `Hello, world!`

const props = {
  escapeHtml: false,
  source,
}

const reactComponent = React.createElement(ReactMarkdown, props, null)
const html = ReactDOMServer.renderToStaticMarkup(reactComponent)

console.log(html)

{ type: 'root',
  children:
   [ { type: 'paragraph', children: [Array], position: [Position] } ],
  position:
   { start: { line: 1, column: 1, offset: 0 },
     end: { line: 1, column: 14, offset: 13 } } }
{ type: 'paragraph',
  children:
   [ { type: 'text', value: 'Hello, world!', position: [Position] } ],
  position:
   Position {
     start: { line: 1, column: 1, offset: 0 },
     end: { line: 1, column: 14, offset: 13 },
     indent: [] } }
{ type: 'text',
  value: 'Hello, world!',
  position:
   Position {
     start: { line: 1, column: 1, offset: 0 },
     end: { line: 1, column: 14, offset: 13 },
     indent: [] } }
<p>Hello, world!</p>

So there is a gap between what parsed html looks like between the two systems.

How dows notebook-render call this code?

// node_modules/@nteract/nodebook-render/src/index.tsx:164
              case "markdown":
                const remarkPlugins = [math, remark2rehype, katex, stringify];
                const remarkRenderers = {
                  math: function blockMath(node: { value: string }) {
                    return <BlockMath>{node.value}</BlockMath>;
                  },
                  inlineMath: function inlineMath(node: { value: string }) {
                    return <InlineMath>{node.value}</InlineMath>;
                  }
                } as any;
                return (
                  <Cell key={cellId}>
                    <ContentMargin>
                      <ReactMarkdown
                        escapeHtml={false}
                        source={source}
                        plugins={remarkPlugins}
                        renderers={remarkRenderers}
                      />
                    </ContentMargin>
                  </Cell>
                );

intuition immediately suggests that it is the plugins and renderers props causing the error

Hand editing the modules:

// node_modules/@nteract/nodebook-render/lib/index.js:97
                    case "markdown":
                        // const remarkPlugins = [remark_math_1.default, remark_rehype_1.default, rehype_katex_1.default, rehype_stringify_1.default];
                        const remarkPlugins = undefined
```javascript
$ node repro.js
{ type: 'root',
  children:
   [ { type: 'paragraph', children: [Array], position: [Position] } ],
  position:
   { start: { line: 1, column: 1, offset: 0 },
     end: { line: 1, column: 14, offset: 13 } } }
{ type: 'paragraph',
  children:
   [ { type: 'text', value: 'Hello, world!', position: [Position] } ],
  position:
   Position {
     start: { line: 1, column: 1, offset: 0 },
     end: { line: 1, column: 14, offset: 13 },
     indent: [] } }
{ type: 'text',
  value: 'Hello, world!',
  position:
   Position {
     start: { line: 1, column: 1, offset: 0 },
     end: { line: 1, column: 14, offset: 13 },
     indent: [] } }
<div class="notebook-render"><div class="sc-htoDjs dTnZgJ"><div class="sc-gzVnrw kvbchc"><div class="sc-dnqmqq dSsrkY"><p>Hello, world!</p></div></div></div></div>

Ahha! by removing the plugins the code runs and the nodes look identical. Which plugin is giving us grief...

const remarkPlugins = [remark_rehype_1.default]

boom. Wut dis

An html preprocessor?

Hm. Can we reproduce this minimally?

// repro-markdown.js
const React = require("react")
const ReactDOMServer = require(`react-dom/server`)
const ReactMarkdown = require("react-markdown/with-html")
const remark2rehype = require("remark-rehype")

const source = `Hello, world!`

const props = {
  escapeHtml: false,
  source,
  plugins: [remark2rehype],
}

const reactComponent = React.createElement(ReactMarkdown, props, null)
const html = ReactDOMServer.renderToStaticMarkup(reactComponent)

console.log(html)

Yup, same error

yarn add react-markdown@4.0.0

Searching through published versions reveals 4.1.0 to 4.2.0 is where things stopped working. What changed?

github compare/v4.1.0...v4.2.0

Plugins with transforms were added. Changing this code caused the repro to succeed:

// node_modules/@nteract/notebook-render/node_modules/react-markdown/lib/react-markdown.js:56
/*
  var transformedAst = parser.runSync(rawAst);
  var ast = astPlugins.reduce(function (node, plugin) {
    return plugin(node, renderProps);
  }, transformedAst);
*/
var ast = astPlugins.reduce(function(node, plugin) {
  return plugin(node, renderProps)
}, rawAst)

At this point a PR to notebook-render pinning react-markdown to 4.1.0 would fix the issue, NOT GOOD ENOUGH

Diving deeper this is a lower level implementation of what's going on:

// repro-rehype.js
var unified = require("unified")
var markdown = require("remark-parse")
var remark2rehype = require("remark-rehype")
var html = require("rehype-stringify")

unified()
  .use(markdown)
  .use(remark2rehype)
  .use(html)
  .process("Hello World!", function(err, file) {
    console.log(String(file))
  })

node repro-rehype.js
<p>Hello World!</p>

I see that the stringify transformer is required to turn the AST back into text. At this point there was a many hour digression into exploring how the entire unified plugin ecosystem works with respect to react-markdown. Many false leads, much wailing and gnashing of teeth.

In summary react-markdown breaks down the unified plugin system into it's manual steps by executing parser.parse and parser.runSync in react-markdown.js

This means that rehype-stringify, which is in the compile phase, never runs.

react-markdown provides a renderers prop which allows us to define custom renderers

Providing an "element" renderer:

// repro-markdown.js
const React = require("react")
const ReactDOMServer = require(`react-dom/server`)
const ReactMarkdown = require("react-markdown")
const remark2rehype = require("remark-rehype")

const source = `Hello, world!`

const props = {
  source,
  plugins: [remark2rehype],
  renderers: {
    element: props => {
      console.log(props)
      return ""
    },
  },
}

const reactComponent = React.createElement(ReactMarkdown, props, null)
const html = ReactDOMServer.renderToStaticMarkup(reactComponent)

console.log(html)

Produces

{ tagName: 'p',
  properties: {},
  children:
   [ { '$$typeof': Symbol(react.element),
       type: [Function: TextRenderer],
       key: 'text-1-1-0',
       ref: null,
       props: [Object],
       _owner: null,
       _store: {} } ] }

Rewriting a new renderer for constructing html...

    "element": (props) => React.createElement(props.tagName, {}, props.children)

Produces

<p>Hello, world!</p>

Adding the renderer to notebook-render requires linking a local version of the library

mkdir -p lib
cd lib
git clone https://github.com/nteract/notebook-render
cd notebook-render
yarn
yarn build
yarn link
cd ../..
yarn link @nteract/notebook-render

At this point the repro repostory uses the notebook-render from our local copy

Before continuing development many typescript errors were fixed

And then the custom renderer was added

// lib/notebook-render/src/index.tsx:195
                const remarkRenderers = {
                  math: function blockMath(node: { value: string }) {
                    return <BlockMath>{node.value}</BlockMath>;
                  },
                  inlineMath: function inlineMath(node: { value: string }) {
                    return <InlineMath>{node.value}</InlineMath>;
                  },
                  element: function remarkElement(node: { tagName: string, children: any }) {
                    return React.createElement(node.tagName, null, node.children);
                  }
                } as any;

node repro.js
<div class="notebook-render">
  <div class="sc-iwsKbI Dticx">
    <div class="sc-dnqmqq iXBJUB">
      <div class="sc-gZMcBi gmeReZ"><p>Hello, world!</p></div>
    </div>
  </div>
</div>

In order to install this code from github as a dependency prepare was added

PR created