Progress towards making single "import everything from GitHub, including every tag" script.

docgen/package.json

@@ -13,11 +13,14 @@
     "flow-parser": "^0.32.0",
     "fs.extra": "1.3.2",
     "glob": "6.0.4",
+    "glob-promise": "^3.2.0",
     "jsdoc-api": "^1.1.0",
     "jsdom": "^11.1.0",
     "jstransform": "11.0.3",
     "memory-cache": "^0.1.6",
+    "minimist": "^1.2.0",
     "mkdirp": "^0.5.1",
+    "node-fetch": "^1.7.3",
     "optimist": "0.6.0",
     "prop-types": "^15.5.8",
     "react": "~0.13.0",
@@ -25,7 +28,8 @@
     "react-page-middleware": "0.4.1",
     "remove-markdown": "^0.1.0",
     "request": "^2.69.0",
-    "semver-compare": "^1.0.0"
+    "semver-compare": "^1.0.0",
+    "shelljs": "^0.7.8"
   },
   "devDependencies": {
     "front-matter": "^2.1.2",

docgen/server/convert.js

@@ -12,11 +12,11 @@
 var fs = require('fs')
 var glob = require('glob');
 var mkdirp = require('mkdirp');
-var optimist = require('optimist');
 var path = require('path');
 var removeMd = require('remove-markdown');
 var extractDocs = require('./extractDocs');
 var cache = require('memory-cache');
+var optimist = require('optimist');
 var argv = optimist.argv;
 
 function splitHeader(content) {

docgen/server/extractDocs.js

@@ -313,8 +313,7 @@ function parseAPIJsDocFormat(filepath, fileContent) {
     'filename': fileName,
     'sourceFileName': fileName,
     'plugins': [
-      'transform-flow-strip-types',
-      'babel-plugin-syntax-trailing-function-commas',
+      'transform-flow-strip-types'
     ]
   };
   // Babel transform

docgen/server/generate.js

@@ -8,59 +8,201 @@
  */
 'use strict';
 
-var Promise = require('bluebird');
-var request = require('request');
-var glob = require('glob');
-var fs = require('fs.extra');
-var mkdirp = require('mkdirp');
-var server = require('./server.js');
-var Feed = require('feed');
+const fetch = require("node-fetch");
+const fs = require('fs.extra');
+const glob = require('glob-promise');
+const jsdom = require("jsdom");
+const mkdirp = require('mkdirp');
+const Promise = require('bluebird');
 
-require('./convert.js')({extractDocs: true});
-server.noconvert = true;
+const convert = require('./convert.js');
+const slugify = require("../core/slugify");
 
-var queue = Promise.resolve();
+const FORMAT_HTML = 'html';
+const FORMAT_MARKDOWN = 'markdown';
 
-// Generate HTML for each non-source code JS file
-glob('src/**/*.*', function(er, files) {
-  files.forEach(function(file) {
-    var targetFile = file.replace(/^src/, 'build');
+const { JSDOM } = jsdom;
 
-    if (file.match(/\.js$/) && !file.match(/src\/react-native\/js/)) {
-      targetFile = targetFile.replace(/\.js$/, '.html');
-      queue = queue.then(function() {
-        return new Promise(function(resolve, reject) {
-          request('http://localhost:8079/' + targetFile.replace(/^build\//, ''), function(error, response, body) {
-            if (error) {
-              reject(error);
-              return;
-            }
-            if (response.statusCode != 200) {
-              reject(new Error('Status ' + response.statusCode + ':\n' + body));
-              return;
-            }
-            mkdirp.sync(targetFile.replace(new RegExp('/[^/]*$'), ''));
-            fs.writeFileSync(targetFile, body);
-            resolve();
-          });
-        });
-      });
-    } else {
-      queue = queue.then(function() {
-        return new Promise(function(resolve, reject) {
-          mkdirp.sync(targetFile.replace(new RegExp('/[^/]*$'), ''));
-          fs.copy(file, targetFile, resolve);
-        });
-      });
-    }
-  });
+const argv = require('minimist')(process.argv.slice(2), {
+  alias: {
+    'f': 'format'
+  },
+  default: {
+    'autodocs': true,
+    'format': FORMAT_HTML
+  }
+});
 
+/**
+ * Generates documentation for a given file.
+ * 
+ * @param {*} file 
+ * @param {*} options 
+ */
+function generateAutodocForFile(file, options) {
+  if (file.match(/src\/react-native\/js/)) {
+    // Ensure we're only processing extracted docs
+    console.log(`Skipping ${file}`);
+    return;
+  }
+
+  console.log(`Processing ${file}`);
+  const targetFile = file
+    .replace(/^src/, 'build')
+    .replace(/\.js$/, '.html');
+
+  const targetFileServerPath = targetFile.replace(/^build\//, '');
+
+  const url = 'http://localhost:8079/' + targetFileServerPath;
+  return fetch(url)
+    .then(response => {
+      if(response.ok) {
+        return response.text();
+      }
+      throw new Error('Network response was not ok.');
+    })
+    .then(body => {
+      mkdirp.sync(targetFile.replace(new RegExp('/[^/]*$'), ''));
+      fs.writeFileSync(targetFile, body);
+      return JSDOM.fromFile(targetFile);
+    })
+    .then(dom => {
+      // TODO: Generate Markdown file as needed.
+      // Also, do we need to generate metadata now as well, regardless of format? Should we go straight to Markdown anyway?
+      // Figure out where these files should be written.
+      const metadata = generateMarkdownFromDOM(dom);
+      console.log(metadata);
+      // if (categories[metadata.category]) {
+      //   categories[metadata.category].push(metadata.id);
+      // } else {
+      //   categories[metadata.category] = [metadata.id];
+      // }
+    })
+    .catch(error => {
+      console.log(error);
+      reject(error);
+    });
+}
+
+/**
+ * Generates HTML or Markdown documentation. Uses the convert script to extract docs from source files.
+ */
+function generateAutodocs(options) {
+  if (options === undefined) {
+    options = { format: FORMAT_HTML };
+  }
+
+  // Start up a server. Don't forget to close the connection when done.
+  const server = require('./server.js');
+  convert({extractDocs: true});
+  server.noconvert = true;
+  
+  let queue = Promise.resolve();
+  
   queue = queue.then(function() {
-    console.log('Generated HTML files from JS');
+    return glob('src/**/*.js');
+  }).then(function(files) {
+    let p = Promise.resolve();
+    files.forEach(function(file) {
+      p = p.then(function() {
+        return generateAutodocForFile(file);
+      });
+    });
+  
+    return p;
+  }).then(function() {
+    console.log(`Generated ${options.format} files from JavaScript sources.`);
   }).finally(function() {
     server.close();
   }).catch(function(e) {
     console.error(e);
     process.exit(1);
   });
-});
+}
+
+// DOM FORMATTING FUNCS
+
+function bodyContentFromDOM(dom) {
+  const el = dom.window.document.querySelector("#componentContent");
+  if (el) {
+    return el.innerHTML;
+  } else {
+    return null;
+  }
+}
+
+function componentNameFromDOM(dom) {
+  const el = dom.window.document.querySelector("title");
+  if (el) {
+    return el.innerHTML;
+  } else {
+    return "Component";
+  }
+}
+
+function componentCategoryFromDOM(dom) {
+  const el = dom.window.document.querySelector('meta[property="rn:category"]');
+  if (el) {
+    return el.content;
+  } else {
+    return "Components";
+  }
+}
+
+/**
+ * Generates a markdown formatted file, including frontmatter.
+ * 
+ * @param {*} dom 
+ */
+function generateMarkdownFromDOM(dom) {
+  const body = bodyContentFromDOM(dom);
+  if (!body) {
+    return;
+  }
+  const componentName = componentNameFromDOM(dom);
+  const slug = slugify(componentName);
+  let category = "Components";
+  const componentCategory = componentCategoryFromDOM(dom);
+  if (componentCategory) {
+    category = componentCategory;
+  }
+  const metadata = { id: slug, category: category };
+  const res = [
+    "---",
+    "id: " + slug,
+    "title: " + componentName,
+    "category: " + category,
+    "permalink: docs/" + slug + ".html",
+    "---",
+    body
+  ]
+    .filter(function(line) {
+      return line;
+    })
+    .join("\n");
+
+  // ORIGINAL
+  // const targetFile = "../docs/autogen_" + componentName + ".md";
+  
+  mkdirp.sync('build/autodocs');
+  const targetFile = "build/autodocs/autogen_" + componentName + ".md";
+    // ORIGINAL
+  // mkdirp.sync(targetFile.replace(new RegExp("/[^/]*$"), ""));
+  console.log("Writing " + targetFile);
+  fs.writeFileSync(targetFile, res);
+  return metadata;
+}
+
+function generateMetatadaFile(categories) {
+  const categoriesMetadataFile = "build/sidebar-metadata.json";
+  fs.writeFileSync(categoriesMetadataFile, JSON.stringify(categories));
+}
+// END DOM
+
+if (argv.autodocs) {
+  let format = argv.format;
+  console.log(`Generating ${format} files from JavaScript sources.`);
+  generateAutodocs({ format });
+}
+
+module.exports = generateAutodocs;

docgen/server/import-existing-docs.js

@@ -10,6 +10,19 @@
  */
 "use strict";
 
+/** 
+ * This is meant to be a one-time run script that checks out every version of the docs off GitHub. This includes both the Markdown-formatted guides, as well as the autodocs generated from JavaScript code.
+ * Given that the autodocs are generated from JavaScript code and stored as HTML in source control, we'll need to go back and regenerate autodocs for every single version.
+ * Once we have Markdown formatted docs, we'll need to go through these and generate all the necessary sidebar files.
+ * We'll be working with a few directories:
+ * 
+ *   - /docs - This is where the latest version of the docs will reside. 
+ *   - /website/versioned_docs - Here we will have version-XXX folders, one for each React Native version. It shall include both "guides" (regular docs already stored as a Markdown file), as well as "autodocs" (Markdown docs generated from JavaScript comments).
+ *   - /website/versioned_sidebars - We'll also have version-XXX folders here, one for each React Native version. These sidebar files will be generated based on the Markdown docs present in the corresponding versioned_docs sub-folder.
+ *   - /docgen/build - Here we will store any intermediary build files that shall not be stored in source control. These can be regenerated from source control, and include such things as original files checked out from git tags, as well as intermediary metadata files used to generate the final sidebar files.
+ * 
+ */
+
 const fm = require("front-matter");
 const fs = require("fs");
 const glob = require("glob");
@@ -20,38 +33,47 @@ const GIT_USER = process.env.GIT_USER;
 const GITHUB_USERNAME = process.env.GITHUB_USERNAME;
 const GITHUB_REPONAME = process.env.GITHUB_REPONAME;
 const remoteBranch = `https://${GIT_USER}@github.com/${GITHUB_USERNAME}/${GITHUB_REPONAME}.git`;
-const targetDir = `${GITHUB_REPONAME}-docs`;
+const localCheckoutDir = `${GITHUB_REPONAME}-docs`;
 const DOCS_DIR = `../docs`;
+const buildDir = `build/`;
 
-if (!GIT_USER) {
-  shell.echo("GIT_USER undefined.");
-  shell.exit(1);
-}
-if (!GITHUB_USERNAME) {
-  shell.echo("GITHUB_USERNAME undefined.");
-  shell.exit(1);
-}
-if (!GITHUB_REPONAME) {
-  shell.echo("GITHUB_REPONAME undefined.");
-  shell.exit(1);
-}
-
-if (!shell.which("git")) {
-  shell.echo("Sorry, this script requires git");
-  shell.exit(1);
+function runChecks() {
+  if (!shell.which("git")) {
+    shell.echo("Sorry, this script requires git");
+    shell.exit(1);
+  }
+  
+  if (!GIT_USER) {
+    shell.echo("GIT_USER undefined.");
+    shell.exit(1);
+  }
+  if (!GITHUB_USERNAME) {
+    shell.echo("GITHUB_USERNAME undefined.");
+    shell.exit(1);
+  }
+  if (!GITHUB_REPONAME) {
+    shell.echo("GITHUB_REPONAME undefined.");
+    shell.exit(1);
+  }
 }
 
+/**
+ * Remove any existing build directories and start with a clean slate.
+ */
 function prepareFilesystem() {
   shell.cd(process.cwd());
-  shell.exec(`rm -rf build/`);
-  shell.mkdir(`build`);
-  shell.cd(`build`);
-  shell.exec(`rm -rf ${targetDir}`);
-  shell.mkdir(targetDir);
+  shell.rm('-rf', buildDir);
+  shell.mkdir('-p', buildDir + localCheckoutDir);
 }
 
+/**
+ * Check out each version tag, then run node server/generate.js to get HTML files built.
+ * 
+ * Generate requires a server to be started up. Can we do this from one location?
+ */
+
 function checkOutDocs() {
-  shell.cd(targetDir);
+  shell.cd(localCheckoutDir);
 
   shell.exec(`git init`).code !== 0;
 
@@ -73,7 +95,31 @@ function checkOutDocs() {
     shell.exit(1);
   }
 
-  shell.echo(`Checked out ${targetDir}`);
+  shell.echo(`Checked out ${localCheckoutDir}`);
+  shell.cd(`../..`);
+
+  processDocs(`build/${localCheckoutDir}/docs`);
+}
+
+function checkOutVersionedDocs() {
+  shell.cd(process.cwd() + buildDir + localCheckoutDir);
+  shell.exec(`git fetch`);
+
+  const tags = shell.exec(`git tag --sort=version:refname -l 'v0.??.?' 'v0.?.?'`).toString().split('\n');
+  console.log(tags);
+  tags.forEach(function(tag) {
+    if (shell.exec(`git checkout ${tag}`).code !== 0) {
+      shell.echo("Error: git checkout failed");
+      shell.exit(1);
+    }
+    shell.echo(`Checked out ${tag}`);
+    const version = tag.substring(1);
+    const versionDir = `../../website/versioned_docs/${version}`;
+    shell.mkdir(versionDir);
+    shell.cp(`docs/*`, `${versionDir}/.`)
+    processDocs(versionDir, version);
+  })
+
   shell.cd(`../..`);
 }
 
@@ -192,11 +238,12 @@ function generateMarkdownFromMetadata(sidebarMetadata) {
   });
 }
 
-function processDocs() {
+function processDocs(workingDir, version) {
   // Generate sidebars.json
-  glob(`build/${targetDir}/docs/*.md`, function(er, files) {
+  glob(`${workingDir}/*.md`, function(er, files) {
     const sidebarsMetadata = generateDocsMetadata(files);
     const sidebars = generateSidebarsFromMetadata(sidebarsMetadata);
+    const sidebarFile = `../website/` + version ? `versioned_sidebars/${version}/` : '' + 'sidebars.json'
     fs.writeFileSync(
       `../website/sidebars.json`,
       JSON.stringify({
@@ -209,6 +256,7 @@ function processDocs() {
   });
 }
 
+runChecks();
 prepareFilesystem();
 checkOutDocs();
-processDocs();
+// checkOutVersionedDocs();

docgen/src/react-native/docs/alertios.js

@@ -132,7 +132,7 @@ var content = `\{
           500,
           724
         ],
-        "filename": "7u4rbu5xfaldzaqcktauzw.js",
+        "filename": "3e7nokk59jgk3kebjlabvb.js",
         "lineno": 27,
         "path": "/var/folders/82/w8hcc_tj7yq80482srkc1r2x0bj21j/T",
         "code": \{}
@@ -181,7 +181,7 @@ var content = `\{
           1301,
           6425
         ],
-        "filename": "7u4rbu5xfaldzaqcktauzw.js",
+        "filename": "3e7nokk59jgk3kebjlabvb.js",
         "lineno": 64,
         "path": "/var/folders/82/w8hcc_tj7yq80482srkc1r2x0bj21j/T",
         "code": \{
@@ -205,7 +205,7 @@ var content = `\{
           1320,
           2364
         ],
-        "filename": "7u4rbu5xfaldzaqcktauzw.js",
+        "filename": "3e7nokk59jgk3kebjlabvb.js",
         "lineno": 65,
         "path": "/var/folders/82/w8hcc_tj7yq80482srkc1r2x0bj21j/T",
         "code": \{}
@@ -280,7 +280,7 @@ var content = `\{
           2746,
           4559
         ],
-        "filename": "7u4rbu5xfaldzaqcktauzw.js",
+        "filename": "3e7nokk59jgk3kebjlabvb.js",
         "lineno": 101,
         "path": "/var/folders/82/w8hcc_tj7yq80482srkc1r2x0bj21j/T",
         "code": \{}

docgen/src/react-native/docs/asyncstorage.js

@@ -11,7 +11,7 @@ var content = `\{
           1920,
           14385
         ],
-        "filename": "pz04z98pfkludq58axh9bc.js",
+        "filename": "3nr1wx3lpaaxbu4d7zw7yf.js",
         "lineno": 62,
         "path": "/var/folders/82/w8hcc_tj7yq80482srkc1r2x0bj21j/T",
         "code": \{
@@ -35,7 +35,7 @@ var content = `\{
           2285,
           2786
         ],
-        "filename": "pz04z98pfkludq58axh9bc.js",
+        "filename": "3nr1wx3lpaaxbu4d7zw7yf.js",
         "lineno": 75,
         "path": "/var/folders/82/w8hcc_tj7yq80482srkc1r2x0bj21j/T",
         "code": \{
@@ -92,7 +92,7 @@ var content = `\{
           3094,
           3459
         ],
-        "filename": "pz04z98pfkludq58axh9bc.js",
+        "filename": "3nr1wx3lpaaxbu4d7zw7yf.js",
         "lineno": 99,
         "path": "/var/folders/82/w8hcc_tj7yq80482srkc1r2x0bj21j/T",
         "code": \{
@@ -158,7 +158,7 @@ var content = `\{
           3725,
           4080
         ],
-        "filename": "pz04z98pfkludq58axh9bc.js",
+        "filename": "3nr1wx3lpaaxbu4d7zw7yf.js",
         "lineno": 120,
         "path": "/var/folders/82/w8hcc_tj7yq80482srkc1r2x0bj21j/T",
         "code": \{
@@ -215,7 +215,7 @@ var content = `\{
           5240,
           5609
         ],
-        "filename": "pz04z98pfkludq58axh9bc.js",
+        "filename": "3nr1wx3lpaaxbu4d7zw7yf.js",
         "lineno": 169,
         "path": "/var/folders/82/w8hcc_tj7yq80482srkc1r2x0bj21j/T",
         "code": \{
@@ -284,7 +284,7 @@ var content = `\{
           5934,
           6263
         ],
-        "filename": "pz04z98pfkludq58axh9bc.js",
+        "filename": "3nr1wx3lpaaxbu4d7zw7yf.js",
         "lineno": 190,
         "path": "/var/folders/82/w8hcc_tj7yq80482srkc1r2x0bj21j/T",
         "code": \{
@@ -332,7 +332,7 @@ var content = `\{
           6547,
           6875
         ],
-        "filename": "pz04z98pfkludq58axh9bc.js",
+        "filename": "3nr1wx3lpaaxbu4d7zw7yf.js",
         "lineno": 211,
         "path": "/var/folders/82/w8hcc_tj7yq80482srkc1r2x0bj21j/T",
         "code": \{
@@ -380,7 +380,7 @@ var content = `\{
           7392,
           8575
         ],
-        "filename": "pz04z98pfkludq58axh9bc.js",
+        "filename": "3nr1wx3lpaaxbu4d7zw7yf.js",
         "lineno": 235,
         "path": "/var/folders/82/w8hcc_tj7yq80482srkc1r2x0bj21j/T",
         "code": \{
@@ -429,7 +429,7 @@ var content = `\{
           9513,
           10248
         ],
-        "filename": "pz04z98pfkludq58axh9bc.js",
+        "filename": "3nr1wx3lpaaxbu4d7zw7yf.js",
         "lineno": 292,
         "path": "/var/folders/82/w8hcc_tj7yq80482srkc1r2x0bj21j/T",
         "code": \{
@@ -492,7 +492,7 @@ var content = `\{
           10805,
           11163
         ],
-        "filename": "pz04z98pfkludq58axh9bc.js",
+        "filename": "3nr1wx3lpaaxbu4d7zw7yf.js",
         "lineno": 341,
         "path": "/var/folders/82/w8hcc_tj7yq80482srkc1r2x0bj21j/T",
         "code": \{
@@ -549,7 +549,7 @@ var content = `\{
           11710,
           12056
         ],
-        "filename": "pz04z98pfkludq58axh9bc.js",
+        "filename": "3nr1wx3lpaaxbu4d7zw7yf.js",
         "lineno": 371,
         "path": "/var/folders/82/w8hcc_tj7yq80482srkc1r2x0bj21j/T",
         "code": \{
@@ -609,7 +609,7 @@ var content = `\{
           14021,
           14383
         ],
-        "filename": "pz04z98pfkludq58axh9bc.js",
+        "filename": "3nr1wx3lpaaxbu4d7zw7yf.js",
         "lineno": 443,
         "path": "/var/folders/82/w8hcc_tj7yq80482srkc1r2x0bj21j/T",
         "code": \{