Add support for RAM bundle groups

Summary: Adds support for “RAM bundle groups” (common section for module groups) to the new Buck integration

Reviewed By: jeanlauliac

Differential Revision: D5112065

fbshipit-source-id: 038c06b8f4133c7fcd39aba8bb04a5ef42594f3e

packager/src/ModuleGraph/output/__tests__/indexed-ram-bundle-test.js

@@ -26,14 +26,14 @@ let ids, modules, requireCall;
 const idForPath = ({path}) => getId(path);
 beforeAll(() => {
   modules = [
-    makeModule('a', 'script'),
-    makeModule('b'),
-    makeModule('c'),
-    makeModule('d'),
+    makeModule('a', [], 'script'),
+    makeModule('b', ['c']),
+    makeModule('c', ['f']),
+    makeModule('d', ['e']),
     makeModule('e'),
     makeModule('f'),
   ];
-  requireCall = makeModule('r', 'script', 'require(1);');
+  requireCall = makeModule('r', [], 'script', 'require(1);');
 
   ids = new Map(modules.map(({file}, i) => [file.path, i]));
   ({code, map} = createRamBundle());
@@ -79,7 +79,7 @@ it('creates a source map', () => {
   expect(map.x_facebook_offsets).toEqual([1, 2, 3, 4, 5, 6]);
 });
 
-describe('Optimization:', () => {
+describe('Startup section optimization', () => {
   let last, preloaded;
   beforeAll(() => {
     last = modules[modules.length - 1];
@@ -130,12 +130,62 @@ describe('Optimization:', () => {
           return section;
         }
     ));
-
   });
 });
 
-function createRamBundle(preloadedModules = new Set()) {
-  const build = indexedRamBundle.createBuilder(preloadedModules);
+describe('RAM groups / common sections', () => {
+  let groups, groupHeads;
+  beforeAll(() => {
+    groups = [
+      [modules[1], modules[2], modules[5]],
+      [modules[3], modules[4]],
+    ];
+    groupHeads = groups.map(g => g[0]);
+    ({code, map} = createRamBundle(undefined, groupHeads.map(getPath)));
+  });
+
+  it('supports grouping the transitive dependencies of files into common sections', () => {
+    const {codeOffset, table} = parseOffsetTable(code);
+
+    groups.forEach(group => {
+      const [head, ...deps] = group.map(x => idForPath(x.file));
+      const groupEntry = table[head];
+      deps.forEach(id => expect(table[id]).toEqual(groupEntry));
+
+      const [offset, length] = groupEntry;
+      const groupCode = code.slice(codeOffset + offset, codeOffset + offset + length - 1);
+      expect(groupCode.toString())
+        .toEqual(group.map(m => m.file.code).join('\n'));
+    });
+  });
+
+  it('reflects section groups in the source map', () => {
+    expect(map.x_facebook_offsets).toEqual([1, 2, 2, 5, 5, 2]);
+    const maps = map.sections.slice(-2);
+    const toplevelOffsets = [2, 5];
+
+    maps.map((groupMap, i) => [groups[i], groupMap]).forEach(([group, groupMap], i) => {
+      const offsets = group.reduce(moduleLineOffsets, [])[0];
+      expect(groupMap).toEqual({
+        map: {
+          version: 3,
+          sections: group.map((module, j) => ({
+            map: module.file.map,
+            offset: {line: offsets[j], column: 0},
+          })),
+        },
+        offset: {line: toplevelOffsets[i], column: 0},
+      });
+    });
+  });
+
+  function moduleLineOffsets([offsets = [], line = 0], module) {
+    return [[...offsets, line], line + countLines(module)];
+  }
+});
+
+function createRamBundle(preloadedModules = new Set(), ramGroups) {
+  const build = indexedRamBundle.createBuilder(preloadedModules, ramGroups);
   const result = build({
     filename: 'arbitrary/filename.js',
     idForPath,
@@ -149,10 +199,10 @@ function createRamBundle(preloadedModules = new Set()) {
   return {code: result.code, map: result.map};
 }
 
-function makeModule(name, type = 'module', moduleCode = `var ${name};`) {
-  const path = `/${name}.js`;
+function makeModule(name, deps = [], type = 'module', moduleCode = `var ${name};`) {
+  const path = makeModulePath(name);
   return {
-    dependencies: [],
+    dependencies: deps.map(makeDependency),
     file: {
       code: type === 'module' ? makeModuleCode(moduleCode) : moduleCode,
       map: type !== 'module'
@@ -177,6 +227,18 @@ function makeModuleCode(moduleCode) {
   return `__d(() => {${moduleCode}})`;
 }
 
+function makeModulePath(name) {
+  return `/${name}.js`;
+}
+
+function makeDependency(name) {
+  const path = makeModulePath(name);
+  return {
+    id: name,
+    path,
+  };
+}
+
 function getId(path) {
   if (path === requireCall.file.path) {
     return -1;

packager/src/ModuleGraph/output/indexed-ram-bundle.js

@@ -11,8 +11,10 @@
 'use strict';
 
 const buildSourceMapWithMetaData = require('../../../../local-cli/bundle/output/unbundle/build-unbundle-sourcemap-with-metadata.js');
+const nullthrows = require('fbjs/lib/nullthrows');
 
 const {buildTableAndContents, createModuleGroups} = require('../../../../local-cli/bundle/output/unbundle/as-indexed-file');
+const {createRamBundleGroups} = require('../../Bundler/util');
 const {concat} = require('./util');
 
 import type {FBIndexMap} from '../../lib/SourceMap.js';
@@ -23,12 +25,14 @@ function asIndexedRamBundle({
   idForPath,
   modules,
   preloadedModules,
+  ramGroupHeads,
   requireCalls,
 }) {
   const [startup, deferred] = partition(modules, preloadedModules);
   const startupModules = Array.from(concat(startup, requireCalls));
-  const deferredModules = deferred.map(m => toModuleTransport(m.file, idForPath));
-  const moduleGroups = createModuleGroups(new Map(), deferredModules);
+  const deferredModules = deferred.map(m => toModuleTransport(m, idForPath));
+  const ramGroups = createRamBundleGroups(ramGroupHeads || [], deferredModules, subtree);
+  const moduleGroups = createModuleGroups(ramGroups, deferredModules);
 
   const tableAndContents = buildTableAndContents(
     startupModules.map(getModuleCode).join('\n'),
@@ -41,15 +45,17 @@ function asIndexedRamBundle({
     code: Buffer.concat(tableAndContents),
     map: buildSourceMapWithMetaData({
       fixWrapperOffset: false,
-      startupModules: startupModules.map(m => toModuleTransport(m.file, idForPath)),
       lazyModules: deferredModules,
+      moduleGroups,
+      startupModules: startupModules.map(m => toModuleTransport(m, idForPath)),
     }),
   };
 }
 
-function toModuleTransport(file, idForPath) {
+function toModuleTransport({dependencies, file}, idForPath) {
   return {
     code: file.code,
+    dependencies,
     id: idForPath(file),
     map: file.map,
     name: file.path,
@@ -71,7 +77,26 @@ function partition(modules, preloadedModules) {
   return [startup, deferred];
 }
 
-function createBuilder(preloadedModules: Set<string>): OutputFn<FBIndexMap> {
-  return x => asIndexedRamBundle({preloadedModules, ...x});
+function *subtree(
+  moduleTransport,
+  moduleTransportsByPath,
+  seen = new Set(),
+) {
+  seen.add(moduleTransport.id);
+  for (const {path} of moduleTransport.dependencies) {
+    const dependency = nullthrows(moduleTransportsByPath.get(path));
+    if (!seen.has(dependency.id)) {
+      yield dependency.id;
+      yield *subtree(dependency, moduleTransportsByPath, seen);
+    }
+  }
 }
+
+function createBuilder(
+  preloadedModules: Set<string>,
+  ramGroupHeads: ?$ReadOnlyArray<string>,
+): OutputFn<FBIndexMap> {
+  return x => asIndexedRamBundle({...x, preloadedModules, ramGroupHeads});
+}
+
 exports.createBuilder = createBuilder;