diff --git a/local-cli/bundle/output/unbundle/as-assets.js b/local-cli/bundle/output/unbundle/as-assets.js index 8c167a0c4b0..9f3577118eb 100644 --- a/local-cli/bundle/output/unbundle/as-assets.js +++ b/local-cli/bundle/output/unbundle/as-assets.js @@ -69,6 +69,7 @@ function saveAsAssets( buildSourceMapWithMetaData({ startupModules: startupModules.concat(), lazyModules: lazyModules.concat(), + fixWrapperOffset: true, }), sourcemapSourcesRoot ); diff --git a/local-cli/bundle/output/unbundle/as-indexed-file.js b/local-cli/bundle/output/unbundle/as-indexed-file.js index f8fbe8ac6d6..46b18f230ed 100644 --- a/local-cli/bundle/output/unbundle/as-indexed-file.js +++ b/local-cli/bundle/output/unbundle/as-indexed-file.js @@ -19,8 +19,7 @@ const writeSourceMap = require('./write-sourcemap'); const {joinModules} = require('./util'); -import type ModuleTransport from '../../../../packager/src//lib/ModuleTransport'; -import type {Bundle, ModuleGroups, OutputOptions} from '../../types.flow'; +import type {Bundle, ModuleGroups, ModuleTransportLike, OutputOptions} from '../../types.flow'; const SIZEOF_UINT32 = 4; @@ -62,6 +61,7 @@ function saveAsIndexedFile( startupModules: startupModules.concat(), lazyModules: lazyModules.concat(), moduleGroups, + fixWrapperOffset: true, }), sourcemapSourcesRoot ); @@ -104,7 +104,7 @@ function entryOffset(n) { return (2 + n * 2) * SIZEOF_UINT32; } -function buildModuleTable(startupCode, buffers, moduleGroups) { +function buildModuleTable(startupCode, moduleBuffers, moduleGroups) { // table format: // - num_entries: uint_32 number of entries // - startup_code_len: uint_32 length of the startup section @@ -127,7 +127,7 @@ function buildModuleTable(startupCode, buffers, moduleGroups) { // entries let codeOffset = startupCode.length; - buffers.forEach(({id, buffer}) => { + moduleBuffers.forEach(({id, buffer}) => { const group = moduleGroups.groups.get(id); const idsInGroup = group ? [id].concat(Array.from(group)) : [id]; @@ -170,7 +170,12 @@ function buildModuleBuffers(modules, moduleGroups, encoding) { )); } -function buildTableAndContents(startupCode, modules, moduleGroups, encoding) { +function buildTableAndContents( + startupCode: string, + modules: $ReadOnlyArray, + moduleGroups: ModuleGroups, + encoding?: 'utf8' | 'utf16le' | 'ascii', +) { // file contents layout: // - magic number char[4] 0xE5 0xD1 0x0B 0xFB (0xFB0BD1E5 uint32 LE) // - offset table table see `buildModuleTables` @@ -190,7 +195,7 @@ function buildTableAndContents(startupCode, modules, moduleGroups, encoding) { function createModuleGroups( groups: Map>, - modules: Array, + modules: $ReadOnlyArray, ): ModuleGroups { return { groups, @@ -205,4 +210,6 @@ function * concat(iterators) { } } -module.exports = saveAsIndexedFile; +exports.save = saveAsIndexedFile; +exports.buildTableAndContents = buildTableAndContents; +exports.createModuleGroups = createModuleGroups; diff --git a/local-cli/bundle/output/unbundle/build-unbundle-sourcemap-with-metadata.js b/local-cli/bundle/output/unbundle/build-unbundle-sourcemap-with-metadata.js index 74c2348fb1b..23089834b41 100644 --- a/local-cli/bundle/output/unbundle/build-unbundle-sourcemap-with-metadata.js +++ b/local-cli/bundle/output/unbundle/build-unbundle-sourcemap-with-metadata.js @@ -10,25 +10,31 @@ */ 'use strict'; -const {combineSourceMaps, joinModules} = require('./util'); +const {combineSourceMaps, combineSourceMapsAddingOffsets, joinModules} = require('./util'); import type {ModuleGroups, ModuleTransportLike} from '../../types.flow'; type Params = { + fixWrapperOffset: boolean, lazyModules: Array, moduleGroups?: ModuleGroups, startupModules: Array, }; -module.exports = ({startupModules, lazyModules, moduleGroups}: Params) => { +module.exports = ({fixWrapperOffset, lazyModules, moduleGroups, startupModules}: Params) => { + const options = fixWrapperOffset ? {fixWrapperOffset: true} : undefined; const startupModule: ModuleTransportLike = { code: joinModules(startupModules), id: Number.MIN_SAFE_INTEGER, - map: combineSourceMaps({modules: startupModules}), + map: combineSourceMaps(startupModules, undefined, options), + sourcePath: '', }; - return combineSourceMaps({ - modules: [startupModule].concat(lazyModules), + + const map = combineSourceMapsAddingOffsets( + [startupModule].concat(lazyModules), moduleGroups, - withCustomOffsets: true, - }); + options, + ); + delete map.x_facebook_offsets[Number.MIN_SAFE_INTEGER]; + return map; }; diff --git a/local-cli/bundle/output/unbundle/index.js b/local-cli/bundle/output/unbundle/index.js index 62c13f819ee..780368bef8d 100644 --- a/local-cli/bundle/output/unbundle/index.js +++ b/local-cli/bundle/output/unbundle/index.js @@ -14,7 +14,7 @@ const Server = require('../../../../packager/src/Server'); const asAssets = require('./as-assets'); -const asIndexedFile = require('./as-indexed-file'); +const asIndexedFile = require('./as-indexed-file').save; import type Bundle from '../../../../packager/src//Bundler/Bundle'; import type {OutputOptions, RequestOptions} from '../../types.flow'; diff --git a/local-cli/bundle/output/unbundle/util.js b/local-cli/bundle/output/unbundle/util.js index 6974f3eb7e1..a43de796237 100644 --- a/local-cli/bundle/output/unbundle/util.js +++ b/local-cli/bundle/output/unbundle/util.js @@ -12,7 +12,7 @@ const invariant = require('fbjs/lib/invariant'); -import type {IndexMap, MappingsMap, SourceMap} from '../../../../packager/src/lib/SourceMap'; +import type {FBIndexMap, IndexMap, MappingsMap, SourceMap} from '../../../../packager/src/lib/SourceMap'; import type {ModuleGroups, ModuleTransportLike} from '../../types.flow'; const newline = /\r\n?|\n|\u2028|\u2029/g; @@ -43,39 +43,45 @@ const Section = (line: number, column: number, map: SourceMap) => ({map, offset: {line, column}}); -type CombineSourceMapsOptions = { - moduleGroups?: ModuleGroups, - modules: Array, - withCustomOffsets?: boolean, -}; +type CombineOptions = {fixWrapperOffset: boolean}; -function combineSourceMaps({ - moduleGroups, - modules, - withCustomOffsets, -}: CombineSourceMapsOptions): IndexMap { - const offsets = []; +function combineSourceMaps( + modules: Array, + moduleGroups?: ModuleGroups, + options?: ?CombineOptions, +): IndexMap { + const sections = combineMaps(modules, null, moduleGroups, options); + return {sections, version: 3}; +} + +function combineSourceMapsAddingOffsets( + modules: Array, + moduleGroups?: ModuleGroups, + options?: ?CombineOptions, +): FBIndexMap { + const x_facebook_offsets = []; + const sections = combineMaps(modules, x_facebook_offsets, moduleGroups, options); + return {sections, version: 3, x_facebook_offsets}; +} + +function combineMaps(modules, offsets: ?Array, moduleGroups, options) { const sections = []; - const sourceMap: IndexMap = withCustomOffsets - ? {sections, version: 3, x_facebook_offsets: offsets} - : {sections, version: 3}; let line = 0; modules.forEach(moduleTransport => { const {code, id, name} = moduleTransport; let column = 0; - let hasOffset = false; let group; let groupLines = 0; let {map} = moduleTransport; - if (withCustomOffsets) { - if (moduleGroups && moduleGroups.modulesInGroups.has(id)) { - // this is a module appended to another module - return; - } + if (moduleGroups && moduleGroups.modulesInGroups.has(id)) { + // this is a module appended to another module + return; + } + if (offsets != null) { group = moduleGroups && moduleGroups.groups.get(id); if (group && moduleGroups) { const {modulesById} = moduleGroups; @@ -86,13 +92,10 @@ function combineSourceMaps({ otherModules.forEach(m => { groupLines += countLines(m.code); }); - map = combineSourceMaps({ - modules: [moduleTransport].concat(otherModules), - }); + map = combineSourceMaps([moduleTransport].concat(otherModules)); } - hasOffset = id != null; - column = wrapperEnd(code); + column = options && options.fixWrapperOffset ? wrapperEnd(code) : 0; } invariant( @@ -100,7 +103,7 @@ function combineSourceMaps({ 'Random Access Bundle source maps cannot be built from raw mappings', ); sections.push(Section(line, column, map || lineToLineSourceMap(code, name))); - if (hasOffset) { + if (offsets != null && id != null) { offsets[id] = line; for (const moduleId of group || []) { offsets[moduleId] = line; @@ -109,7 +112,7 @@ function combineSourceMaps({ line += countLines(code) + groupLines; }); - return sourceMap; + return sections; } const joinModules = @@ -117,8 +120,9 @@ const joinModules = modules.map(m => m.code).join('\n'); module.exports = { - countLines, - lineToLineSourceMap, combineSourceMaps, + combineSourceMapsAddingOffsets, + countLines, joinModules, + lineToLineSourceMap, }; diff --git a/local-cli/bundle/types.flow.js b/local-cli/bundle/types.flow.js index 8f74bf12e58..6939aab08eb 100644 --- a/local-cli/bundle/types.flow.js +++ b/local-cli/bundle/types.flow.js @@ -19,7 +19,7 @@ export type {Bundle, FBSourceMap, ModuleTransport, SourceMap, Unbundle}; export type ModuleGroups = {| groups: Map>, - modulesById: Map, + modulesById: Map, modulesInGroups: Set, |}; @@ -28,6 +28,7 @@ export type ModuleTransportLike = { id: number, map?: $PropertyType, +name?: string, + sourcePath: string, }; export type OutputOptions = { diff --git a/packager/src/ModuleGraph/output/__tests__/as-indexed-ram-bundle-test.js b/packager/src/ModuleGraph/output/__tests__/as-indexed-ram-bundle-test.js new file mode 100644 index 00000000000..31937b61422 --- /dev/null +++ b/packager/src/ModuleGraph/output/__tests__/as-indexed-ram-bundle-test.js @@ -0,0 +1,231 @@ +/** + * Copyright (c) 2016-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + * @flow + */ +'use strict'; + +declare var jest: any; +jest.disableAutomock(); + +const indexedRamBundle = require('../indexed-ram-bundle'); + +declare var describe: any; +declare var expect: any; +declare var it: (string, () => ?Promise) => void; +declare var beforeAll: (() => ?Promise) => void; + +let code: Buffer; +let map; +let ids, modules, requireCall; +const idForPath = ({path}) => getId(path); +beforeAll(() => { + modules = [ + makeModule('a', 'script'), + makeModule('b'), + makeModule('c'), + makeModule('d'), + makeModule('e'), + makeModule('f'), + ]; + requireCall = makeModule('r', 'script', 'require(1);'); + + ids = new Map(modules.map(({file}, i) => [file.path, i])); + ({code, map} = createRamBundle()); +}); + +it('starts the bundle file with the magic number', () => { + expect(code.readUInt32LE(0)).toBe(0xFB0BD1E5); +}); + +it('contains the number of modules in the module table', () => { + expect(code.readUInt32LE(SIZEOF_INT32)).toBe(modules.length); +}); + +it('has the length correct of the startup section', () => { + expect(code.readUInt32LE(SIZEOF_INT32 * 2)) + .toBe(requireCall.file.code.length + 1); +}); + +it('contains the code after the offset table', () => { + const {codeOffset, startupSectionLength, table} = parseOffsetTable(code); + + const startupSection = + code.slice(codeOffset, codeOffset + startupSectionLength - 1); + expect(startupSection.toString()).toBe(requireCall.file.code); + + table.forEach(([offset, length], i) => { + const moduleCode = + code.slice(codeOffset + offset, codeOffset + offset + length - 1); + expect(moduleCode.toString()).toBe(modules[i].file.code); + }); +}); + +it('creates a source map', () => { + let line = countLines(requireCall); + expect(map.sections.slice(1)).toEqual(modules.map(m => { + const section = { + map: m.file.map || lineByLineMap(m.file.path), + offset: {column: 0, line}, + }; + line += countLines(m); + return section; + })); + expect(map.x_facebook_offsets).toEqual([1, 2, 3, 4, 5, 6]); +}); + +describe('Optimization:', () => { + let last, preloaded; + beforeAll(() => { + last = modules[modules.length - 1]; + preloaded = [modules[2], modules[3], last]; + ({code, map} = createRamBundle(new Set(preloaded.map(getPath)))); + }); + + it('supports additional modules in the startup section', () => { + const {codeOffset, startupSectionLength, table} = parseOffsetTable(code); + + const startupSection = + code.slice(codeOffset, codeOffset + startupSectionLength - 1); + expect(startupSection.toString()) + .toBe(preloaded.concat([requireCall]).map(getCode).join('\n')); + + + preloaded.forEach(m => { + const idx = idForPath(m.file); + expect(table[idx]).toEqual(m === last ? undefined : [0, 0]); + }); + + table.forEach(([offset, length], i) => { + if (offset !== 0 && length !== 0) { + const moduleCode = + code.slice(codeOffset + offset, codeOffset + offset + length - 1); + expect(moduleCode.toString()).toBe(modules[i].file.code); + } + }); + }); + + it('reflects additional sources in the startup section in the source map', () => { + let line = preloaded.reduce( + (l, m) => l + countLines(m), + countLines(requireCall), + ); + + expect(map.x_facebook_offsets).toEqual([4, 5,,, 6]); // eslint-disable-line no-sparse-arrays + + expect(map.sections.slice(1)).toEqual( + modules + .filter(not(Set.prototype.has), new Set(preloaded)) + .map(m => { + const section = { + map: m.file.map || lineByLineMap(m.file.path), + offset: {column: 0, line}, + }; + line += countLines(m); + return section; + } + )); + + }); +}); + +function createRamBundle(preloadedModules = new Set()) { + const build = indexedRamBundle.createBuilder(preloadedModules); + const result = build({ + filename: 'arbitrary/filename.js', + idForPath, + modules, + requireCalls: [requireCall], + }); + + if (typeof result.code === 'string') { + throw new Error('Expected a buffer, not a string'); + } + return {code: result.code, map: result.map}; +} + +function makeModule(name, type = 'module', moduleCode = `var ${name};`) { + const path = `/${name}.js`; + return { + dependencies: [], + file: { + code: type === 'module' ? makeModuleCode(moduleCode) : moduleCode, + map: type !== 'module' + ? null + : makeModuleMap(name, path), + path, + type, + }, + }; +} + +function makeModuleMap(name, path) { + return { + version: 3, + mappings: Array(parseInt(name, 36) + 1).join(','), + names: [name], + sources: [path], + }; +} + +function makeModuleCode(moduleCode) { + return `__d(() => {${moduleCode}})`; +} + +function getId(path) { + if (path === requireCall.file.path) { + return -1; + } + + const id = ids.get(path); + if (id == null) { + throw new Error(`Unknown file: ${path}`); + } + return id; +} + +function getCode(module) { + return module.file.code; +} + +function getPath(module) { + return module.file.path; +} + +const SIZEOF_INT32 = 4; +function parseOffsetTable(buffer) { + const n = buffer.readUInt32LE(SIZEOF_INT32); + const startupSectionLength = buffer.readUInt32LE(SIZEOF_INT32 * 2); + const baseOffset = SIZEOF_INT32 * 3; + const table = Array(n); + for (let i = 0; i < n; ++i) { + const offset = baseOffset + i * 2 * SIZEOF_INT32; + table[i] = [buffer.readUInt32LE(offset), buffer.readUInt32LE(offset + SIZEOF_INT32)]; + } + return { + codeOffset: baseOffset + n * 2 * SIZEOF_INT32, + startupSectionLength, + table, + }; +} + +function countLines(module) { + return module.file.code.split('\n').length; +} + +function lineByLineMap(file) { + return { + file: file, + mappings: 'AAAA;', + names: [], + sources: [file], + version: 3, + }; +} + +const not = fn => function() { return !fn.apply(this, arguments); }; diff --git a/packager/src/ModuleGraph/output/indexed-ram-bundle.js b/packager/src/ModuleGraph/output/indexed-ram-bundle.js new file mode 100644 index 00000000000..8f8712ad29c --- /dev/null +++ b/packager/src/ModuleGraph/output/indexed-ram-bundle.js @@ -0,0 +1,77 @@ +/** + * Copyright (c) 2016-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + * @flow + */ +'use strict'; + +const buildSourceMapWithMetaData = require('../../../../local-cli/bundle/output/unbundle/build-unbundle-sourcemap-with-metadata.js'); + +const {buildTableAndContents, createModuleGroups} = require('../../../../local-cli/bundle/output/unbundle/as-indexed-file'); +const {concat} = require('./util'); + +import type {FBIndexMap} from '../../lib/SourceMap.js'; +import type {OutputFn} from '../types.flow'; + +function asIndexedRamBundle({ + filename, + idForPath, + modules, + preloadedModules, + requireCalls, +}) { + const [startup, deferred] = partition(modules, preloadedModules); + const startupModules = Array.from(concat(startup, requireCalls)); + const deferredModules = deferred.map(m => toModuleTransport(m.file, idForPath)); + const moduleGroups = createModuleGroups(new Map(), deferredModules); + + const tableAndContents = buildTableAndContents( + startupModules.map(getModuleCode).join('\n'), + deferredModules, + moduleGroups, + 'utf8', + ); + + return { + code: Buffer.concat(tableAndContents), + map: buildSourceMapWithMetaData({ + fixWrapperOffset: false, + startupModules: startupModules.map(m => toModuleTransport(m.file, idForPath)), + lazyModules: deferredModules, + }), + }; +} + +function toModuleTransport(file, idForPath) { + return { + code: file.code, + id: idForPath(file), + map: file.map, + name: file.path, + sourcePath: file.path, + }; +} + +function getModuleCode(module) { + return module.file.code; +} + +function partition(modules, preloadedModules) { + const startup = []; + const deferred = []; + for (const module of modules) { + (preloadedModules.has(module.file.path) ? startup : deferred).push(module); + } + + return [startup, deferred]; +} + +function createBuilder(preloadedModules: Set): OutputFn { + return x => asIndexedRamBundle({preloadedModules, ...x}); +} +exports.createBuilder = createBuilder; diff --git a/packager/src/ModuleGraph/output/as-plain-bundle.js b/packager/src/ModuleGraph/output/plain-bundle.js similarity index 86% rename from packager/src/ModuleGraph/output/as-plain-bundle.js rename to packager/src/ModuleGraph/output/plain-bundle.js index 9c1bf99ecdc..813b31e0eaa 100644 --- a/packager/src/ModuleGraph/output/as-plain-bundle.js +++ b/packager/src/ModuleGraph/output/plain-bundle.js @@ -13,7 +13,7 @@ const meta = require('../../../../local-cli/bundle/output/meta'); const {createIndexMap} = require('./source-map'); -const {addModuleIdsToModuleWrapper} = require('./util'); +const {addModuleIdsToModuleWrapper, concat} = require('./util'); import type {OutputFn} from '../types.flow'; @@ -55,16 +55,10 @@ function asPlainBundle({ }; } -module.exports = (asPlainBundle: OutputFn); +module.exports = (asPlainBundle: OutputFn<>); const reLine = /^/gm; function countLines(string: string): number { //$FlowFixMe This regular expression always matches return string.match(reLine).length; } - -function* concat(...iterables: Array>): Iterable { - for (const it of iterables) { - yield* it; - } -} diff --git a/packager/src/ModuleGraph/output/util.js b/packager/src/ModuleGraph/output/util.js index b2fae65d1cb..eb84ed880b9 100644 --- a/packager/src/ModuleGraph/output/util.js +++ b/packager/src/ModuleGraph/output/util.js @@ -44,6 +44,14 @@ exports.addModuleIdsToModuleWrapper = ( ); }; +exports.concat = function* concat( + ...iterables: Array> +): Iterable { + for (const it of iterables) { + yield* it; + } +}; + // Creates an idempotent function that returns numeric IDs for objects based // on their `path` property. exports.createIdForPathFn = (): ({path: string} => number) => { diff --git a/packager/src/ModuleGraph/types.flow.js b/packager/src/ModuleGraph/types.flow.js index e2a8eea88dc..5820ddaf2ae 100644 --- a/packager/src/ModuleGraph/types.flow.js +++ b/packager/src/ModuleGraph/types.flow.js @@ -10,7 +10,7 @@ */ 'use strict'; -import type {MappingsMap, SourceMap} from '../lib/SourceMap'; +import type {FBSourceMap, MappingsMap, SourceMap} from '../lib/SourceMap'; import type {Ast} from 'babel-core'; import type {Console} from 'console'; export type {Transformer} from '../JSTransformer/worker/worker.js'; @@ -26,7 +26,7 @@ type Dependency = {| export type File = {| code: string, - map?: ?Object, + map?: ?MappingsMap, path: string, type: CodeFileTypes, |}; @@ -75,18 +75,18 @@ export type PostProcessModules = ( entryPoints: Array, ) => Iterable; -export type OutputFn = ({| +export type OutputFn = ({| filename: string, idForPath: IdForPathFn, modules: Iterable, requireCalls: Iterable, sourceMapPath?: string, -|}) => OutputResult; +|}) => OutputResult; -type OutputResult = {| +type OutputResult = {| code: string | Buffer, extraFiles?: Iterable<[string, string | Buffer]>, - map: SourceMap, + map: M, |}; export type PackageData = {| diff --git a/packager/src/lib/SourceMap.js b/packager/src/lib/SourceMap.js index afbce1016ec..8efaddba5c1 100644 --- a/packager/src/lib/SourceMap.js +++ b/packager/src/lib/SourceMap.js @@ -28,8 +28,9 @@ export type IndexMap = { version: number, }; +export type FBIndexMap = IndexMap & FBExtensions; export type SourceMap = IndexMap | MappingsMap; -export type FBSourceMap = (IndexMap & FBExtensions) | (MappingsMap & FBExtensions); +export type FBSourceMap = FBIndexMap | (MappingsMap & FBExtensions); function isMappingsMap(map: SourceMap)/*: %checks*/ { return map.mappings !== undefined;