Merge pull request #29161 from Microsoft/incrementalBuild

Supports incremental build in tsc --b --w mode

src/compiler/builder.ts

@@ -49,7 +49,27 @@ namespace ts {
         /**
          * program corresponding to this state
          */
-        program: Program;
+        program: Program | undefined;
+        /**
+         * compilerOptions for the program
+         */
+        compilerOptions: CompilerOptions;
+        /**
+         * Files pending to be emitted
+         */
+        affectedFilesPendingEmit: ReadonlyArray<Path> | undefined;
+        /**
+         * Current index to retrieve pending affected file
+         */
+        affectedFilesPendingEmitIndex: number | undefined;
+        /**
+         * Already seen affected files
+         */
+        seenEmittedFiles: Map<true> | undefined;
+        /**
+         * true if program has been emitted
+         */
+        programEmitComplete?: true;
     }
 
     function hasSameKeys<T, U>(map1: ReadonlyMap<T> | undefined, map2: ReadonlyMap<U> | undefined): boolean {
@@ -64,6 +84,7 @@ namespace ts {
         const state = BuilderState.create(newProgram, getCanonicalFileName, oldState) as BuilderProgramState;
         state.program = newProgram;
         const compilerOptions = newProgram.getCompilerOptions();
+        state.compilerOptions = compilerOptions;
         // With --out or --outFile, any change affects all semantic diagnostics so no need to cache them
         // With --isolatedModules, emitting changed file doesnt emit dependent files so we cant know of dependent files to retrieve errors so dont cache the errors
         if (!compilerOptions.outFile && !compilerOptions.out && !compilerOptions.isolatedModules) {
@@ -72,7 +93,7 @@ namespace ts {
         state.changedFilesSet = createMap<true>();
 
         const useOldState = BuilderState.canReuseOldState(state.referencedMap, oldState);
-        const oldCompilerOptions = useOldState ? oldState!.program.getCompilerOptions() : undefined;
+        const oldCompilerOptions = useOldState ? oldState!.compilerOptions : undefined;
         const canCopySemanticDiagnostics = useOldState && oldState!.semanticDiagnosticsPerFile && !!state.semanticDiagnosticsPerFile &&
             !compilerOptionsAffectSemanticDiagnostics(compilerOptions, oldCompilerOptions!);
         if (useOldState) {
@@ -87,6 +108,10 @@ namespace ts {
 
             // Copy old state's changed files set
             copyEntries(oldState!.changedFilesSet, state.changedFilesSet);
+            if (!compilerOptions.outFile && !compilerOptions.out && oldState!.affectedFilesPendingEmit) {
+                state.affectedFilesPendingEmit = oldState!.affectedFilesPendingEmit;
+                state.affectedFilesPendingEmitIndex = oldState!.affectedFilesPendingEmitIndex;
+            }
         }
 
         // Update changed files and copy semantic diagnostics if we can
@@ -112,7 +137,7 @@ namespace ts {
                 state.changedFilesSet.set(sourceFilePath, true);
             }
             else if (canCopySemanticDiagnostics) {
-                const sourceFile = state.program.getSourceFileByPath(sourceFilePath as Path)!;
+                const sourceFile = newProgram.getSourceFileByPath(sourceFilePath as Path)!;
 
                 if (sourceFile.isDeclarationFile && !copyDeclarationFileDiagnostics) { return; }
                 if (sourceFile.hasNoDefaultLib && !copyLibFileDiagnostics) { return; }
@@ -132,6 +157,38 @@ namespace ts {
         return state;
     }
 
+    /**
+     * Releases program and other related not needed properties
+     */
+    function releaseCache(state: BuilderProgramState) {
+        BuilderState.releaseCache(state);
+        state.program = undefined;
+    }
+
+    /**
+     * Creates a clone of the state
+     */
+    function cloneBuilderProgramState(state: Readonly<BuilderProgramState>): BuilderProgramState {
+        const newState = BuilderState.clone(state) as BuilderProgramState;
+        newState.semanticDiagnosticsPerFile = cloneMapOrUndefined(state.semanticDiagnosticsPerFile);
+        newState.changedFilesSet = cloneMap(state.changedFilesSet);
+        newState.affectedFiles = state.affectedFiles;
+        newState.affectedFilesIndex = state.affectedFilesIndex;
+        newState.currentChangedFilePath = state.currentChangedFilePath;
+        newState.currentAffectedFilesSignatures = cloneMapOrUndefined(state.currentAffectedFilesSignatures);
+        newState.currentAffectedFilesExportedModulesMap = cloneMapOrUndefined(state.currentAffectedFilesExportedModulesMap);
+        newState.seenAffectedFiles = cloneMapOrUndefined(state.seenAffectedFiles);
+        newState.cleanedDiagnosticsOfLibFiles = state.cleanedDiagnosticsOfLibFiles;
+        newState.semanticDiagnosticsFromOldState = cloneMapOrUndefined(state.semanticDiagnosticsFromOldState);
+        newState.program = state.program;
+        newState.compilerOptions = state.compilerOptions;
+        newState.affectedFilesPendingEmit = state.affectedFilesPendingEmit;
+        newState.affectedFilesPendingEmitIndex = state.affectedFilesPendingEmitIndex;
+        newState.seenEmittedFiles = cloneMapOrUndefined(state.seenEmittedFiles);
+        newState.programEmitComplete = state.programEmitComplete;
+        return newState;
+    }
+
     /**
      * Verifies that source file is ok to be used in calls that arent handled by next
      */
@@ -182,10 +239,11 @@ namespace ts {
 
             // With --out or --outFile all outputs go into single file
             // so operations are performed directly on program, return program
-            const compilerOptions = state.program.getCompilerOptions();
+            const program = Debug.assertDefined(state.program);
+            const compilerOptions = program.getCompilerOptions();
             if (compilerOptions.outFile || compilerOptions.out) {
                 Debug.assert(!state.semanticDiagnosticsPerFile);
-                return state.program;
+                return program;
             }
 
             // Get next batch of affected files
@@ -193,13 +251,34 @@ namespace ts {
             if (state.exportedModulesMap) {
                 state.currentAffectedFilesExportedModulesMap = state.currentAffectedFilesExportedModulesMap || createMap<BuilderState.ReferencedSet | false>();
             }
-            state.affectedFiles = BuilderState.getFilesAffectedBy(state, state.program, nextKey.value as Path, cancellationToken, computeHash, state.currentAffectedFilesSignatures, state.currentAffectedFilesExportedModulesMap);
+            state.affectedFiles = BuilderState.getFilesAffectedBy(state, program, nextKey.value as Path, cancellationToken, computeHash, state.currentAffectedFilesSignatures, state.currentAffectedFilesExportedModulesMap);
             state.currentChangedFilePath = nextKey.value as Path;
             state.affectedFilesIndex = 0;
             state.seenAffectedFiles = state.seenAffectedFiles || createMap<true>();
         }
     }
 
+    /**
+     * Returns next file to be emitted from files that retrieved semantic diagnostics but did not emit yet
+     */
+    function getNextAffectedFilePendingEmit(state: BuilderProgramState): SourceFile | undefined {
+        const { affectedFilesPendingEmit } = state;
+        if (affectedFilesPendingEmit) {
+            const seenEmittedFiles = state.seenEmittedFiles || (state.seenEmittedFiles = createMap());
+            for (let i = state.affectedFilesPendingEmitIndex!; i < affectedFilesPendingEmit.length; i++) {
+                const affectedFile = Debug.assertDefined(state.program).getSourceFileByPath(affectedFilesPendingEmit[i]);
+                if (affectedFile && !seenEmittedFiles.has(affectedFile.path)) {
+                    // emit this file
+                    state.affectedFilesPendingEmitIndex = i;
+                    return affectedFile;
+                }
+            }
+            state.affectedFilesPendingEmit = undefined;
+            state.affectedFilesPendingEmitIndex = undefined;
+        }
+        return undefined;
+    }
+
     /**
      * Remove the semantic diagnostics cached from old state for affected File and the files that are referencing modules that export entities from affected file
      */
@@ -212,9 +291,10 @@ namespace ts {
         // Clean lib file diagnostics if its all files excluding default files to emit
         if (state.allFilesExcludingDefaultLibraryFile === state.affectedFiles && !state.cleanedDiagnosticsOfLibFiles) {
             state.cleanedDiagnosticsOfLibFiles = true;
-            const options = state.program.getCompilerOptions();
-            if (forEach(state.program.getSourceFiles(), f =>
-                state.program.isSourceFileDefaultLibrary(f) &&
+            const program = Debug.assertDefined(state.program);
+            const options = program.getCompilerOptions();
+            if (forEach(program.getSourceFiles(), f =>
+                program.isSourceFileDefaultLibrary(f) &&
                 !skipTypeChecking(f, options) &&
                 removeSemanticDiagnosticsOf(state, f.path)
             )) {
@@ -317,21 +397,27 @@ namespace ts {
      * This is called after completing operation on the next affected file.
      * The operations here are postponed to ensure that cancellation during the iteration is handled correctly
      */
-    function doneWithAffectedFile(state: BuilderProgramState, affected: SourceFile | Program) {
+    function doneWithAffectedFile(state: BuilderProgramState, affected: SourceFile | Program, isPendingEmit?: boolean) {
         if (affected === state.program) {
             state.changedFilesSet.clear();
+            state.programEmitComplete = true;
         }
         else {
             state.seenAffectedFiles!.set((affected as SourceFile).path, true);
-            state.affectedFilesIndex!++;
+            if (isPendingEmit) {
+                state.affectedFilesPendingEmitIndex!++;
+            }
+            else {
+                state.affectedFilesIndex!++;
+            }
         }
     }
 
     /**
      * Returns the result with affected file
      */
-    function toAffectedFileResult<T>(state: BuilderProgramState, result: T, affected: SourceFile | Program): AffectedFileResult<T> {
-        doneWithAffectedFile(state, affected);
+    function toAffectedFileResult<T>(state: BuilderProgramState, result: T, affected: SourceFile | Program, isPendingEmit?: boolean): AffectedFileResult<T> {
+        doneWithAffectedFile(state, affected, isPendingEmit);
         return { result, affected };
     }
 
@@ -350,7 +436,7 @@ namespace ts {
         }
 
         // Diagnostics werent cached, get them from program, and cache the result
-        const diagnostics = state.program.getSemanticDiagnostics(sourceFile, cancellationToken);
+        const diagnostics = Debug.assertDefined(state.program).getSemanticDiagnostics(sourceFile, cancellationToken);
         if (state.semanticDiagnosticsPerFile) {
             state.semanticDiagnosticsPerFile.set(path, diagnostics);
         }
@@ -386,7 +472,7 @@ namespace ts {
                 rootNames: newProgramOrRootNames,
                 options: hostOrOptions as CompilerOptions,
                 host: oldProgramOrHost as CompilerHost,
-                oldProgram: oldProgram && oldProgram.getProgram(),
+                oldProgram: oldProgram && oldProgram.getProgramOrUndefined(),
                 configFileParsingDiagnostics,
                 projectReferences
             });
@@ -419,28 +505,31 @@ namespace ts {
         /**
          * Computing hash to for signature verification
          */
-        const computeHash = host.createHash || identity;
-        const state = createBuilderProgramState(newProgram, getCanonicalFileName, oldState);
+        const computeHash = host.createHash || generateDjb2Hash;
+        let state = createBuilderProgramState(newProgram, getCanonicalFileName, oldState);
+        let backupState: BuilderProgramState | undefined;
 
         // To ensure that we arent storing any references to old program or new program without state
         newProgram = undefined!; // TODO: GH#18217
         oldProgram = undefined;
         oldState = undefined;
 
-        const result: BuilderProgram = {
-            getState: () => state,
-            getProgram: () => state.program,
-            getCompilerOptions: () => state.program.getCompilerOptions(),
-            getSourceFile: fileName => state.program.getSourceFile(fileName),
-            getSourceFiles: () => state.program.getSourceFiles(),
-            getOptionsDiagnostics: cancellationToken => state.program.getOptionsDiagnostics(cancellationToken),
-            getGlobalDiagnostics: cancellationToken => state.program.getGlobalDiagnostics(cancellationToken),
-            getConfigFileParsingDiagnostics: () => configFileParsingDiagnostics || state.program.getConfigFileParsingDiagnostics(),
-            getSyntacticDiagnostics: (sourceFile, cancellationToken) => state.program.getSyntacticDiagnostics(sourceFile, cancellationToken),
-            getSemanticDiagnostics,
-            emit,
-            getAllDependencies: sourceFile => BuilderState.getAllDependencies(state, state.program, sourceFile),
-            getCurrentDirectory: () => state.program.getCurrentDirectory()
+        const result = createRedirectedBuilderProgram(state, configFileParsingDiagnostics);
+        result.getState = () => state;
+        result.backupState = () => {
+            Debug.assert(backupState === undefined);
+            backupState = cloneBuilderProgramState(state);
+        };
+        result.restoreState = () => {
+            state = Debug.assertDefined(backupState);
+            backupState = undefined;
+        };
+        result.getAllDependencies = sourceFile => BuilderState.getAllDependencies(state, Debug.assertDefined(state.program), sourceFile);
+        result.getSemanticDiagnostics = getSemanticDiagnostics;
+        result.emit = emit;
+        result.releaseProgram = () => {
+            releaseCache(state);
+            backupState = undefined;
         };
 
         if (kind === BuilderProgramKind.SemanticDiagnosticsBuilderProgram) {
@@ -461,18 +550,39 @@ namespace ts {
          * in that order would be used to write the files
          */
         function emitNextAffectedFile(writeFile?: WriteFileCallback, cancellationToken?: CancellationToken, emitOnlyDtsFiles?: boolean, customTransformers?: CustomTransformers): AffectedFileResult<EmitResult> {
-            const affected = getNextAffectedFile(state, cancellationToken, computeHash);
+            let affected = getNextAffectedFile(state, cancellationToken, computeHash);
+            let isPendingEmitFile = false;
             if (!affected) {
-                // Done
-                return undefined;
+                if (!state.compilerOptions.out && !state.compilerOptions.outFile) {
+                    affected = getNextAffectedFilePendingEmit(state);
+                    if (!affected) {
+                        return undefined;
+                    }
+                    isPendingEmitFile = true;
+                }
+                else {
+                    const program = Debug.assertDefined(state.program);
+                    // Check if program uses any prepend project references, if thats the case we cant track of the js files of those, so emit even though there are no changes
+                    if (state.programEmitComplete || !some(program.getProjectReferences(), ref => !!ref.prepend)) {
+                        state.programEmitComplete = true;
+                        return undefined;
+                    }
+                    affected = program;
+                }
+            }
+
+            // Mark seen emitted files if there are pending files to be emitted
+            if (state.affectedFilesPendingEmit && state.program !== affected) {
+                (state.seenEmittedFiles || (state.seenEmittedFiles = createMap())).set((affected as SourceFile).path, true);
             }
 
             return toAffectedFileResult(
                 state,
                 // When whole program is affected, do emit only once (eg when --out or --outFile is specified)
                 // Otherwise just affected file
-                state.program.emit(affected === state.program ? undefined : affected as SourceFile, writeFile || host.writeFile, cancellationToken, emitOnlyDtsFiles, customTransformers),
-                affected
+                Debug.assertDefined(state.program).emit(affected === state.program ? undefined : affected as SourceFile, writeFile || host.writeFile, cancellationToken, emitOnlyDtsFiles, customTransformers),
+                affected,
+                isPendingEmitFile
             );
         }
 
@@ -512,7 +622,7 @@ namespace ts {
                     };
                 }
             }
-            return state.program.emit(targetSourceFile, writeFile || host.writeFile, cancellationToken, emitOnlyDtsFiles, customTransformers);
+            return Debug.assertDefined(state.program).emit(targetSourceFile, writeFile || host.writeFile, cancellationToken, emitOnlyDtsFiles, customTransformers);
         }
 
         /**
@@ -560,33 +670,74 @@ namespace ts {
          */
         function getSemanticDiagnostics(sourceFile?: SourceFile, cancellationToken?: CancellationToken): ReadonlyArray<Diagnostic> {
             assertSourceFileOkWithoutNextAffectedCall(state, sourceFile);
-            const compilerOptions = state.program.getCompilerOptions();
+            const compilerOptions = Debug.assertDefined(state.program).getCompilerOptions();
             if (compilerOptions.outFile || compilerOptions.out) {
                 Debug.assert(!state.semanticDiagnosticsPerFile);
                 // We dont need to cache the diagnostics just return them from program
-                return state.program.getSemanticDiagnostics(sourceFile, cancellationToken);
+                return Debug.assertDefined(state.program).getSemanticDiagnostics(sourceFile, cancellationToken);
             }
 
             if (sourceFile) {
                 return getSemanticDiagnosticsOfFile(state, sourceFile, cancellationToken);
             }
 
-            if (kind === BuilderProgramKind.SemanticDiagnosticsBuilderProgram) {
-                // When semantic builder asks for diagnostics of the whole program,
-                // ensure that all the affected files are handled
-                let affected: SourceFile | Program | undefined;
-                while (affected = getNextAffectedFile(state, cancellationToken, computeHash)) {
-                    doneWithAffectedFile(state, affected);
+            // When semantic builder asks for diagnostics of the whole program,
+            // ensure that all the affected files are handled
+            let affected: SourceFile | Program | undefined;
+            let affectedFilesPendingEmit: Path[] | undefined;
+            while (affected = getNextAffectedFile(state, cancellationToken, computeHash)) {
+                if (affected !== state.program && kind === BuilderProgramKind.EmitAndSemanticDiagnosticsBuilderProgram) {
+                    (affectedFilesPendingEmit || (affectedFilesPendingEmit = [])).push((affected as SourceFile).path);
+                }
+                doneWithAffectedFile(state, affected);
+            }
+
+            // In case of emit builder, cache the files to be emitted
+            if (affectedFilesPendingEmit) {
+                state.affectedFilesPendingEmit = concatenate(state.affectedFilesPendingEmit, affectedFilesPendingEmit);
+                // affectedFilesPendingEmitIndex === undefined
+                // - means the emit state.affectedFilesPendingEmit was undefined before adding current affected files
+                //   so start from 0 as array would be affectedFilesPendingEmit
+                // else, continue to iterate from existing index, the current set is appended to existing files
+                if (state.affectedFilesPendingEmitIndex === undefined) {
+                    state.affectedFilesPendingEmitIndex = 0;
                 }
             }
 
             let diagnostics: Diagnostic[] | undefined;
-            for (const sourceFile of state.program.getSourceFiles()) {
+            for (const sourceFile of Debug.assertDefined(state.program).getSourceFiles()) {
                 diagnostics = addRange(diagnostics, getSemanticDiagnosticsOfFile(state, sourceFile, cancellationToken));
             }
             return diagnostics || emptyArray;
         }
     }
+
+    export function createRedirectedBuilderProgram(state: { program: Program | undefined; compilerOptions: CompilerOptions; }, configFileParsingDiagnostics: ReadonlyArray<Diagnostic>): BuilderProgram {
+        return {
+            getState: notImplemented,
+            backupState: noop,
+            restoreState: noop,
+            getProgram,
+            getProgramOrUndefined: () => state.program,
+            releaseProgram: () => state.program = undefined,
+            getCompilerOptions: () => state.compilerOptions,
+            getSourceFile: fileName => getProgram().getSourceFile(fileName),
+            getSourceFiles: () => getProgram().getSourceFiles(),
+            getOptionsDiagnostics: cancellationToken => getProgram().getOptionsDiagnostics(cancellationToken),
+            getGlobalDiagnostics: cancellationToken => getProgram().getGlobalDiagnostics(cancellationToken),
+            getConfigFileParsingDiagnostics: () => configFileParsingDiagnostics,
+            getSyntacticDiagnostics: (sourceFile, cancellationToken) => getProgram().getSyntacticDiagnostics(sourceFile, cancellationToken),
+            getDeclarationDiagnostics: (sourceFile, cancellationToken) => getProgram().getDeclarationDiagnostics(sourceFile, cancellationToken),
+            getSemanticDiagnostics: (sourceFile, cancellationToken) => getProgram().getSemanticDiagnostics(sourceFile, cancellationToken),
+            emit: (sourceFile, writeFile, cancellationToken, emitOnlyDts, customTransformers) => getProgram().emit(sourceFile, writeFile, cancellationToken, emitOnlyDts, customTransformers),
+            getAllDependencies: notImplemented,
+            getCurrentDirectory: () => getProgram().getCurrentDirectory()
+        };
+
+        function getProgram() {
+            return Debug.assertDefined(state.program);
+        }
+    }
 }
 
 namespace ts {
@@ -614,10 +765,24 @@ namespace ts {
     export interface BuilderProgram {
         /*@internal*/
         getState(): BuilderProgramState;
+        /*@internal*/
+        backupState(): void;
+        /*@internal*/
+        restoreState(): void;
         /**
          * Returns current program
          */
         getProgram(): Program;
+        /**
+         * Returns current program that could be undefined if the program was released
+         */
+        /*@internal*/
+        getProgramOrUndefined(): Program | undefined;
+        /**
+         * Releases reference to the program, making all the other operations that need program to fail.
+         */
+        /*@internal*/
+        releaseProgram(): void;
         /**
          * Get compiler options of the program
          */
@@ -646,10 +811,15 @@ namespace ts {
          * Get the syntax diagnostics, for all source files if source file is not supplied
          */
         getSyntacticDiagnostics(sourceFile?: SourceFile, cancellationToken?: CancellationToken): ReadonlyArray<Diagnostic>;
+        /**
+         * Get the declaration diagnostics, for all source files if source file is not supplied
+         */
+        getDeclarationDiagnostics(sourceFile?: SourceFile, cancellationToken?: CancellationToken): ReadonlyArray<DiagnosticWithLocation>;
         /**
          * Get all the dependencies of the file
          */
         getAllDependencies(sourceFile: SourceFile): ReadonlyArray<string>;
+
         /**
          * Gets the semantic diagnostics from the program corresponding to this state of file (if provided) or whole program
          * The semantic diagnostics are cached and managed here
@@ -726,22 +896,7 @@ namespace ts {
     export function createAbstractBuilder(newProgram: Program, host: BuilderProgramHost, oldProgram?: BuilderProgram, configFileParsingDiagnostics?: ReadonlyArray<Diagnostic>): BuilderProgram;
     export function createAbstractBuilder(rootNames: ReadonlyArray<string> | undefined, options: CompilerOptions | undefined, host?: CompilerHost, oldProgram?: BuilderProgram, configFileParsingDiagnostics?: ReadonlyArray<Diagnostic>, projectReferences?: ReadonlyArray<ProjectReference>): BuilderProgram;
     export function createAbstractBuilder(newProgramOrRootNames: Program | ReadonlyArray<string> | undefined, hostOrOptions: BuilderProgramHost | CompilerOptions | undefined, oldProgramOrHost?: CompilerHost | BuilderProgram, configFileParsingDiagnosticsOrOldProgram?: ReadonlyArray<Diagnostic> | BuilderProgram, configFileParsingDiagnostics?: ReadonlyArray<Diagnostic>, projectReferences?: ReadonlyArray<ProjectReference>): BuilderProgram {
-        const { newProgram: program } = getBuilderCreationParameters(newProgramOrRootNames, hostOrOptions, oldProgramOrHost, configFileParsingDiagnosticsOrOldProgram, configFileParsingDiagnostics, projectReferences);
-        return {
-            // Only return program, all other methods are not implemented
-            getProgram: () => program,
-            getState: notImplemented,
-            getCompilerOptions: notImplemented,
-            getSourceFile: notImplemented,
-            getSourceFiles: notImplemented,
-            getOptionsDiagnostics: notImplemented,
-            getGlobalDiagnostics: notImplemented,
-            getConfigFileParsingDiagnostics: notImplemented,
-            getSyntacticDiagnostics: notImplemented,
-            getSemanticDiagnostics: notImplemented,
-            emit: notImplemented,
-            getAllDependencies: notImplemented,
-            getCurrentDirectory: notImplemented
-        };
+        const { newProgram, configFileParsingDiagnostics: newConfigFileParsingDiagnostics } = getBuilderCreationParameters(newProgramOrRootNames, hostOrOptions, oldProgramOrHost, configFileParsingDiagnosticsOrOldProgram, configFileParsingDiagnostics, projectReferences);
+        return createRedirectedBuilderProgram({ program: newProgram, compilerOptions: newProgram.getCompilerOptions() }, newConfigFileParsingDiagnostics);
     }
 }

src/compiler/builderState.ts

@@ -50,11 +50,15 @@ namespace ts {
         /**
          * Cache of all files excluding default library file for the current program
          */
-        allFilesExcludingDefaultLibraryFile: ReadonlyArray<SourceFile> | undefined;
+        allFilesExcludingDefaultLibraryFile?: ReadonlyArray<SourceFile>;
         /**
          * Cache of all the file names
          */
-        allFileNames: ReadonlyArray<string> | undefined;
+        allFileNames?: ReadonlyArray<string>;
+    }
+
+    export function cloneMapOrUndefined<T>(map: ReadonlyMap<T> | undefined) {
+        return map ? cloneMap(map) : undefined;
     }
 }
 
@@ -230,9 +234,32 @@ namespace ts.BuilderState {
             fileInfos,
             referencedMap,
             exportedModulesMap,
-            hasCalledUpdateShapeSignature,
-            allFilesExcludingDefaultLibraryFile: undefined,
-            allFileNames: undefined
+            hasCalledUpdateShapeSignature
+        };
+    }
+
+    /**
+     * Releases needed properties
+     */
+    export function releaseCache(state: BuilderState) {
+        state.allFilesExcludingDefaultLibraryFile = undefined;
+        state.allFileNames = undefined;
+    }
+
+    /**
+     * Creates a clone of the state
+     */
+    export function clone(state: Readonly<BuilderState>): BuilderState {
+        const fileInfos = createMap<FileInfo>();
+        state.fileInfos.forEach((value, key) => {
+            fileInfos.set(key, { ...value });
+        });
+        // Dont need to backup allFiles info since its cache anyway
+        return {
+            fileInfos,
+            referencedMap: cloneMapOrUndefined(state.referencedMap),
+            exportedModulesMap: cloneMapOrUndefined(state.exportedModulesMap),
+            hasCalledUpdateShapeSignature: cloneMap(state.hasCalledUpdateShapeSignature),
         };
     }
 
@@ -505,14 +532,14 @@ namespace ts.BuilderState {
 
         // Start with the paths this file was referenced by
         seenFileNamesMap.set(sourceFileWithUpdatedShape.path, sourceFileWithUpdatedShape);
-        const queue = getReferencedByPaths(state, sourceFileWithUpdatedShape.path);
+        const queue = getReferencedByPaths(state, sourceFileWithUpdatedShape.resolvedPath);
         while (queue.length > 0) {
             const currentPath = queue.pop()!;
             if (!seenFileNamesMap.has(currentPath)) {
                 const currentSourceFile = programOfThisState.getSourceFileByPath(currentPath)!;
                 seenFileNamesMap.set(currentPath, currentSourceFile);
                 if (currentSourceFile && updateShapeSignature(state, programOfThisState, currentSourceFile, cacheToUpdateSignature, cancellationToken, computeHash!, exportedModulesMapCache)) { // TODO: GH#18217
-                    queue.push(...getReferencedByPaths(state, currentPath));
+                    queue.push(...getReferencedByPaths(state, currentSourceFile.resolvedPath));
                 }
             }
         }

src/compiler/core.ts

@@ -1387,6 +1387,18 @@ namespace ts {
         return result;
     }
 
+    export function copyProperties<T1 extends T2, T2>(first: T1, second: T2) {
+        for (const id in second) {
+            if (hasOwnProperty.call(second, id)) {
+                (first as any)[id] = second[id];
+            }
+        }
+    }
+
+    export function maybeBind<T, A extends any[], R>(obj: T, fn: ((this: T, ...args: A) => R) | undefined): ((...args: A) => R) | undefined {
+        return fn ? fn.bind(obj) : undefined;
+    }
+
     export interface MultiMap<T> extends Map<T[]> {
         /**
          * Adds the value to an array of values associated with the key, and returns the array.

src/compiler/diagnosticMessages.json

@@ -3957,6 +3957,10 @@
         "category": "Error",
         "code": 6370
     },
+    "Updating unchanged output timestamps of project '{0}'...": {
+        "category": "Message",
+        "code": 6371
+    },
 
     "The expected type comes from property '{0}' which is declared here on type '{1}'": {
         "category": "Message",

src/compiler/program.ts

@@ -69,6 +69,7 @@ namespace ts {
     export function createCompilerHost(options: CompilerOptions, setParentNodes?: boolean): CompilerHost {
         return createCompilerHostWorker(options, setParentNodes);
     }
+
     /*@internal*/
     // TODO(shkamat): update this after reworking ts build API
     export function createCompilerHostWorker(options: CompilerOptions, setParentNodes?: boolean, system = sys): CompilerHost {
@@ -93,7 +94,6 @@ namespace ts {
                 }
                 text = "";
             }
-
             return text !== undefined ? createSourceFile(fileName, text, languageVersion, setParentNodes) : undefined;
         }
 
@@ -203,18 +203,25 @@ namespace ts {
         return compilerHost;
     }
 
+    interface CompilerHostLikeForCache {
+        fileExists(fileName: string): boolean;
+        readFile(fileName: string, encoding?: string): string | undefined;
+        directoryExists?(directory: string): boolean;
+        createDirectory?(directory: string): void;
+        writeFile?: WriteFileCallback;
+    }
+
     /*@internal*/
-    export function changeCompilerHostToUseCache(
-        host: CompilerHost,
+    export function changeCompilerHostLikeToUseCache(
+        host: CompilerHostLikeForCache,
         toPath: (fileName: string) => Path,
-        useCacheForSourceFile: boolean
+        getSourceFile?: CompilerHost["getSourceFile"]
     ) {
         const originalReadFile = host.readFile;
         const originalFileExists = host.fileExists;
         const originalDirectoryExists = host.directoryExists;
         const originalCreateDirectory = host.createDirectory;
         const originalWriteFile = host.writeFile;
-        const originalGetSourceFile = host.getSourceFile;
         const readFileCache = createMap<string | false>();
         const fileExistsCache = createMap<boolean>();
         const directoryExistsCache = createMap<boolean>();
@@ -242,19 +249,17 @@ namespace ts {
             return setReadFileCache(key, fileName);
         };
 
-        if (useCacheForSourceFile) {
-            host.getSourceFile = (fileName, languageVersion, onError, shouldCreateNewSourceFile) => {
-                const key = toPath(fileName);
-                const value = sourceFileCache.get(key);
-                if (value) return value;
+        const getSourceFileWithCache: CompilerHost["getSourceFile"] | undefined = getSourceFile ? (fileName, languageVersion, onError, shouldCreateNewSourceFile) => {
+            const key = toPath(fileName);
+            const value = sourceFileCache.get(key);
+            if (value) return value;
 
-                const sourceFile = originalGetSourceFile.call(host, fileName, languageVersion, onError, shouldCreateNewSourceFile);
-                if (sourceFile && (isDeclarationFileName(fileName) || fileExtensionIs(fileName, Extension.Json))) {
-                    sourceFileCache.set(key, sourceFile);
-                }
-                return sourceFile;
-            };
-        }
+            const sourceFile = getSourceFile(fileName, languageVersion, onError, shouldCreateNewSourceFile);
+            if (sourceFile && (isDeclarationFileName(fileName) || fileExtensionIs(fileName, Extension.Json))) {
+                sourceFileCache.set(key, sourceFile);
+            }
+            return sourceFile;
+        } : undefined;
 
         // fileExists for any kind of extension
         host.fileExists = fileName => {
@@ -265,23 +270,25 @@ namespace ts {
             fileExistsCache.set(key, !!newValue);
             return newValue;
         };
-        host.writeFile = (fileName, data, writeByteOrderMark, onError, sourceFiles) => {
-            const key = toPath(fileName);
-            fileExistsCache.delete(key);
+        if (originalWriteFile) {
+            host.writeFile = (fileName, data, writeByteOrderMark, onError, sourceFiles) => {
+                const key = toPath(fileName);
+                fileExistsCache.delete(key);
 
-            const value = readFileCache.get(key);
-            if (value && value !== data) {
-                readFileCache.delete(key);
-                sourceFileCache.delete(key);
-            }
-            else if (useCacheForSourceFile) {
-                const sourceFile = sourceFileCache.get(key);
-                if (sourceFile && sourceFile.text !== data) {
+                const value = readFileCache.get(key);
+                if (value && value !== data) {
+                    readFileCache.delete(key);
                     sourceFileCache.delete(key);
                 }
-            }
-            originalWriteFile.call(host, fileName, data, writeByteOrderMark, onError, sourceFiles);
-        };
+                else if (getSourceFileWithCache) {
+                    const sourceFile = sourceFileCache.get(key);
+                    if (sourceFile && sourceFile.text !== data) {
+                        sourceFileCache.delete(key);
+                    }
+                }
+                originalWriteFile.call(host, fileName, data, writeByteOrderMark, onError, sourceFiles);
+            };
+        }
 
         // directoryExists
         if (originalDirectoryExists && originalCreateDirectory) {
@@ -306,7 +313,7 @@ namespace ts {
             originalDirectoryExists,
             originalCreateDirectory,
             originalWriteFile,
-            originalGetSourceFile,
+            getSourceFileWithCache,
             readFileWithCache
         };
     }
@@ -735,7 +742,7 @@ namespace ts {
         performance.mark("beforeProgram");
 
         const host = createProgramOptions.host || createCompilerHost(options);
-        const configParsingHost = parseConfigHostFromCompilerHost(host);
+        const configParsingHost = parseConfigHostFromCompilerHostLike(host);
 
         let skipDefaultLib = options.noLib;
         const getDefaultLibraryFileName = memoize(() => host.getDefaultLibFileName(options));
@@ -3104,18 +3111,28 @@ namespace ts {
         }
     }
 
+    interface CompilerHostLike {
+        useCaseSensitiveFileNames(): boolean;
+        getCurrentDirectory(): string;
+        fileExists(fileName: string): boolean;
+        readFile(fileName: string): string | undefined;
+        readDirectory?(rootDir: string, extensions: ReadonlyArray<string>, excludes: ReadonlyArray<string> | undefined, includes: ReadonlyArray<string>, depth?: number): string[];
+        trace?(s: string): void;
+        onUnRecoverableConfigFileDiagnostic?: DiagnosticReporter;
+    }
+
     /* @internal */
-    export function parseConfigHostFromCompilerHost(host: CompilerHost): ParseConfigFileHost {
+    export function parseConfigHostFromCompilerHostLike(host: CompilerHostLike, directoryStructureHost: DirectoryStructureHost = host): ParseConfigFileHost {
         return {
-            fileExists: f => host.fileExists(f),
+            fileExists: f => directoryStructureHost.fileExists(f),
             readDirectory(root, extensions, excludes, includes, depth) {
-                Debug.assertDefined(host.readDirectory, "'CompilerHost.readDirectory' must be implemented to correctly process 'projectReferences'");
-                return host.readDirectory!(root, extensions, excludes, includes, depth);
+                Debug.assertDefined(directoryStructureHost.readDirectory, "'CompilerHost.readDirectory' must be implemented to correctly process 'projectReferences'");
+                return directoryStructureHost.readDirectory!(root, extensions, excludes, includes, depth);
             },
-            readFile: f => host.readFile(f),
+            readFile: f => directoryStructureHost.readFile(f),
             useCaseSensitiveFileNames: host.useCaseSensitiveFileNames(),
             getCurrentDirectory: () => host.getCurrentDirectory(),
-            onUnRecoverableConfigFileDiagnostic: () => undefined,
+            onUnRecoverableConfigFileDiagnostic: host.onUnRecoverableConfigFileDiagnostic || (() => undefined),
             trace: host.trace ? (s) => host.trace!(s) : undefined
         };
     }

src/compiler/sys.ts

@@ -2,6 +2,16 @@ declare function setTimeout(handler: (...args: any[]) => void, timeout: number):
 declare function clearTimeout(handle: any): void;
 
 namespace ts {
+    /**
+     * djb2 hashing algorithm
+     * http://www.cse.yorku.ca/~oz/hash.html
+     */
+    /* @internal */
+    export function generateDjb2Hash(data: string): string {
+        const chars = data.split("").map(str => str.charCodeAt(0));
+        return `${chars.reduce((prev, curr) => ((prev << 5) + prev) + curr, 5381)}`;
+    }
+
     /**
      * Set a high stack trace limit to provide more information in case of an error.
      * Called for command-line and server use cases.
@@ -1115,15 +1125,6 @@ namespace ts {
                 }
             }
 
-            /**
-             * djb2 hashing algorithm
-             * http://www.cse.yorku.ca/~oz/hash.html
-             */
-            function generateDjb2Hash(data: string): string {
-                const chars = data.split("").map(str => str.charCodeAt(0));
-                return `${chars.reduce((prev, curr) => ((prev << 5) + prev) + curr, 5381)}`;
-            }
-
             function createMD5HashUsingNativeCrypto(data: string): string {
                 const hash = _crypto!.createHash("md5");
                 hash.update(data);

src/compiler/tsbuild.ts

@@ -119,7 +119,7 @@ namespace ts {
             newestDeclarationFileContentChangedTime?: Date;
             newestOutputFileTime?: Date;
             newestOutputFileName?: string;
-            oldestOutputFileName?: string;
+            oldestOutputFileName: string;
         }
 
         /**
@@ -321,7 +321,7 @@ namespace ts {
         return fileExtensionIs(fileName, Extension.Dts);
     }
 
-    export interface SolutionBuilderHostBase extends CompilerHost {
+    export interface SolutionBuilderHostBase<T extends BuilderProgram> extends ProgramHost<T> {
         getModifiedTime(fileName: string): Date | undefined;
         setModifiedTime(fileName: string, date: Date): void;
         deleteFile(fileName: string): void;
@@ -331,15 +331,17 @@ namespace ts {
 
         // TODO: To do better with watch mode and normal build mode api that creates program and emits files
         // This currently helps enable --diagnostics and --extendedDiagnostics
-        beforeCreateProgram?(options: CompilerOptions): void;
-        afterProgramEmitAndDiagnostics?(program: Program): void;
+        afterProgramEmitAndDiagnostics?(program: T): void;
+
+        // For testing
+        now?(): Date;
     }
 
-    export interface SolutionBuilderHost extends SolutionBuilderHostBase {
+    export interface SolutionBuilderHost<T extends BuilderProgram> extends SolutionBuilderHostBase<T> {
         reportErrorSummary?: ReportEmitErrorSummary;
     }
 
-    export interface SolutionBuilderWithWatchHost extends SolutionBuilderHostBase, WatchHost {
+    export interface SolutionBuilderWithWatchHost<T extends BuilderProgram> extends SolutionBuilderHostBase<T>, WatchHost {
     }
 
     export interface SolutionBuilder {
@@ -372,8 +374,8 @@ namespace ts {
         };
     }
 
-    function createSolutionBuilderHostBase(system = sys, reportDiagnostic?: DiagnosticReporter, reportSolutionBuilderStatus?: DiagnosticReporter) {
-        const host = createCompilerHostWorker({}, /*setParentNodes*/ undefined, system) as SolutionBuilderHostBase;
+    function createSolutionBuilderHostBase<T extends BuilderProgram>(system: System, createProgram: CreateProgram<T>, reportDiagnostic?: DiagnosticReporter, reportSolutionBuilderStatus?: DiagnosticReporter) {
+        const host = createProgramHost(system, createProgram) as SolutionBuilderHostBase<T>;
         host.getModifiedTime = system.getModifiedTime ? path => system.getModifiedTime!(path) : () => undefined;
         host.setModifiedTime = system.setModifiedTime ? (path, date) => system.setModifiedTime!(path, date) : noop;
         host.deleteFile = system.deleteFile ? path => system.deleteFile!(path) : noop;
@@ -382,20 +384,16 @@ namespace ts {
         return host;
     }
 
-    export function createSolutionBuilderHost(system = sys, reportDiagnostic?: DiagnosticReporter, reportSolutionBuilderStatus?: DiagnosticReporter, reportErrorSummary?: ReportEmitErrorSummary) {
-        const host = createSolutionBuilderHostBase(system, reportDiagnostic, reportSolutionBuilderStatus) as SolutionBuilderHost;
+    export function createSolutionBuilderHost<T extends BuilderProgram = BuilderProgram>(system = sys, createProgram?: CreateProgram<T>, reportDiagnostic?: DiagnosticReporter, reportSolutionBuilderStatus?: DiagnosticReporter, reportErrorSummary?: ReportEmitErrorSummary) {
+        const host = createSolutionBuilderHostBase(system, createProgram || createAbstractBuilder as any as CreateProgram<T>, reportDiagnostic, reportSolutionBuilderStatus) as SolutionBuilderHost<T>;
         host.reportErrorSummary = reportErrorSummary;
         return host;
     }
 
-    export function createSolutionBuilderWithWatchHost(system?: System, reportDiagnostic?: DiagnosticReporter, reportSolutionBuilderStatus?: DiagnosticReporter, reportWatchStatus?: WatchStatusReporter) {
-        const host = createSolutionBuilderHostBase(system, reportDiagnostic, reportSolutionBuilderStatus) as SolutionBuilderWithWatchHost;
+    export function createSolutionBuilderWithWatchHost<T extends BuilderProgram = SemanticDiagnosticsBuilderProgram>(system = sys, createProgram?: CreateProgram<T>, reportDiagnostic?: DiagnosticReporter, reportSolutionBuilderStatus?: DiagnosticReporter, reportWatchStatus?: WatchStatusReporter) {
+        const host = createSolutionBuilderHostBase(system, createProgram || createEmitAndSemanticDiagnosticsBuilderProgram as any as CreateProgram<T>, reportDiagnostic, reportSolutionBuilderStatus) as SolutionBuilderWithWatchHost<T>;
         const watchHost = createWatchHost(system, reportWatchStatus);
-        host.onWatchStatusChange = watchHost.onWatchStatusChange;
-        host.watchFile = watchHost.watchFile;
-        host.watchDirectory = watchHost.watchDirectory;
-        host.setTimeout = watchHost.setTimeout;
-        host.clearTimeout = watchHost.clearTimeout;
+        copyProperties(host, watchHost);
         return host;
     }
 
@@ -413,13 +411,13 @@ namespace ts {
      * TODO: use SolutionBuilderWithWatchHost => watchedSolution
      *  use SolutionBuilderHost => Solution
      */
-    export function createSolutionBuilder(host: SolutionBuilderHost, rootNames: ReadonlyArray<string>, defaultOptions: BuildOptions): SolutionBuilder;
-    export function createSolutionBuilder(host: SolutionBuilderWithWatchHost, rootNames: ReadonlyArray<string>, defaultOptions: BuildOptions): SolutionBuilderWithWatch;
-    export function createSolutionBuilder(host: SolutionBuilderHost | SolutionBuilderWithWatchHost, rootNames: ReadonlyArray<string>, defaultOptions: BuildOptions): SolutionBuilderWithWatch {
-        const hostWithWatch = host as SolutionBuilderWithWatchHost;
+    export function createSolutionBuilder<T extends BuilderProgram>(host: SolutionBuilderHost<T>, rootNames: ReadonlyArray<string>, defaultOptions: BuildOptions): SolutionBuilder;
+    export function createSolutionBuilder<T extends BuilderProgram>(host: SolutionBuilderWithWatchHost<T>, rootNames: ReadonlyArray<string>, defaultOptions: BuildOptions): SolutionBuilderWithWatch;
+    export function createSolutionBuilder<T extends BuilderProgram>(host: SolutionBuilderHost<T> | SolutionBuilderWithWatchHost<T>, rootNames: ReadonlyArray<string>, defaultOptions: BuildOptions): SolutionBuilderWithWatch {
+        const hostWithWatch = host as SolutionBuilderWithWatchHost<T>;
         const currentDirectory = host.getCurrentDirectory();
         const getCanonicalFileName = createGetCanonicalFileName(host.useCaseSensitiveFileNames());
-        const parseConfigFileHost = parseConfigHostFromCompilerHost(host);
+        const parseConfigFileHost = parseConfigHostFromCompilerHostLike(host);
 
         // State of the solution
         let options = defaultOptions;
@@ -434,8 +432,14 @@ namespace ts {
         let globalDependencyGraph: DependencyGraph | undefined;
         const writeFileName = (s: string) => host.trace && host.trace(s);
         let readFileWithCache = (f: string) => host.readFile(f);
+        let projectCompilerOptions = baseCompilerOptions;
+        const compilerHost = createCompilerHostFromProgramHost(host, () => projectCompilerOptions);
+        const originalGetSourceFile = compilerHost.getSourceFile;
+        const computeHash = host.createHash || generateDjb2Hash;
+        updateGetSourceFile();
 
         // Watch state
+        const builderPrograms = createFileMap<T>(toPath);
         const diagnostics = createFileMap<ReadonlyArray<Diagnostic>>(toPath);
         const projectPendingBuild = createFileMap<ConfigFileProgramReloadLevel>(toPath);
         const projectErrorsReported = createFileMap<true>(toPath);
@@ -443,6 +447,7 @@ namespace ts {
         let nextProjectToBuild = 0;
         let timerToBuildInvalidatedProject: any;
         let reportFileChangeDetected = false;
+        const { watchFile, watchFilePath, watchDirectory, writeLog } = createWatchFactory<ResolvedConfigFileName>(host, options);
 
         // Watches for the solution
         const allWatchedWildcardDirectories = createFileMap<Map<WildcardDirectoryWatcher>>(toPath);
@@ -492,6 +497,27 @@ namespace ts {
             clearMap(allWatchedWildcardDirectories, wildCardWatches => clearMap(wildCardWatches, closeFileWatcherOf));
             clearMap(allWatchedInputFiles, inputFileWatches => clearMap(inputFileWatches, closeFileWatcher));
             clearMap(allWatchedConfigFiles, closeFileWatcher);
+            builderPrograms.clear();
+            updateGetSourceFile();
+        }
+
+        function updateGetSourceFile() {
+            if (options.watch) {
+                if (compilerHost.getSourceFile === originalGetSourceFile) {
+                    compilerHost.getSourceFile = (...args) => {
+                        const result = originalGetSourceFile.call(compilerHost, ...args);
+                        if (result && options.watch) {
+                            result.version = computeHash.call(host, result.text);
+                        }
+                        return result;
+                    };
+                }
+            }
+            else {
+                if (compilerHost.getSourceFile !== originalGetSourceFile) {
+                    compilerHost.getSourceFile = originalGetSourceFile;
+                }
+            }
         }
 
         function isParsedCommandLine(entry: ConfigFileCacheEntry): entry is ParsedCommandLine {
@@ -542,9 +568,16 @@ namespace ts {
 
         function watchConfigFile(resolved: ResolvedConfigFileName) {
             if (options.watch && !allWatchedConfigFiles.hasKey(resolved)) {
-                allWatchedConfigFiles.setValue(resolved, hostWithWatch.watchFile(resolved, () => {
-                    invalidateProjectAndScheduleBuilds(resolved, ConfigFileProgramReloadLevel.Full);
-                }));
+                allWatchedConfigFiles.setValue(resolved, watchFile(
+                    hostWithWatch,
+                    resolved,
+                    () => {
+                        invalidateProjectAndScheduleBuilds(resolved, ConfigFileProgramReloadLevel.Full);
+                    },
+                    PollingInterval.High,
+                    WatchType.ConfigFile,
+                    resolved
+                ));
             }
         }
 
@@ -554,20 +587,27 @@ namespace ts {
                 getOrCreateValueMapFromConfigFileMap(allWatchedWildcardDirectories, resolved),
                 createMapFromTemplate(parsed.configFileSpecs!.wildcardDirectories),
                 (dir, flags) => {
-                    return hostWithWatch.watchDirectory(dir, fileOrDirectory => {
-                        const fileOrDirectoryPath = toPath(fileOrDirectory);
-                        if (fileOrDirectoryPath !== toPath(dir) && hasExtension(fileOrDirectoryPath) && !isSupportedSourceFileName(fileOrDirectory, parsed.options)) {
-                            // writeLog(`Project: ${configFileName} Detected file add/remove of non supported extension: ${fileOrDirectory}`);
-                            return;
-                        }
+                    return watchDirectory(
+                        hostWithWatch,
+                        dir,
+                        fileOrDirectory => {
+                            const fileOrDirectoryPath = toPath(fileOrDirectory);
+                            if (fileOrDirectoryPath !== toPath(dir) && hasExtension(fileOrDirectoryPath) && !isSupportedSourceFileName(fileOrDirectory, parsed.options)) {
+                                writeLog(`Project: ${resolved} Detected file add/remove of non supported extension: ${fileOrDirectory}`);
+                                return;
+                            }
 
-                        if (isOutputFile(fileOrDirectory, parsed)) {
-                            // writeLog(`${fileOrDirectory} is output file`);
-                            return;
-                        }
+                            if (isOutputFile(fileOrDirectory, parsed)) {
+                                writeLog(`${fileOrDirectory} is output file`);
+                                return;
+                            }
 
-                        invalidateProjectAndScheduleBuilds(resolved, ConfigFileProgramReloadLevel.Partial);
-                    }, !!(flags & WatchDirectoryFlags.Recursive));
+                            invalidateProjectAndScheduleBuilds(resolved, ConfigFileProgramReloadLevel.Partial);
+                        },
+                        flags,
+                        WatchType.WildcardDirectory,
+                        resolved
+                    );
                 }
             );
         }
@@ -578,9 +618,15 @@ namespace ts {
                 getOrCreateValueMapFromConfigFileMap(allWatchedInputFiles, resolved),
                 arrayToMap(parsed.fileNames, toPath),
                 {
-                    createNewValue: (_key, input) => hostWithWatch.watchFile(input, () => {
-                        invalidateProjectAndScheduleBuilds(resolved, ConfigFileProgramReloadLevel.None);
-                    }),
+                    createNewValue: (path, input) => watchFilePath(
+                        hostWithWatch,
+                        input,
+                        () => invalidateProjectAndScheduleBuilds(resolved, ConfigFileProgramReloadLevel.None),
+                        PollingInterval.Low,
+                        path as Path,
+                        WatchType.SourceFile,
+                        resolved
+                    ),
                     onDeleteValue: closeFileWatcher,
                 }
             );
@@ -898,7 +944,7 @@ namespace ts {
         }
 
         function reportErrorSummary() {
-            if (options.watch || (host as SolutionBuilderHost).reportErrorSummary) {
+            if (options.watch || (host as SolutionBuilderHost<T>).reportErrorSummary) {
                 // Report errors from the other projects
                 getGlobalDependencyGraph().buildQueue.forEach(project => {
                     if (!projectErrorsReported.hasKey(project)) {
@@ -911,7 +957,7 @@ namespace ts {
                     reportWatchStatus(getWatchErrorSummaryDiagnosticMessage(totalErrors), totalErrors);
                 }
                 else {
-                    (host as SolutionBuilderHost).reportErrorSummary!(totalErrors);
+                    (host as SolutionBuilderHost<T>).reportErrorSummary!(totalErrors);
                 }
             }
         }
@@ -944,16 +990,40 @@ namespace ts {
                 return;
             }
 
+            if (status.type === UpToDateStatusType.UpToDateWithUpstreamTypes) {
+                // Fake that files have been built by updating output file stamps
+                updateOutputTimestamps(proj);
+                return;
+            }
+
             const buildResult = buildSingleProject(resolved);
-            const dependencyGraph = getGlobalDependencyGraph();
-            const referencingProjects = dependencyGraph.referencingProjectsMap.getValue(resolved);
+            if (buildResult & BuildResultFlags.AnyErrors) return;
+
+            const { referencingProjectsMap, buildQueue } = getGlobalDependencyGraph();
+            const referencingProjects = referencingProjectsMap.getValue(resolved);
             if (!referencingProjects) return;
+
             // Always use build order to queue projects
-            for (const project of dependencyGraph.buildQueue) {
+            for (let index = buildQueue.indexOf(resolved) + 1; index < buildQueue.length; index++) {
+                const project = buildQueue[index];
                 const prepend = referencingProjects.getValue(project);
-                // If the project is referenced with prepend, always build downstream projectm,
-                // otherwise queue it only if declaration output changed
-                if (prepend || (prepend !== undefined && !(buildResult & BuildResultFlags.DeclarationOutputUnchanged))) {
+                if (prepend !== undefined) {
+                    // If the project is referenced with prepend, always build downstream projects,
+                    // If declaration output is changed, build the project
+                    // otherwise mark the project UpToDateWithUpstreamTypes so it updates output time stamps
+                    const status = projectStatus.getValue(project);
+                    if (prepend || !(buildResult & BuildResultFlags.DeclarationOutputUnchanged)) {
+                        if (status && (status.type === UpToDateStatusType.UpToDate || status.type === UpToDateStatusType.UpToDateWithUpstreamTypes)) {
+                            projectStatus.setValue(project, {
+                                type: UpToDateStatusType.OutOfDateWithUpstream,
+                                outOfDateOutputFileName: status.oldestOutputFileName,
+                                newerProjectName: resolved
+                            });
+                        }
+                    }
+                    else if (status && status.type === UpToDateStatusType.UpToDate) {
+                        status.type = UpToDateStatusType.UpToDateWithUpstreamTypes;
+                    }
                     addProjToQueue(project);
                 }
             }
@@ -1030,22 +1100,23 @@ namespace ts {
                 return BuildResultFlags.None;
             }
 
-            const programOptions: CreateProgramOptions = {
-                projectReferences: configFile.projectReferences,
-                host,
-                rootNames: configFile.fileNames,
-                options: configFile.options,
-                configFileParsingDiagnostics: configFile.errors
-            };
-            if (host.beforeCreateProgram) {
-                host.beforeCreateProgram(options);
-            }
-            const program = createProgram(programOptions);
+            // TODO: handle resolve module name to cache result in project reference redirect
+            projectCompilerOptions = configFile.options;
+            const program = host.createProgram(
+                configFile.fileNames,
+                configFile.options,
+                compilerHost,
+                builderPrograms.getValue(proj),
+                configFile.errors,
+                configFile.projectReferences
+            );
+            projectCompilerOptions = baseCompilerOptions;
 
             // Don't emit anything in the presence of syntactic errors or options diagnostics
             const syntaxDiagnostics = [
-                ...program.getOptionsDiagnostics(),
                 ...program.getConfigFileParsingDiagnostics(),
+                ...program.getOptionsDiagnostics(),
+                ...program.getGlobalDiagnostics(),
                 ...program.getSyntacticDiagnostics()];
             if (syntaxDiagnostics.length) {
                 return buildErrors(syntaxDiagnostics, BuildResultFlags.SyntaxErrors, "Syntactic");
@@ -1057,6 +1128,8 @@ namespace ts {
                 return buildErrors(semanticDiagnostics, BuildResultFlags.TypeErrors, "Semantic");
             }
 
+            // Before emitting lets backup state, so we can revert it back if there are declaration errors to handle emit and declaration errors correctly
+            program.backupState();
             let newestDeclarationFileContentChangedTime = minimumDate;
             let anyDtsChanged = false;
             let declDiagnostics: Diagnostic[] | undefined;
@@ -1065,11 +1138,13 @@ namespace ts {
             emitFilesAndReportErrors(program, reportDeclarationDiagnostics, writeFileName, /*reportSummary*/ undefined, (name, text, writeByteOrderMark) => outputFiles.push({ name, text, writeByteOrderMark }));
             // Don't emit .d.ts if there are decl file errors
             if (declDiagnostics) {
+                program.restoreState();
                 return buildErrors(declDiagnostics, BuildResultFlags.DeclarationEmitErrors, "Declaration file");
             }
 
             // Actual Emit
             const emitterDiagnostics = createDiagnosticCollection();
+            const emittedOutputs = createFileMap<true>(toPath as ToPath);
             outputFiles.forEach(({ name, text, writeByteOrderMark }) => {
                 let priorChangeTime: Date | undefined;
                 if (!anyDtsChanged && isDeclarationFile(name)) {
@@ -1083,7 +1158,8 @@ namespace ts {
                     }
                 }
 
-                writeFile(host, emitterDiagnostics, name, text, writeByteOrderMark);
+                emittedOutputs.setValue(name, true);
+                writeFile(compilerHost, emitterDiagnostics, name, text, writeByteOrderMark);
                 if (priorChangeTime !== undefined) {
                     newestDeclarationFileContentChangedTime = newer(priorChangeTime, newestDeclarationFileContentChangedTime);
                     unchangedOutputs.setValue(name, priorChangeTime);
@@ -1095,51 +1171,72 @@ namespace ts {
                 return buildErrors(emitDiagnostics, BuildResultFlags.EmitErrors, "Emit");
             }
 
+            // Update time stamps for rest of the outputs
+            newestDeclarationFileContentChangedTime = updateOutputTimestampsWorker(configFile, newestDeclarationFileContentChangedTime, Diagnostics.Updating_unchanged_output_timestamps_of_project_0, emittedOutputs);
+
             const status: UpToDateStatus = {
                 type: UpToDateStatusType.UpToDate,
-                newestDeclarationFileContentChangedTime: anyDtsChanged ? maximumDate : newestDeclarationFileContentChangedTime
+                newestDeclarationFileContentChangedTime: anyDtsChanged ? maximumDate : newestDeclarationFileContentChangedTime,
+                oldestOutputFileName: outputFiles.length ? outputFiles[0].name : getFirstProjectOutput(configFile)
             };
             diagnostics.removeKey(proj);
             projectStatus.setValue(proj, status);
-            if (host.afterProgramEmitAndDiagnostics) {
-                host.afterProgramEmitAndDiagnostics(program);
-            }
+            afterProgramCreate(proj, program);
             return resultFlags;
 
             function buildErrors(diagnostics: ReadonlyArray<Diagnostic>, errorFlags: BuildResultFlags, errorType: string) {
                 resultFlags |= errorFlags;
                 reportAndStoreErrors(proj, diagnostics);
                 projectStatus.setValue(proj, { type: UpToDateStatusType.Unbuildable, reason: `${errorType} errors` });
-                if (host.afterProgramEmitAndDiagnostics) {
-                    host.afterProgramEmitAndDiagnostics(program);
-                }
+                afterProgramCreate(proj, program);
                 return resultFlags;
             }
         }
 
+        function afterProgramCreate(proj: ResolvedConfigFileName, program: T) {
+            if (host.afterProgramEmitAndDiagnostics) {
+                host.afterProgramEmitAndDiagnostics(program);
+            }
+            if (options.watch) {
+                program.releaseProgram();
+                builderPrograms.setValue(proj, program);
+            }
+        }
+
         function updateOutputTimestamps(proj: ParsedCommandLine) {
             if (options.dry) {
                 return reportStatus(Diagnostics.A_non_dry_build_would_build_project_0, proj.options.configFilePath!);
             }
-
-            if (options.verbose) {
-                reportStatus(Diagnostics.Updating_output_timestamps_of_project_0, proj.options.configFilePath!);
-            }
-
-            const now = new Date();
-            const outputs = getAllProjectOutputs(proj);
-            let priorNewestUpdateTime = minimumDate;
-            for (const file of outputs) {
-                if (isDeclarationFile(file)) {
-                    priorNewestUpdateTime = newer(priorNewestUpdateTime, host.getModifiedTime(file) || missingFileModifiedTime);
-                }
-
-                host.setModifiedTime(file, now);
-            }
-
+            const priorNewestUpdateTime = updateOutputTimestampsWorker(proj, minimumDate, Diagnostics.Updating_output_timestamps_of_project_0);
             projectStatus.setValue(proj.options.configFilePath as ResolvedConfigFilePath, { type: UpToDateStatusType.UpToDate, newestDeclarationFileContentChangedTime: priorNewestUpdateTime } as UpToDateStatus);
         }
 
+        function updateOutputTimestampsWorker(proj: ParsedCommandLine, priorNewestUpdateTime: Date, verboseMessage: DiagnosticMessage, skipOutputs?: FileMap<true>) {
+            const outputs = getAllProjectOutputs(proj);
+            if (!skipOutputs || outputs.length !== skipOutputs.getSize()) {
+                if (options.verbose) {
+                    reportStatus(verboseMessage, proj.options.configFilePath!);
+                }
+                const now = host.now ? host.now() : new Date();
+                for (const file of outputs) {
+                    if (skipOutputs && skipOutputs.hasKey(file)) {
+                        continue;
+                    }
+
+                    if (isDeclarationFile(file)) {
+                        priorNewestUpdateTime = newer(priorNewestUpdateTime, host.getModifiedTime(file) || missingFileModifiedTime);
+                    }
+
+                    host.setModifiedTime(file, now);
+                    if (proj.options.listEmittedFiles) {
+                        writeFileName(`TSFILE: ${file}`);
+                    }
+                }
+            }
+
+            return priorNewestUpdateTime;
+        }
+
         function getFilesToClean(): string[] {
             // Get the same graph for cleaning we'd use for building
             const graph = getGlobalDependencyGraph();
@@ -1187,12 +1284,15 @@ namespace ts {
             if (options.watch) { reportWatchStatus(Diagnostics.Starting_compilation_in_watch_mode); }
             // TODO:: In watch mode as well to use caches for incremental build once we can invalidate caches correctly and have right api
             // Override readFile for json files and output .d.ts to cache the text
-            const { originalReadFile, originalFileExists, originalDirectoryExists,
-                originalCreateDirectory, originalWriteFile, originalGetSourceFile,
-                readFileWithCache: newReadFileWithCache
-            } = changeCompilerHostToUseCache(host, toPath, /*useCacheForSourceFile*/ true);
             const savedReadFileWithCache = readFileWithCache;
+            const savedGetSourceFile = compilerHost.getSourceFile;
+
+            const { originalReadFile, originalFileExists, originalDirectoryExists,
+                originalCreateDirectory, originalWriteFile, getSourceFileWithCache,
+                readFileWithCache: newReadFileWithCache
+            } = changeCompilerHostLikeToUseCache(host, toPath, (...args) => savedGetSourceFile.call(compilerHost, ...args));
             readFileWithCache = newReadFileWithCache;
+            compilerHost.getSourceFile = getSourceFileWithCache!;
 
             const graph = getGlobalDependencyGraph();
             reportBuildQueue(graph);
@@ -1249,8 +1349,8 @@ namespace ts {
             host.directoryExists = originalDirectoryExists;
             host.createDirectory = originalCreateDirectory;
             host.writeFile = originalWriteFile;
+            compilerHost.getSourceFile = savedGetSourceFile;
             readFileWithCache = savedReadFileWithCache;
-            host.getSourceFile = originalGetSourceFile;
             return anyFailed ? ExitStatus.DiagnosticsPresent_OutputsSkipped : ExitStatus.Success;
         }
 
@@ -1278,7 +1378,7 @@ namespace ts {
         }
 
         function relName(path: string): string {
-            return convertToRelativePath(path, host.getCurrentDirectory(), f => host.getCanonicalFileName(f));
+            return convertToRelativePath(path, host.getCurrentDirectory(), f => compilerHost.getCanonicalFileName(f));
         }
 
         /**
@@ -1311,6 +1411,20 @@ namespace ts {
         }
     }
 
+    function getFirstProjectOutput(project: ParsedCommandLine): string {
+        if (project.options.outFile || project.options.out) {
+            return first(getOutFileOutputs(project));
+        }
+
+        for (const inputFile of project.fileNames) {
+            const outputs = getOutputFileNames(inputFile, project);
+            if (outputs.length) {
+                return first(outputs);
+            }
+        }
+        return Debug.fail(`project ${project.options.configFilePath} expected to have at least one output`);
+    }
+
     export function formatUpToDateStatus<T>(configFileName: string, status: UpToDateStatus, relName: (fileName: string) => string, formatMessage: (message: DiagnosticMessage, ...args: string[]) => T) {
         switch (status.type) {
             case UpToDateStatusType.OutOfDateWithSelf:

src/compiler/types.ts

@@ -2827,7 +2827,7 @@ namespace ts {
         fileName: string,
         data: string,
         writeByteOrderMark: boolean,
-        onError: ((message: string) => void) | undefined,
+        onError?: (message: string) => void,
         sourceFiles?: ReadonlyArray<SourceFile>,
     ) => void;
 
@@ -5002,7 +5002,6 @@ namespace ts {
         getDefaultLibLocation?(): string;
         writeFile: WriteFileCallback;
         getCurrentDirectory(): string;
-        getDirectories(path: string): string[];
         getCanonicalFileName(fileName: string): string;
         useCaseSensitiveFileNames(): boolean;
         getNewLine(): string;

src/compiler/watch.ts

@@ -88,21 +88,6 @@ namespace ts {
         return result;
     }
 
-    /**
-     * Program structure needed to emit the files and report diagnostics
-     */
-    export interface ProgramToEmitFilesAndReportErrors {
-        getCurrentDirectory(): string;
-        getCompilerOptions(): CompilerOptions;
-        getSourceFiles(): ReadonlyArray<SourceFile>;
-        getSyntacticDiagnostics(): ReadonlyArray<Diagnostic>;
-        getOptionsDiagnostics(): ReadonlyArray<Diagnostic>;
-        getGlobalDiagnostics(): ReadonlyArray<Diagnostic>;
-        getSemanticDiagnostics(): ReadonlyArray<Diagnostic>;
-        getConfigFileParsingDiagnostics(): ReadonlyArray<Diagnostic>;
-        emit(targetSourceFile?: SourceFile, writeFile?: WriteFileCallback): EmitResult;
-    }
-
     export type ReportEmitErrorSummary = (errorCount: number) => void;
 
     export function getErrorCountForSummary(diagnostics: ReadonlyArray<Diagnostic>) {
@@ -121,6 +106,21 @@ namespace ts {
         return `${newLine}${flattenDiagnosticMessageText(d.messageText, newLine)}${newLine}${newLine}`;
     }
 
+    /**
+     * Program structure needed to emit the files and report diagnostics
+     */
+    export interface ProgramToEmitFilesAndReportErrors {
+        getCurrentDirectory(): string;
+        getCompilerOptions(): CompilerOptions;
+        getSourceFiles(): ReadonlyArray<SourceFile>;
+        getSyntacticDiagnostics(): ReadonlyArray<Diagnostic>;
+        getOptionsDiagnostics(): ReadonlyArray<Diagnostic>;
+        getGlobalDiagnostics(): ReadonlyArray<Diagnostic>;
+        getSemanticDiagnostics(): ReadonlyArray<Diagnostic>;
+        getConfigFileParsingDiagnostics(): ReadonlyArray<Diagnostic>;
+        emit(targetSourceFile?: SourceFile, writeFile?: WriteFileCallback): EmitResult;
+    }
+
     /**
      * Helper that emit files, report diagnostics and lists emitted and/or source files depending on compiler options
      */
@@ -187,30 +187,110 @@ namespace ts {
         const onWatchStatusChange = reportWatchStatus || createWatchStatusReporter(system);
         return {
             onWatchStatusChange,
-            watchFile: system.watchFile ? ((path, callback, pollingInterval) => system.watchFile!(path, callback, pollingInterval)) : () => noopFileWatcher,
-            watchDirectory: system.watchDirectory ? ((path, callback, recursive) => system.watchDirectory!(path, callback, recursive)) : () => noopFileWatcher,
-            setTimeout: system.setTimeout ? ((callback, ms, ...args: any[]) => system.setTimeout!.call(system, callback, ms, ...args)) : noop,
-            clearTimeout: system.clearTimeout ? (timeoutId => system.clearTimeout!(timeoutId)) : noop
+            watchFile: maybeBind(system, system.watchFile) || (() => noopFileWatcher),
+            watchDirectory: maybeBind(system, system.watchDirectory) || (() => noopFileWatcher),
+            setTimeout: maybeBind(system, system.setTimeout) || noop,
+            clearTimeout: maybeBind(system, system.clearTimeout) || noop
         };
     }
 
+    export const enum WatchType {
+        ConfigFile = "Config file",
+        SourceFile = "Source file",
+        MissingFile = "Missing file",
+        WildcardDirectory = "Wild card directory",
+        FailedLookupLocations = "Failed Lookup Locations",
+        TypeRoots = "Type roots"
+    }
+
+    interface WatchFactory<X, Y = undefined> extends ts.WatchFactory<X, Y> {
+        writeLog: (s: string) => void;
+    }
+
+    export function createWatchFactory<Y = undefined>(host: { trace?(s: string): void; }, options: { extendedDiagnostics?: boolean; diagnostics?: boolean; }) {
+        const watchLogLevel = host.trace ? options.extendedDiagnostics ? WatchLogLevel.Verbose : options.diagnostics ? WatchLogLevel.TriggerOnly : WatchLogLevel.None : WatchLogLevel.None;
+        const writeLog: (s: string) => void = watchLogLevel !== WatchLogLevel.None ? (s => host.trace!(s)) : noop;
+        const result = getWatchFactory<WatchType, Y>(watchLogLevel, writeLog) as WatchFactory<WatchType, Y>;
+        result.writeLog = writeLog;
+        return result;
+    }
+
+    export function createCompilerHostFromProgramHost(host: ProgramHost<any>, getCompilerOptions: () => CompilerOptions, directoryStructureHost: DirectoryStructureHost = host): CompilerHost {
+        const useCaseSensitiveFileNames = host.useCaseSensitiveFileNames();
+        const hostGetNewLine = memoize(() => host.getNewLine());
+        return {
+            getSourceFile: (fileName, languageVersion, onError) => {
+                let text: string | undefined;
+                try {
+                    performance.mark("beforeIORead");
+                    text = host.readFile(fileName, getCompilerOptions().charset);
+                    performance.mark("afterIORead");
+                    performance.measure("I/O Read", "beforeIORead", "afterIORead");
+                }
+                catch (e) {
+                    if (onError) {
+                        onError(e.message);
+                    }
+                    text = "";
+                }
+
+                return text !== undefined ? createSourceFile(fileName, text, languageVersion) : undefined;
+            },
+            getDefaultLibLocation: maybeBind(host, host.getDefaultLibLocation),
+            getDefaultLibFileName: options => host.getDefaultLibFileName(options),
+            writeFile,
+            getCurrentDirectory: memoize(() => host.getCurrentDirectory()),
+            useCaseSensitiveFileNames: () => useCaseSensitiveFileNames,
+            getCanonicalFileName: createGetCanonicalFileName(useCaseSensitiveFileNames),
+            getNewLine: () => getNewLineCharacter(getCompilerOptions(), hostGetNewLine),
+            fileExists: f => host.fileExists(f),
+            readFile: f => host.readFile(f),
+            trace: maybeBind(host, host.trace),
+            directoryExists: maybeBind(directoryStructureHost, directoryStructureHost.directoryExists),
+            getDirectories: maybeBind(directoryStructureHost, directoryStructureHost.getDirectories),
+            realpath: maybeBind(host, host.realpath),
+            getEnvironmentVariable: maybeBind(host, host.getEnvironmentVariable) || (() => ""),
+            createHash: maybeBind(host, host.createHash),
+            readDirectory: maybeBind(host, host.readDirectory),
+        };
+
+        function ensureDirectoriesExist(directoryPath: string) {
+            if (directoryPath.length > getRootLength(directoryPath) && !host.directoryExists!(directoryPath)) {
+                const parentDirectory = getDirectoryPath(directoryPath);
+                ensureDirectoriesExist(parentDirectory);
+                if (host.createDirectory) host.createDirectory(directoryPath);
+            }
+        }
+
+        function writeFile(fileName: string, text: string, writeByteOrderMark: boolean, onError: (message: string) => void) {
+            try {
+                performance.mark("beforeIOWrite");
+                ensureDirectoriesExist(getDirectoryPath(normalizePath(fileName)));
+
+                host.writeFile!(fileName, text, writeByteOrderMark);
+
+                performance.mark("afterIOWrite");
+                performance.measure("I/O Write", "beforeIOWrite", "afterIOWrite");
+            }
+            catch (e) {
+                if (onError) {
+                    onError(e.message);
+                }
+            }
+        }
+    }
+
     /**
      * Creates the watch compiler host that can be extended with config file or root file names and options host
      */
-    function createWatchCompilerHost<T extends BuilderProgram = EmitAndSemanticDiagnosticsBuilderProgram>(system = sys, createProgram: CreateProgram<T> | undefined, reportDiagnostic: DiagnosticReporter, reportWatchStatus?: WatchStatusReporter): WatchCompilerHost<T> {
-        if (!createProgram) {
-            createProgram = createEmitAndSemanticDiagnosticsBuilderProgram as any as CreateProgram<T>;
-        }
-
+    export function createProgramHost<T extends BuilderProgram>(system: System, createProgram: CreateProgram<T>): ProgramHost<T> {
+        const getDefaultLibLocation = memoize(() => getDirectoryPath(normalizePath(system.getExecutingFilePath())));
         let host: DirectoryStructureHost = system;
         host; // tslint:disable-line no-unused-expression (TODO: `host` is unused!)
-        const useCaseSensitiveFileNames = () => system.useCaseSensitiveFileNames;
-        const writeFileName = (s: string) => system.write(s + system.newLine);
-        const { onWatchStatusChange, watchFile, watchDirectory, setTimeout, clearTimeout } = createWatchHost(system, reportWatchStatus);
         return {
-            useCaseSensitiveFileNames,
+            useCaseSensitiveFileNames: () => system.useCaseSensitiveFileNames,
             getNewLine: () => system.newLine,
-            getCurrentDirectory: () => system.getCurrentDirectory(),
+            getCurrentDirectory: memoize(() => system.getCurrentDirectory()),
             getDefaultLibLocation,
             getDefaultLibFileName: options => combinePaths(getDefaultLibLocation(), getDefaultLibFileName(options)),
             fileExists: path => system.fileExists(path),
@@ -218,27 +298,25 @@ namespace ts {
             directoryExists: path => system.directoryExists(path),
             getDirectories: path => system.getDirectories(path),
             readDirectory: (path, extensions, exclude, include, depth) => system.readDirectory(path, extensions, exclude, include, depth),
-            realpath: system.realpath && (path => system.realpath!(path)),
-            getEnvironmentVariable: system.getEnvironmentVariable && (name => system.getEnvironmentVariable(name)),
-            watchFile,
-            watchDirectory,
-            setTimeout,
-            clearTimeout,
-            trace: s => system.write(s),
-            onWatchStatusChange,
+            realpath: maybeBind(system, system.realpath),
+            getEnvironmentVariable: maybeBind(system, system.getEnvironmentVariable),
+            trace: s => system.write(s + system.newLine),
             createDirectory: path => system.createDirectory(path),
             writeFile: (path, data, writeByteOrderMark) => system.writeFile(path, data, writeByteOrderMark),
             onCachedDirectoryStructureHostCreate: cacheHost => host = cacheHost || system,
-            createHash: system.createHash && (s => system.createHash!(s)),
-            createProgram,
-            afterProgramCreate: emitFilesAndReportErrorUsingBuilder
+            createHash: maybeBind(system, system.createHash),
+            createProgram
         };
+    }
 
-        function getDefaultLibLocation() {
-            return getDirectoryPath(normalizePath(system.getExecutingFilePath()));
-        }
-
-        function emitFilesAndReportErrorUsingBuilder(builderProgram: BuilderProgram) {
+    /**
+     * Creates the watch compiler host that can be extended with config file or root file names and options host
+     */
+    function createWatchCompilerHost<T extends BuilderProgram = EmitAndSemanticDiagnosticsBuilderProgram>(system = sys, createProgram: CreateProgram<T> | undefined, reportDiagnostic: DiagnosticReporter, reportWatchStatus?: WatchStatusReporter): WatchCompilerHost<T> {
+        const writeFileName = (s: string) => system.write(s + system.newLine);
+        const result = createProgramHost(system, createProgram || createEmitAndSemanticDiagnosticsBuilderProgram as any as CreateProgram<T>) as WatchCompilerHost<T>;
+        copyProperties(result, createWatchHost(system, reportWatchStatus));
+        result.afterProgramCreate = builderProgram => {
             const compilerOptions = builderProgram.getCompilerOptions();
             const newLine = getNewLineCharacter(compilerOptions, () => system.newLine);
 
@@ -246,13 +324,14 @@ namespace ts {
                 builderProgram,
                 reportDiagnostic,
                 writeFileName,
-                errorCount => onWatchStatusChange!(
+                errorCount => result.onWatchStatusChange!(
                     createCompilerDiagnostic(getWatchErrorSummaryDiagnosticMessage(errorCount), errorCount),
                     newLine,
                     compilerOptions
                 )
             );
-        }
+        };
+        return result;
     }
 
     /**
@@ -291,6 +370,7 @@ namespace ts {
     export type WatchStatusReporter = (diagnostic: Diagnostic, newLine: string, options: CompilerOptions) => void;
     /** Create the program with rootNames and options, if they are undefined, oldProgram and new configFile diagnostics create new program */
     export type CreateProgram<T extends BuilderProgram> = (rootNames: ReadonlyArray<string> | undefined, options: CompilerOptions | undefined, host?: CompilerHost, oldProgram?: T, configFileParsingDiagnostics?: ReadonlyArray<Diagnostic>, projectReferences?: ReadonlyArray<ProjectReference> | undefined) => T;
+
     /** Host that has watch functionality used in --watch mode */
     export interface WatchHost {
         /** If provided, called with Diagnostic message that informs about change in watch status */
@@ -305,19 +385,11 @@ namespace ts {
         /** If provided, will be used to reset existing delayed compilation */
         clearTimeout?(timeoutId: any): void;
     }
-    export interface WatchCompilerHost<T extends BuilderProgram> extends WatchHost {
-        // TODO: GH#18217 Optional methods are frequently asserted
-
+    export interface ProgramHost<T extends BuilderProgram> {
         /**
          * Used to create the program when need for program creation or recreation detected
          */
         createProgram: CreateProgram<T>;
-        /** If provided, callback to invoke after every new program creation */
-        afterProgramCreate?(program: T): void;
-
-        // Only for testing
-        /*@internal*/
-        maxNumberOfFilesToIterateForInvalidation?: number;
 
         // Sub set of compiler host methods to read and generate new program
         useCaseSensitiveFileNames(): boolean;
@@ -357,16 +429,25 @@ namespace ts {
         /** If provided, used to resolve type reference directives, otherwise typescript's default resolution */
         resolveTypeReferenceDirectives?(typeReferenceDirectiveNames: string[], containingFile: string, redirectedReference?: ResolvedProjectReference): (ResolvedTypeReferenceDirective | undefined)[];
     }
-
     /** Internal interface used to wire emit through same host */
+
     /*@internal*/
-    export interface WatchCompilerHost<T extends BuilderProgram> {
+    export interface ProgramHost<T extends BuilderProgram> {
         // TODO: GH#18217 Optional methods are frequently asserted
         createDirectory?(path: string): void;
         writeFile?(path: string, data: string, writeByteOrderMark?: boolean): void;
         onCachedDirectoryStructureHostCreate?(host: CachedDirectoryStructureHost): void;
     }
 
+    export interface WatchCompilerHost<T extends BuilderProgram> extends ProgramHost<T>, WatchHost {
+        /** If provided, callback to invoke after every new program creation */
+        afterProgramCreate?(program: T): void;
+
+        // Only for testing
+        /*@internal*/
+        maxNumberOfFilesToIterateForInvalidation?: number;
+    }
+
     /**
      * Host to create watch with root files and options
      */
@@ -479,8 +560,6 @@ namespace ts {
 
         const useCaseSensitiveFileNames = host.useCaseSensitiveFileNames();
         const currentDirectory = host.getCurrentDirectory();
-        const getCurrentDirectory = () => currentDirectory;
-        const readFile: (path: string, encoding?: string) => string | undefined = (path, encoding) => host.readFile(path, encoding);
         const { configFileName, optionsToExtend: optionsToExtendForConfigFile = {}, createProgram } = host;
         let { rootFiles: rootFileNames, options: compilerOptions, projectReferences } = host;
         let configFileSpecs: ConfigFileSpecs;
@@ -493,15 +572,7 @@ namespace ts {
             host.onCachedDirectoryStructureHostCreate(cachedDirectoryStructureHost);
         }
         const directoryStructureHost: DirectoryStructureHost = cachedDirectoryStructureHost || host;
-        const parseConfigFileHost: ParseConfigFileHost = {
-            useCaseSensitiveFileNames,
-            readDirectory: (path, extensions, exclude, include, depth) => directoryStructureHost.readDirectory!(path, extensions, exclude, include, depth),
-            fileExists: path => host.fileExists(path),
-            readFile,
-            getCurrentDirectory,
-            onUnRecoverableConfigFileDiagnostic: host.onUnRecoverableConfigFileDiagnostic,
-            trace: host.trace ? s => host.trace!(s) : undefined
-        };
+        const parseConfigFileHost = parseConfigHostFromCompilerHostLike(host, directoryStructureHost);
 
         // From tsc we want to get already parsed result and hence check for rootFileNames
         let newLine = updateNewLine();
@@ -517,55 +588,37 @@ namespace ts {
             newLine = updateNewLine();
         }
 
-        const trace = host.trace && ((s: string) => { host.trace!(s + newLine); });
-        const watchLogLevel = trace ? compilerOptions.extendedDiagnostics ? WatchLogLevel.Verbose :
-            compilerOptions.diagnostics ? WatchLogLevel.TriggerOnly : WatchLogLevel.None : WatchLogLevel.None;
-        const writeLog: (s: string) => void = watchLogLevel !== WatchLogLevel.None ? trace! : noop; // TODO: GH#18217
-        const { watchFile, watchFilePath, watchDirectory } = getWatchFactory<string>(watchLogLevel, writeLog);
-
+        const { watchFile, watchFilePath, watchDirectory, writeLog } = createWatchFactory<string>(host, compilerOptions);
         const getCanonicalFileName = createGetCanonicalFileName(useCaseSensitiveFileNames);
 
         writeLog(`Current directory: ${currentDirectory} CaseSensitiveFileNames: ${useCaseSensitiveFileNames}`);
         if (configFileName) {
-            watchFile(host, configFileName, scheduleProgramReload, PollingInterval.High, "Config file");
+            watchFile(host, configFileName, scheduleProgramReload, PollingInterval.High, WatchType.ConfigFile);
         }
 
-        const compilerHost: CompilerHost & ResolutionCacheHost = {
-            // Members for CompilerHost
-            getSourceFile: (fileName, languageVersion, onError?, shouldCreateNewSourceFile?) => getVersionedSourceFileByPath(fileName, toPath(fileName), languageVersion, onError, shouldCreateNewSourceFile),
-            getSourceFileByPath: getVersionedSourceFileByPath,
-            getDefaultLibLocation: host.getDefaultLibLocation && (() => host.getDefaultLibLocation!()),
-            getDefaultLibFileName: options => host.getDefaultLibFileName(options),
-            writeFile,
-            getCurrentDirectory,
-            useCaseSensitiveFileNames: () => useCaseSensitiveFileNames,
-            getCanonicalFileName,
-            getNewLine: () => newLine,
-            fileExists,
-            readFile,
-            trace,
-            directoryExists: directoryStructureHost.directoryExists && (path => directoryStructureHost.directoryExists!(path)),
-            getDirectories: (directoryStructureHost.getDirectories && ((path: string) => directoryStructureHost.getDirectories!(path)))!, // TODO: GH#18217
-            realpath: host.realpath && (s => host.realpath!(s)),
-            getEnvironmentVariable: host.getEnvironmentVariable ? (name => host.getEnvironmentVariable!(name)) : (() => ""),
-            onReleaseOldSourceFile,
-            createHash: host.createHash && (data => host.createHash!(data)),
-            // Members for ResolutionCacheHost
-            toPath,
-            getCompilationSettings: () => compilerOptions,
-            watchDirectoryOfFailedLookupLocation: (dir, cb, flags) => watchDirectory(host, dir, cb, flags, "Failed Lookup Locations"),
-            watchTypeRootsDirectory: (dir, cb, flags) => watchDirectory(host, dir, cb, flags, "Type roots"),
-            getCachedDirectoryStructureHost: () => cachedDirectoryStructureHost,
-            onInvalidatedResolution: scheduleProgramUpdate,
-            onChangedAutomaticTypeDirectiveNames: () => {
-                hasChangedAutomaticTypeDirectiveNames = true;
-                scheduleProgramUpdate();
-            },
-            maxNumberOfFilesToIterateForInvalidation: host.maxNumberOfFilesToIterateForInvalidation,
-            getCurrentProgram,
-            writeLog,
-            readDirectory: (path, extensions, exclude, include, depth?) => directoryStructureHost.readDirectory!(path, extensions, exclude, include, depth),
+        const compilerHost = createCompilerHostFromProgramHost(host, () => compilerOptions, directoryStructureHost) as CompilerHost & ResolutionCacheHost;
+        // Members for CompilerHost
+        const getNewSourceFile = compilerHost.getSourceFile;
+        compilerHost.getSourceFile = (fileName, ...args) => getVersionedSourceFileByPath(fileName, toPath(fileName), ...args);
+        compilerHost.getSourceFileByPath = getVersionedSourceFileByPath;
+        compilerHost.getNewLine = () => newLine;
+        compilerHost.fileExists = fileExists;
+        compilerHost.onReleaseOldSourceFile = onReleaseOldSourceFile;
+        // Members for ResolutionCacheHost
+        compilerHost.toPath = toPath;
+        compilerHost.getCompilationSettings = () => compilerOptions;
+        compilerHost.watchDirectoryOfFailedLookupLocation = (dir, cb, flags) => watchDirectory(host, dir, cb, flags, WatchType.FailedLookupLocations);
+        compilerHost.watchTypeRootsDirectory = (dir, cb, flags) => watchDirectory(host, dir, cb, flags, WatchType.TypeRoots);
+        compilerHost.getCachedDirectoryStructureHost = () => cachedDirectoryStructureHost;
+        compilerHost.onInvalidatedResolution = scheduleProgramUpdate;
+        compilerHost.onChangedAutomaticTypeDirectiveNames = () => {
+            hasChangedAutomaticTypeDirectiveNames = true;
+            scheduleProgramUpdate();
         };
+        compilerHost.maxNumberOfFilesToIterateForInvalidation = host.maxNumberOfFilesToIterateForInvalidation;
+        compilerHost.getCurrentProgram = getCurrentProgram;
+        compilerHost.writeLog = writeLog;
+
         // Cache for the module resolution
         const resolutionCache = createResolutionCache(compilerHost, configFileName ?
             getDirectoryPath(getNormalizedAbsolutePath(configFileName, currentDirectory)) :
@@ -630,11 +683,9 @@ namespace ts {
 
         function createNewProgram(program: Program, hasInvalidatedResolution: HasInvalidatedResolution) {
             // Compile the program
-            if (watchLogLevel !== WatchLogLevel.None) {
-                writeLog("CreatingProgramWith::");
-                writeLog(`  roots: ${JSON.stringify(rootFileNames)}`);
-                writeLog(`  options: ${JSON.stringify(compilerOptions)}`);
-            }
+            writeLog("CreatingProgramWith::");
+            writeLog(`  roots: ${JSON.stringify(rootFileNames)}`);
+            writeLog(`  options: ${JSON.stringify(compilerOptions)}`);
 
             const needsUpdateInTypeRootWatch = hasChangedCompilerOptions || !program;
             hasChangedCompilerOptions = false;
@@ -708,7 +759,7 @@ namespace ts {
 
             // Create new source file if requested or the versions dont match
             if (!hostSourceFile || shouldCreateNewSourceFile || !isFilePresentOnHost(hostSourceFile) || hostSourceFile.version.toString() !== hostSourceFile.sourceFile.version) {
-                const sourceFile = getNewSourceFile();
+                const sourceFile = getNewSourceFile(fileName, languageVersion, onError);
                 if (hostSourceFile) {
                     if (shouldCreateNewSourceFile) {
                         hostSourceFile.version++;
@@ -719,7 +770,7 @@ namespace ts {
                         (hostSourceFile as FilePresentOnHost).sourceFile = sourceFile;
                         sourceFile.version = hostSourceFile.version.toString();
                         if (!(hostSourceFile as FilePresentOnHost).fileWatcher) {
-                            (hostSourceFile as FilePresentOnHost).fileWatcher = watchFilePath(host, fileName, onSourceFileChange, PollingInterval.Low, path, "Source file");
+                            (hostSourceFile as FilePresentOnHost).fileWatcher = watchFilePath(host, fileName, onSourceFileChange, PollingInterval.Low, path, WatchType.SourceFile);
                         }
                     }
                     else {
@@ -733,7 +784,7 @@ namespace ts {
                 else {
                     if (sourceFile) {
                         sourceFile.version = initialVersion.toString();
-                        const fileWatcher = watchFilePath(host, fileName, onSourceFileChange, PollingInterval.Low, path, "Source file");
+                        const fileWatcher = watchFilePath(host, fileName, onSourceFileChange, PollingInterval.Low, path, WatchType.SourceFile);
                         sourceFilesCache.set(path, { sourceFile, version: initialVersion, fileWatcher });
                     }
                     else {
@@ -743,23 +794,6 @@ namespace ts {
                 return sourceFile;
             }
             return hostSourceFile.sourceFile;
-
-            function getNewSourceFile() {
-                let text: string | undefined;
-                try {
-                    performance.mark("beforeIORead");
-                    text = host.readFile(fileName, compilerOptions.charset);
-                    performance.mark("afterIORead");
-                    performance.measure("I/O Read", "beforeIORead", "afterIORead");
-                }
-                catch (e) {
-                    if (onError) {
-                        onError(e.message);
-                    }
-                }
-
-                return text !== undefined ? createSourceFile(fileName, text, languageVersion) : undefined;
-            }
         }
 
         function nextSourceFileVersion(path: Path) {
@@ -907,7 +941,7 @@ namespace ts {
         }
 
         function watchMissingFilePath(missingFilePath: Path) {
-            return watchFilePath(host, missingFilePath, onMissingFileChange, PollingInterval.Medium, missingFilePath, "Missing file");
+            return watchFilePath(host, missingFilePath, onMissingFileChange, PollingInterval.Medium, missingFilePath, WatchType.MissingFile);
         }
 
         function onMissingFileChange(fileName: string, eventKind: FileWatcherEventKind, missingFilePath: Path) {
@@ -971,33 +1005,8 @@ namespace ts {
                     }
                 },
                 flags,
-                "Wild card directories"
+                WatchType.WildcardDirectory
             );
         }
-
-        function ensureDirectoriesExist(directoryPath: string) {
-            if (directoryPath.length > getRootLength(directoryPath) && !host.directoryExists!(directoryPath)) {
-                const parentDirectory = getDirectoryPath(directoryPath);
-                ensureDirectoriesExist(parentDirectory);
-                host.createDirectory!(directoryPath);
-            }
-        }
-
-        function writeFile(fileName: string, text: string, writeByteOrderMark: boolean, onError: (message: string) => void) {
-            try {
-                performance.mark("beforeIOWrite");
-                ensureDirectoriesExist(getDirectoryPath(normalizePath(fileName)));
-
-                host.writeFile!(fileName, text, writeByteOrderMark);
-
-                performance.mark("afterIOWrite");
-                performance.measure("I/O Write", "beforeIOWrite", "afterIOWrite");
-            }
-            catch (e) {
-                if (onError) {
-                    onError(e.message);
-                }
-            }
-        }
     }
 }

src/compiler/watchUtilities.ts

@@ -343,10 +343,10 @@ namespace ts {
     export interface WatchDirectoryHost {
         watchDirectory(path: string, callback: DirectoryWatcherCallback, recursive?: boolean): FileWatcher;
     }
-    export type WatchFile<X, Y> = (host: WatchFileHost, file: string, callback: FileWatcherCallback, pollingInterval: PollingInterval, detailInfo1?: X, detailInfo2?: Y) => FileWatcher;
+    export type WatchFile<X, Y> = (host: WatchFileHost, file: string, callback: FileWatcherCallback, pollingInterval: PollingInterval, detailInfo1: X, detailInfo2?: Y) => FileWatcher;
     export type FilePathWatcherCallback = (fileName: string, eventKind: FileWatcherEventKind, filePath: Path) => void;
-    export type WatchFilePath<X, Y> = (host: WatchFileHost, file: string, callback: FilePathWatcherCallback, pollingInterval: PollingInterval, path: Path, detailInfo1?: X, detailInfo2?: Y) => FileWatcher;
-    export type WatchDirectory<X, Y> = (host: WatchDirectoryHost, directory: string, callback: DirectoryWatcherCallback, flags: WatchDirectoryFlags, detailInfo1?: X, detailInfo2?: Y) => FileWatcher;
+    export type WatchFilePath<X, Y> = (host: WatchFileHost, file: string, callback: FilePathWatcherCallback, pollingInterval: PollingInterval, path: Path, detailInfo1: X, detailInfo2?: Y) => FileWatcher;
+    export type WatchDirectory<X, Y> = (host: WatchDirectoryHost, directory: string, callback: DirectoryWatcherCallback, flags: WatchDirectoryFlags, detailInfo1: X, detailInfo2?: Y) => FileWatcher;
 
     export interface WatchFactory<X, Y> {
         watchFile: WatchFile<X, Y>;
@@ -444,7 +444,7 @@ namespace ts {
     }
 
     function getWatchInfo<T, X, Y>(file: string, flags: T, detailInfo1: X, detailInfo2: Y | undefined, getDetailWatchInfo: GetDetailWatchInfo<X, Y> | undefined) {
-        return `WatchInfo: ${file} ${flags} ${getDetailWatchInfo ? getDetailWatchInfo(detailInfo1, detailInfo2) : detailInfo1}`;
+        return `WatchInfo: ${file} ${flags} ${getDetailWatchInfo ? getDetailWatchInfo(detailInfo1, detailInfo2) : detailInfo2 === undefined ? detailInfo1 : `${detailInfo1} ${detailInfo2}`}`;
     }
 
     export function closeFileWatcherOf<T extends { watcher: FileWatcher; }>(objWithWatcher: T) {