All browsers again

Instead of right when response is created, which misses timeouts
that occur later

.airtap.yml

@@ -1,22 +1,23 @@
-ui: tape
+sauce_connect: true
 browsers:
   - name: chrome
     version: 39..latest
   - name: firefox
     version: 34..latest
   - name: safari
-    version: 5..latest
-  - name: microsoftedge
+    version: 8..latest
+  - name: MicrosoftEdge
     version: 13..latest
   - name: ie
     version: 9..latest
-  - name: opera
-    version: 11..latest
   - name: iphone
-    version: '8.1..latest'
+    version: '9.3..latest'
   - name: android
-    version: '4.4..latest'
+    version: '4.4..6.0' # TODO: change this back to latest once https://github.com/airtap/browsers/issues/3 is fixed
 server: ./test/server/index.js
 scripts:
   - "/ie8-polyfill.js"
   - "/test-polyfill.js"
+browserify:
+  - options:
+      dedupe: false

.gitignore

@@ -2,4 +2,4 @@
 bundle.js
 node_modules
 npm-debug.log
-.zuulrc
+.airtaprc

.travis.yml

@@ -1,3 +1,7 @@
 language: node_js
 node_js:
-  - "6.4"
+  - "node"
+addons:
+  sauce_connect: true
+  hosts:
+    - airtap.local

README.md

@@ -13,6 +13,10 @@ This is heavily inspired by, and intended to replace, [http-browserify](https://
 In accordance with its name, `stream-http` tries to provide data to its caller before
 the request has completed whenever possible.
 
+Backpressure, allowing the browser to only pull data from the server as fast as it is
+consumed, is supported in:
+* Chrome >= 58 (using `fetch` and `WritableStream`)
+
 The following browsers support true streaming, where only a small amount of the request
 has to be held in memory at once:
 * Chrome >= 43 (using the `fetch` API)
@@ -80,6 +84,12 @@ capability. Preserves the correctness of binary data and the 'content-type' resp
   * 'prefer-fast': Deprecated; now a synonym for 'default', which has the same performance
 characteristics as this mode did in versions before 1.5.
 
+* `options.requestTimeout` allows setting a timeout in millisecionds for XHR and fetch (if
+supported by the browser). This is a limit on how long the entire process takes from
+beginning to end. Note that this is not the same as the node `setTimeout` functions,
+which apply to pauses in data transfer over the underlying socket, or the node `timeout`
+option, which applies to opening the connection.
+
 ### Features missing compared to Node
 
 * `http.Agent` is only a stub
@@ -94,8 +104,8 @@ the server.
 * `message.trailers` and `message.rawTrailers` will remain empty.
 * Redirects are followed silently by the browser, so it isn't possible to access the 301/302
 redirect pages.
-* The `timeout` options in the `request` method is non-operational in Safari <= 5 and
-Opera <= 12.
+* The `timeout` event/option and `setTimeout` functions, which operate on the underlying
+socket, are not available. However, see `options.requestTimeout` above.
 
 ## Example
 

index.js

@@ -1,4 +1,5 @@
 var ClientRequest = require('./lib/request')
+var response = require('./lib/response')
 var extend = require('xtend')
 var statusCodes = require('builtin-status-codes')
 var url = require('url')
@@ -44,9 +45,14 @@ http.get = function get (opts, cb) {
 	return req
 }
 
+http.ClientRequest = ClientRequest
+http.IncomingMessage = response.IncomingMessage
+
 http.Agent = function () {}
 http.Agent.defaultMaxSockets = 4
 
+http.globalAgent = new http.Agent()
+
 http.STATUS_CODES = statusCodes
 
 http.METHODS = [

lib/capability.js

@@ -1,5 +1,9 @@
 exports.fetch = isFunction(global.fetch) && isFunction(global.ReadableStream)
 
+exports.writableStream = isFunction(global.WritableStream)
+
+exports.abortController = isFunction(global.AbortController)
+
 exports.blobConstructor = false
 try {
 	new Blob([new ArrayBuffer(1)])

lib/request.js

@@ -38,9 +38,8 @@ var ClientRequest = module.exports = function (opts) {
 
 	var preferBinary
 	var useFetch = true
-	if (opts.mode === 'disable-fetch' || 'timeout' in opts) {
-		// If the use of XHR should be preferred and includes preserving the 'content-type' header.
-		// Force XHR to be used since the Fetch API does not yet support timeouts.
+	if (opts.mode === 'disable-fetch' || ('requestTimeout' in opts && !capability.abortController)) {
+		// If the use of XHR should be preferred. Not typically needed.
 		useFetch = false
 		preferBinary = true
 	} else if (opts.mode === 'prefer-streaming') {
@@ -57,6 +56,7 @@ var ClientRequest = module.exports = function (opts) {
 		throw new Error('Invalid value for opts.mode')
 	}
 	self._mode = decideMode(preferBinary, useFetch)
+	self._fetchTimer = null
 
 	self.on('finish', function () {
 		self._onFinish()
@@ -81,8 +81,10 @@ ClientRequest.prototype.setHeader = function (name, value) {
 }
 
 ClientRequest.prototype.getHeader = function (name) {
-	var self = this
-	return self._headers[name.toLowerCase()].value
+	var header = this._headers[name.toLowerCase()]
+	if (header)
+		return header.value
+	return null
 }
 
 ClientRequest.prototype.removeHeader = function (name) {
@@ -100,7 +102,9 @@ ClientRequest.prototype._onFinish = function () {
 	var headersObj = self._headers
 	var body = null
 	if (opts.method !== 'GET' && opts.method !== 'HEAD') {
-		if (capability.blobConstructor) {
+		if (capability.arraybuffer) {
+			body = toArrayBuffer(Buffer.concat(self._body))
+		} else if (capability.blobConstructor) {
 			body = new global.Blob(self._body.map(function (buffer) {
 				return toArrayBuffer(buffer)
 			}), {
@@ -127,17 +131,36 @@ ClientRequest.prototype._onFinish = function () {
 	})
 
 	if (self._mode === 'fetch') {
+		var signal = null
+		var fetchTimer = null
+		if (capability.abortController) {
+			var controller = new AbortController()
+			signal = controller.signal
+			self._fetchAbortController = controller
+
+			if ('requestTimeout' in opts && opts.requestTimeout !== 0) {
+				self._fetchTimer = global.setTimeout(function () {
+					self.emit('requestTimeout')
+					if (self._fetchAbortController)
+						self._fetchAbortController.abort()
+				}, opts.requestTimeout)
+			}
+		}
+
 		global.fetch(self._opts.url, {
 			method: self._opts.method,
 			headers: headersList,
 			body: body || undefined,
 			mode: 'cors',
-			credentials: opts.withCredentials ? 'include' : 'same-origin'
+			credentials: opts.withCredentials ? 'include' : 'same-origin',
+			signal: signal
 		}).then(function (response) {
 			self._fetchResponse = response
 			self._connect()
 		}, function (reason) {
-			self.emit('error', reason)
+			global.clearTimeout(self._fetchTimer)
+			if (!self._destroyed)
+				self.emit('error', reason)
 		})
 	} else {
 		var xhr = self._xhr = new global.XMLHttpRequest()
@@ -160,10 +183,10 @@ ClientRequest.prototype._onFinish = function () {
 		if (self._mode === 'text' && 'overrideMimeType' in xhr)
 			xhr.overrideMimeType('text/plain; charset=x-user-defined')
 
-		if ('timeout' in opts) {
-			xhr.timeout = opts.timeout
+		if ('requestTimeout' in opts) {
+			xhr.timeout = opts.requestTimeout
 			xhr.ontimeout = function () {
-				self.emit('timeout')
+				self.emit('requestTimeout')
 			}
 		}
 
@@ -237,7 +260,7 @@ ClientRequest.prototype._connect = function () {
 	if (self._destroyed)
 		return
 
-	self._response = new IncomingMessage(self._xhr, self._fetchResponse, self._mode)
+	self._response = new IncomingMessage(self._xhr, self._fetchResponse, self._mode, self._fetchTimer)
 	self._response.on('error', function(err) {
 		self.emit('error', err)
 	})
@@ -255,12 +278,13 @@ ClientRequest.prototype._write = function (chunk, encoding, cb) {
 ClientRequest.prototype.abort = ClientRequest.prototype.destroy = function () {
 	var self = this
 	self._destroyed = true
+	global.clearTimeout(self._fetchTimer)
 	if (self._response)
 		self._response._destroyed = true
 	if (self._xhr)
 		self._xhr.abort()
-	// Currently, there isn't a way to truly abort a fetch.
-	// If you like bikeshedding, see https://github.com/whatwg/fetch/issues/27
+	else if (self._fetchAbortController)
+		self._fetchAbortController.abort()
 }
 
 ClientRequest.prototype.end = function (data, encoding, cb) {

lib/response.js

@@ -10,7 +10,7 @@ var rStates = exports.readyStates = {
 	DONE: 4
 }
 
-var IncomingMessage = exports.IncomingMessage = function (xhr, response, mode) {
+var IncomingMessage = exports.IncomingMessage = function (xhr, response, mode, fetchTimer) {
 	var self = this
 	stream.Readable.call(self)
 
@@ -35,30 +35,64 @@ var IncomingMessage = exports.IncomingMessage = function (xhr, response, mode) {
 		self.statusCode = response.status
 		self.statusMessage = response.statusText
 		
-		response.headers.forEach(function(header, key){
+		response.headers.forEach(function (header, key){
 			self.headers[key.toLowerCase()] = header
 			self.rawHeaders.push(key, header)
 		})
 
+		if (capability.writableStream) {
+			var writable = new WritableStream({
+				write: function (chunk) {
+					return new Promise(function (resolve, reject) {
+						if (self._destroyed) {
+							reject()
+						} else if(self.push(new Buffer(chunk))) {
+							resolve()
+						} else {
+							self._resumeFetch = resolve
+						}
+					})
+				},
+				close: function () {
+					global.clearTimeout(fetchTimer)
+					if (!self._destroyed)
+						self.push(null)
+				},
+				abort: function (err) {
+					if (!self._destroyed)
+						self.emit('error', err)
+				}
+			})
 
-		// TODO: this doesn't respect backpressure. Once WritableStream is available, this can be fixed
+			try {
+				response.body.pipeTo(writable).catch(function (err) {
+					global.clearTimeout(fetchTimer)
+					if (!self._destroyed)
+						self.emit('error', err)
+				})
+				return
+			} catch (e) {} // pipeTo method isn't defined. Can't find a better way to feature test this
+		}
+		// fallback for when writableStream or pipeTo aren't available
 		var reader = response.body.getReader()
 		function read () {
 			reader.read().then(function (result) {
 				if (self._destroyed)
 					return
 				if (result.done) {
+					global.clearTimeout(fetchTimer)
 					self.push(null)
 					return
 				}
 				self.push(new Buffer(result.value))
 				read()
-			}).catch(function(err) {
-				self.emit('error', err)
+			}).catch(function (err) {
+				global.clearTimeout(fetchTimer)
+				if (!self._destroyed)
+					self.emit('error', err)
 			})
 		}
 		read()
-
 	} else {
 		self._xhr = xhr
 		self._pos = 0
@@ -102,7 +136,15 @@ var IncomingMessage = exports.IncomingMessage = function (xhr, response, mode) {
 
 inherits(IncomingMessage, stream.Readable)
 
-IncomingMessage.prototype._read = function () {}
+IncomingMessage.prototype._read = function () {
+	var self = this
+
+	var resolve = self._resumeFetch
+	if (resolve) {
+		self._resumeFetch = null
+		resolve()
+	}
+}
 
 IncomingMessage.prototype._onXHRProgress = function () {
 	var self = this

package.json

@@ -1,6 +1,6 @@
 {
   "name": "stream-http",
-  "version": "2.7.0",
+  "version": "2.8.1",
   "description": "Streaming http in the browser",
   "main": "index.js",
   "repository": {
@@ -10,8 +10,8 @@
   "scripts": {
     "test": "npm run test-node && ([ -n \"${TRAVIS_PULL_REQUEST}\" -a \"${TRAVIS_PULL_REQUEST}\" != 'false' ] || npm run test-browser)",
     "test-node": "tape test/node/*.js",
-    "test-browser": "zuul --no-coverage -- test/browser/*.js",
-    "test-browser-local": "zuul --local 8080 --no-coverage -- test/browser/*.js"
+    "test-browser": "DEBUG=airtap airtap --loopback airtap.local -- test/browser/*.js",
+    "test-browser-local": "airtap --no-instrument --local 8080 -- test/browser/*.js"
   },
   "author": "John Hiesey",
   "license": "MIT",
@@ -29,18 +29,18 @@
   "dependencies": {
     "builtin-status-codes": "^3.0.0",
     "inherits": "^2.0.1",
-    "readable-stream": "^2.2.6",
+    "readable-stream": "^2.3.6",
     "to-arraybuffer": "^1.0.0",
     "xtend": "^4.0.0"
   },
   "devDependencies": {
-    "basic-auth": "^1.0.3",
-    "brfs": "^1.4.0",
-    "cookie-parser": "^1.3.5",
-    "express": "^4.15.2",
-    "tape": "^4.0.0",
-    "ua-parser-js": "^0.7.7",
-    "webworkify": "^1.0.2",
-    "zuul": "^3.10.3"
+    "airtap": "^0.0.5",
+    "basic-auth": "^2.0.0",
+    "brfs": "^1.6.1",
+    "cookie-parser": "^1.4.3",
+    "express": "^4.16.3",
+    "tape": "^4.9.0",
+    "ua-parser-js": "^0.7.18",
+    "webworkify": "^1.5.0"
   }
 }

test/browser/binary-streaming.js

@@ -8,8 +8,8 @@ var http = require('../..')
 var browser = (new UAParser()).setUA(navigator.userAgent).getBrowser()
 var browserName = browser.name
 var browserVersion = browser.major
-// Binary streaming doesn't work in IE10 or below or in Opera
-var skipStreamingCheck = (browserName === 'Opera' || (browserName === 'IE' && browserVersion <= 10))
+// Binary streaming doesn't work in IE10 or below
+var skipStreamingCheck = (browserName === 'IE' && browserVersion <= 10)
 
 // Binary data gets corrupted in IE8 or below
 var skipVerification = (browserName === 'IE' && browserVersion <= 8)

test/browser/headers.js

@@ -90,20 +90,26 @@ test('content-type response header', function (t) {
 var browser = (new UAParser()).setUA(navigator.userAgent).getBrowser()
 var browserName = browser.name
 var browserVersion = browser.major
-var browserMinorVersion = browser.minor
+var browserMinorVersion = browser.minor || 0
 // The content-type header is broken when 'prefer-streaming' or 'allow-wrong-content-type'
-// is passed in browsers that rely on xhr.overrideMimeType(), namely older chrome and safari 6-10.0
+// is passed in browsers that rely on xhr.overrideMimeType(), namely older chrome, safari 6-10.0, and the stock Android browser
+// Note that Safari 10.0 on iOS 10.3 doesn't need to override the mime type, so the content-type is preserved.
 var wrongMimeType = ((browserName === 'Chrome' && browserVersion <= 42) ||
-	((browserName === 'Safari' || browserName === 'Mobile Safari') && browserVersion >= 6 && (browserVersion < 10 || (browserVersion === 10 && browserMinorVersion === 0))))
+	((browserName === 'Safari' || browserName === 'Mobile Safari') && browserVersion >= 6 && (browserVersion < 10 || (browserVersion == 10 && browserMinorVersion == 0)))
+	|| (browserName === 'Android Browser'))
 
 test('content-type response header with forced streaming', function (t) {
 	http.get({
 		path: '/testHeaders',
 		mode: 'prefer-streaming'
 	}, function (res) {
-		if (wrongMimeType)
-			t.equal(res.headers['content-type'], 'text/plain; charset=x-user-defined', 'content-type overridden')
-		else
+		if (wrongMimeType) {
+			// allow both the 'wrong' and correct mime type, since sometimes it's impossible to tell which to expect
+			// from the browser version alone (e.g. Safari 10.0 on iOS 10.2 vs iOS 10.3)
+			var contentType = res.headers['content-type']
+			var correct = (contentType === 'text/plain; charset=x-user-defined') || (contentType === 'application/json')
+			t.ok(correct, 'content-type either preserved or overridden')
+		} else
 			t.equal(res.headers['content-type'], 'application/json', 'content-type preserved')
 		t.end()
 	})

test/browser/post-binary.js

@@ -9,8 +9,7 @@ var browser = (new UAParser()).setUA(navigator.userAgent).getBrowser()
 var browserName = browser.name
 var browserVersion = browser.major
 // Binary request bodies don't work in a bunch of browsers
-var skipVerification = ((browserName === 'Opera' && browserVersion <= 11) ||
-	(browserName === 'IE' && browserVersion <= 10) ||
+var skipVerification = ((browserName === 'IE' && browserVersion <= 10) ||
 	(browserName === 'Safari' && browserVersion <= 5) ||
 	(browserName === 'WebKit' && browserVersion <= 534) || // Old mobile safari
 	(browserName === 'Android Browser' && browserVersion <= 4))

test/browser/text-streaming.js

@@ -8,8 +8,8 @@ var http = require('../..')
 var browser = (new UAParser()).setUA(navigator.userAgent).getBrowser()
 var browserName = browser.name
 var browserVersion = browser.major
-// Streaming doesn't work in IE9 or below or in Opera
-var skipStreamingCheck = (browserName === 'Opera' || (browserName === 'IE' && browserVersion <= 9))
+// Streaming doesn't work in IE9 or below
+var skipStreamingCheck = (browserName === 'IE' && browserVersion <= 9)
 
 var COPIES = 1000
 var MIN_PIECES = 5

test/browser/text.js

@@ -10,8 +10,7 @@ var browser = (new UAParser()).setUA(navigator.userAgent).getBrowser()
 var browserName = browser.name
 var browserVersion = browser.major
 // Response urls don't work on many browsers
-var skipResponseUrl = ((browserName === 'Opera') ||
-	(browserName === 'IE') ||
+var skipResponseUrl = ((browserName === 'IE') ||
 	(browserName === 'Edge') ||
 	(browserName === 'Chrome' && browserVersion <= 36) ||
 	(browserName === 'Firefox' && browserVersion <= 31) ||

test/browser/timeout.js

@@ -0,0 +1,43 @@
+var Buffer = require('buffer').Buffer
+var fs = require('fs')
+var test = require('tape')
+
+var http = require('../..')
+
+test('timeout', function (t) {
+	var req = http.get({
+		path: '/browserify.png?copies=5',
+		requestTimeout: 10 // ms
+	}, function (res) {
+		res.on('data', function (data) {
+		})
+		res.on('end', function () {
+			t.fail('request completed (should have timed out)')
+		})
+	})
+	req.on('requestTimeout', function () {
+		t.pass('got timeout')
+		t.end()
+	})
+})
+
+// TODO: reenable this if there's a way to make it simultaneously
+// fast and reliable
+test.skip('no timeout after success', function (t) {
+	var req = http.get({
+		path: '/basic.txt',
+		requestTimeout: 50000 // ms
+	}, function (res) {
+		res.on('data', function (data) {
+		})
+		res.on('end', function () {
+			t.pass('success')
+			global.setTimeout(function () {
+				t.end()
+			}, 50000)
+		})
+	})
+	req.on('requestTimeout', function () {
+		t.fail('unexpected timeout')
+	})
+})

test/browser/timeout.js.disabled

@@ -1,33 +0,0 @@
-var Buffer = require('buffer').Buffer
-var fs = require('fs')
-var test = require('tape')
-var UAParser = require('ua-parser-js')
-var url = require('url')
-
-var http = require('../..')
-
-var browser = (new UAParser()).setUA(navigator.userAgent).getBrowser()
-var browserName = browser.name
-var browserVersion = browser.major
-
-var skipTimeout = ((browserName === 'Opera' && browserVersion <= 12) ||
-	(browserName === 'Safari' && browserVersion <= 5))
-
-
-test('emits timeout events', function (t) {
-	if (skipTimeout) {
-		return t.skip('Browser does not support setting timeouts')
-	}
-
-	var req = http.request({
-		path: '/basic.txt',
-		timeout: 1
-	})
-
-	req.on('timeout', function () {
-		t.pass('timeout caught')
-		t.end() // the test will timeout if this does not happen
-	})
-
-	req.end()
-})

test/browser/webworker.js

@@ -8,8 +8,7 @@ var browser = (new UAParser()).setUA(navigator.userAgent).getBrowser()
 var browserName = browser.name
 var browserVersion = browser.major
 // Skip browsers with poor or nonexistant WebWorker support
-var skip = ((browserName === 'Opera' && browserVersion <= 12) ||
-	(browserName === 'IE' && browserVersion <= 10) ||
+var skip = ((browserName === 'IE' && browserVersion <= 10) ||
 	(browserName === 'Safari' && browserVersion <= 5) ||
 	(browserName === 'WebKit' && browserVersion <= 534) || // Old mobile safari
 	(browserName === 'Android Browser' && browserVersion <= 4))

test/node/http-browserify.js

@@ -1,4 +1,4 @@
-// These tests are teken from http-browserify to ensure compatibility with
+// These tests are taken from http-browserify to ensure compatibility with
 // that module
 var test = require('tape')
 var url = require('url')
@@ -17,6 +17,20 @@ var moduleName = require.resolve('../../')
 delete require.cache[moduleName]
 var http = require('../../')
 
+test('Make sure http object has correct properties', function (t) {
+	t.ok(http.Agent, 'Agent defined')
+	t.ok(http.ClientRequest, 'ClientRequest defined')
+	t.ok(http.ClientRequest.prototype, 'ClientRequest.prototype defined')
+	t.ok(http.IncomingMessage, 'IncomingMessage defined')
+	t.ok(http.IncomingMessage.prototype, 'IncomingMessage.prototype defined')
+	t.ok(http.METHODS, 'METHODS defined')
+	t.ok(http.STATUS_CODES, 'STATUS_CODES defined')
+	t.ok(http.get, 'get defined')
+	t.ok(http.globalAgent, 'globalAgent defined')
+	t.ok(http.request, 'request defined')
+	t.end()
+})
+
 test('Test simple url string', function(t) {
 	var testUrl = { path: '/api/foo' }
 	var request = http.get(testUrl, noop)

test/server/index.js

@@ -32,6 +32,7 @@ app.get('/testHeaders', function (req, res) {
 	})
 
 	res.setHeader('Content-Type', 'application/json')
+	res.setHeader('Cache-Control', 'no-cache')
 
 	// Request headers are sent in the body as json
 	var reqHeaders = {}
@@ -131,6 +132,6 @@ app.use(function (req, res, next) {
 
 app.use(express.static(path.join(__dirname, 'static')))
 
-var port = parseInt(process.env.ZUUL_PORT) || 8199
+var port = parseInt(process.env.AIRTAP_PORT) || 8199
 console.log('Test server listening on port', port)
 server.listen(port)