diff --git a/image/jpeg.cpp b/image/jpeg.cpp
index eaf4f4f9c60..617d4d434bc 100644
--- a/image/jpeg.cpp
+++ b/image/jpeg.cpp
@@ -40,6 +40,8 @@ extern "C" {
 #include <jpeglib.h>
 #include <jerror.h>
 }
+
+#include <setjmp.h>
 #endif
 
 namespace Image {
@@ -170,9 +172,24 @@ void jpeg_scummvm_src(j_decompress_ptr cinfo, Common::SeekableReadStream *stream
 	source->stream = stream;
 }
 
+struct jpeg_error_mgr_ext : public jpeg_error_mgr {
+	jmp_buf jmp;
+	bool jmp_valid;
+};
+
 void errorExit(j_common_ptr cinfo) {
+	jpeg_error_mgr_ext *err = (jpeg_error_mgr_ext *)cinfo->err;
+
 	char buffer[JMSG_LENGTH_MAX];
 	(*cinfo->err->format_message)(cinfo, buffer);
+
+	if (err->jmp_valid) {
+		/* We will jump back to the loading stream function
+		 * but, before, warn the user */
+		warning("libjpeg: %s", buffer);
+		longjmp(err->jmp, 1);
+	}
+
 	// This function is not allowed to return to the caller, thus we simply
 	// error out with our error handling here.
 	error("libjpeg: %s", buffer);
@@ -232,7 +249,8 @@ bool JPEGDecoder::loadStream(Common::SeekableReadStream &stream) {
 	destroy();
 
 	jpeg_decompress_struct cinfo;
-	jpeg_error_mgr jerr;
+	jpeg_error_mgr_ext jerr;
+	jerr.jmp_valid = false;
 
 	// Initialize error handling callbacks
 	cinfo.err = jpeg_std_error(&jerr);
@@ -250,6 +268,13 @@ bool JPEGDecoder::loadStream(Common::SeekableReadStream &stream) {
 	// Initialize our buffer handling
 	jpeg_scummvm_src(&cinfo, &stream);
 
+	if (setjmp(jerr.jmp)) {
+		/* File is invalid */
+		jpeg_destroy_decompress(&cinfo);
+		return false;
+	}
+	jerr.jmp_valid = true;
+
 	// Read the file header
 	jpeg_read_header(&cinfo, TRUE);
 
@@ -283,9 +308,29 @@ bool JPEGDecoder::loadStream(Common::SeekableReadStream &stream) {
 		cinfo.out_color_space = JCS_CMYK;
 	}
 
+	if (setjmp(jerr.jmp)) {
+		/* Output color space seems invalid
+		 * Try again with the most basic one */
+		if (_colorSpace == kColorSpaceRGB && cinfo.num_components == 3) {
+			warning("Falling back to RGB slow path");
+			cinfo.out_color_space = JCS_RGB;
+		}
+		if (setjmp(jerr.jmp)) {
+			/* There is something definitely wrong here */
+			jpeg_destroy_decompress(&cinfo);
+			return false;
+		}
+	}
+
 	// Actually start decompressing the image
 	jpeg_start_decompress(&cinfo);
 
+	if (setjmp(jerr.jmp)) {
+		/* Something went wrong */
+		jpeg_destroy_decompress(&cinfo);
+		return false;
+	}
+
 	// Allocate buffers for the output data
 	switch (_colorSpace) {
 	case kColorSpaceRGB: {