Update Changelog

Complicates merging

.gitignore

@@ -0,0 +1,17 @@
+*.exe
+*.dll
+*.pdb
+*.ilk
+*.exp
+*.lib
+*.suo
+
+/rabcasm
+/rabcdasm
+/abcexport
+/abcreplace
+/swfbinexport
+/swfbinreplace
+/swfdecompress
+/swf7zcompress
+/swflzmacompress

CHANGELOG.md

@@ -1,6 +1,22 @@
 RABCDAsm Changelog
 ==================
 
+RABCDAsm v1.18 (2016.01.16)
+---------------------------
+
+ * Fix disassembly of `pushbyte` instructions (the AVM specification 
+   incorrectly lists the argument as unsigned).
+ * Bump `#version` directive of new disassemblies to 4:
+   - Versions below 4 treat `pushbyte` as unsigned, and throw an exception if 
+     the argument is outside the range `0` ... `255`.
+   - Versions 4 and above treat `pushbyte` as signed, and throw an exception 
+     if the argument is outside the range `-128` ... `127`.
+ * Fix buffer reuse bugs when using macros
+ * Dump floating-point numbers in hex notation if necessary to ensure precision
+ * Cease emitting a number in a comment after opening `script` tags
+ * Detect a known DMD bug in `build_rabcdasm`
+ * Ignore invalid file size in header
+
 RABCDAsm v1.17 (2014.09.10)
 ---------------------------
 

abcfile.d

@@ -237,6 +237,7 @@ class ABCFile
 		Opcode opcode;
 		union Argument
 		{
+			byte bytev;
 			ubyte ubytev;
 			long intv;
 			ulong uintv;
@@ -641,6 +642,7 @@ enum OpcodeArgumentType
 {
 	Unknown,
 
+	ByteLiteral,
 	UByteLiteral,
 	IntLiteral,
 	UIntLiteral,
@@ -702,7 +704,7 @@ const OpcodeInfo[256] opcodeInfo = [
 	/* 0x21 */		{"pushundefined",		[]},
 	/* 0x22 */		{"pushuninitialized",	[OpcodeArgumentType.Unknown]},
 	/* 0x23 */		{"nextvalue",			[]},
-	/* 0x24 */		{"pushbyte",			[OpcodeArgumentType.UByteLiteral]},
+	/* 0x24 */		{"pushbyte",			[OpcodeArgumentType.ByteLiteral]},
 	/* 0x25 */		{"pushshort",			[OpcodeArgumentType.IntLiteral]},
 	/* 0x26 */		{"pushtrue",			[]},
 	/* 0x27 */		{"pushfalse",			[]},
@@ -1378,6 +1380,9 @@ private final class ABCReader
 									case OpcodeArgumentType.Unknown:
 										throw new Exception("Don't know how to decode OP_" ~ opcodeInfo[instruction.opcode].name);
 
+									case OpcodeArgumentType.ByteLiteral:
+										instruction.arguments[i].bytev = readU8();
+										break;
 									case OpcodeArgumentType.UByteLiteral:
 										instruction.arguments[i].ubytev = readU8();
 										break;
@@ -1933,6 +1938,9 @@ private final class ABCWriter
 						case OpcodeArgumentType.Unknown:
 							throw new Exception("Don't know how to encode OP_" ~ opcodeInfo[instruction.opcode].name);
 
+						case OpcodeArgumentType.ByteLiteral:
+							writeU8(instruction.arguments[i].bytev);
+							break;
 						case OpcodeArgumentType.UByteLiteral:
 							writeU8(instruction.arguments[i].ubytev);
 							break;

asprogram.d

@@ -269,6 +269,7 @@ final class ASProgram
 		Opcode opcode;
 		union Argument
 		{
+			byte bytev;
 			ubyte ubytev;
 
 			long intv;
@@ -598,6 +599,9 @@ private final class ABCtoAS
 				case OpcodeArgumentType.Unknown:
 					throw new Exception("Don't know how to convert OP_" ~ opcodeInfo[instruction.opcode].name);
 
+				case OpcodeArgumentType.ByteLiteral:
+					r.arguments[i].bytev = instruction.arguments[i].bytev;
+					break;
 				case OpcodeArgumentType.UByteLiteral:
 					r.arguments[i].ubytev = instruction.arguments[i].ubytev;
 					break;
@@ -831,7 +835,8 @@ private final class AStoABC : ASVisitor
 			// sort
 			static if (is(T == double))
 			{
-				static ulong repr(double d) { static assert(double.sizeof == ulong.sizeof); return *cast(ulong*)(&d); }
+				// Need ref to work around https://issues.dlang.org/show_bug.cgi?id=13998
+				static ulong repr(ref double d) { static assert(double.sizeof == ulong.sizeof); return *cast(ulong*)(&d); }
 				static bool sortPred(Entry* a, Entry* b) { return a.hits > b.hits || (a.hits == b.hits && repr(a.value) < repr(b.value)); }
 			}
 			else
@@ -1239,6 +1244,9 @@ private final class AStoABC : ASVisitor
 				case OpcodeArgumentType.Unknown:
 					throw new Exception("Don't know how to convert OP_" ~ opcodeInfo[instruction.opcode].name);
 
+				case OpcodeArgumentType.ByteLiteral:
+					r.arguments[i].bytev = instruction.arguments[i].bytev;
+					break;
 				case OpcodeArgumentType.UByteLiteral:
 					r.arguments[i].ubytev = instruction.arguments[i].ubytev;
 					break;
@@ -1524,6 +1532,7 @@ class ASVisitor : ASTraitsVisitor
 						case OpcodeArgumentType.Unknown:
 							throw new Exception("Don't know how to visit OP_" ~ opcodeInfo[instruction.opcode].name);
 
+						case OpcodeArgumentType.ByteLiteral:
 						case OpcodeArgumentType.UByteLiteral:
 						case OpcodeArgumentType.IntLiteral:
 						case OpcodeArgumentType.UIntLiteral:

assembler.d

@@ -1,5 +1,5 @@
 /*
- *  Copyright 2010, 2011, 2012, 2013, 2014 Vladimir Panteleev <vladimir@thecybershadow.net>
+ *  Copyright 2010, 2011, 2012, 2013, 2014, 2015 Vladimir Panteleev <vladimir@thecybershadow.net>
  *  This file is part of RABCDAsm.
  *
  *  RABCDAsm is free software: you can redistribute it and/or modify
@@ -25,6 +25,7 @@ import std.path;
 import std.range;
 import std.stdio;
 import std.string;
+import std.typecons;
 
 import abcfile;
 import asprogram;
@@ -34,10 +35,10 @@ final class Assembler
 {
 	static struct Position
 	{
-		File file;
+		SourceFile file;
 		ulong offset;
 
-		File load()
+		SourceFile load()
 		{
 			assert(file.ptr == file.end);
 			file.filePosition = offset;
@@ -45,14 +46,14 @@ final class Assembler
 		}
 	}
 
-	static final class File
+	static final class SourceFile
 	{
 		string name;
 		string[] arguments;
 		string data;
 		@property bool isVirtual() { return data ! is null; }
 
-		File parent;
+		SourceFile parent;
 
 		const(char)* ptr, end;
 
@@ -63,7 +64,7 @@ final class Assembler
 
 		ulong filePosition; // File position of buffer.ptr
 		sizediff_t shift; // Error location adjustment
-		std.stdio.File f;
+		File f;
 
 		this(string name, string data = null, string[] arguments = null)
 		{
@@ -73,7 +74,7 @@ final class Assembler
 			if (data)
 			{
 				createBuffer(data.length);
-				buffer[] = data[];
+				buffer[0..data.length] = data[];
 				ptr = buffer.ptr;
 				end = ptr + data.length;
 			}
@@ -84,6 +85,12 @@ final class Assembler
 			}
 		}
 
+		~this()
+		{
+			if (f.isOpen)
+				f.close();
+		}
+
 		bool loadNextChunk()
 		{
 			if (!f.isOpen)
@@ -120,6 +127,7 @@ final class Assembler
 			{
 				buffer = usedBuffers[0];
 				usedBuffers = usedBuffers[1..$];
+				buffer[size..bufferSize] = 0;
 			}
 			else
 				buffer = new char[bufferSize];
@@ -176,7 +184,7 @@ final class Assembler
 		}
 	}
 
-	File currentFile;
+	SourceFile currentFile;
 
 	string getBasePath()
 	{
@@ -233,17 +241,17 @@ final class Assembler
 		switch (word)
 		{
 			case "mixin":
-				pushFile(new File("#mixin", readImmString()));
+				pushFile(new SourceFile("#mixin", readImmString()));
 				break;
 			case "call": // #mixin with arguments
-				pushFile(new File("#call", readImmString(), readList!('(', ')', readImmString, false)()));
+				pushFile(new SourceFile("#call", readImmString(), readList!('(', ')', readImmString, false)()));
 				break;
 			case "include":
-				pushFile(new File(convertFilename(readString())));
+				pushFile(new SourceFile(convertFilename(readString())));
 				break;
 			case "get":
 				auto filename = convertFilename(readString());
-				pushFile(new File(filename, toStringLiteral(cast(string)read(longPath(filename)))));
+				pushFile(new SourceFile(filename, toStringLiteral(cast(string)read(longPath(filename)))));
 				break;
 			case "set":
 				vars[readWord()] = readImmString();
@@ -258,7 +266,7 @@ final class Assembler
 				break;
 			case "version":
 				sourceVersion = readUInt().to!uint();
-				enforce(sourceVersion >= 1 && sourceVersion <= 3, "Invalid/unknown #version");
+				enforce(sourceVersion >= 1 && sourceVersion <= 4, "Invalid/unknown #version");
 				break;
 			default:
 				backpedal(word.length);
@@ -292,7 +300,7 @@ final class Assembler
 					if (index >= f.arguments.length)
 						throw new Exception("Argument index out-of-bounds");
 					string value = f.arguments[index];
-					pushFile(new File('$' ~ name.assumeUnique(), asStringLiteral ? toStringLiteral(value) : value));
+					pushFile(new SourceFile('$' ~ name.assumeUnique(), asStringLiteral ? toStringLiteral(value) : value));
 					return;
 				}
 			throw new Exception("No arguments in context");
@@ -303,7 +311,7 @@ final class Assembler
 			if (pvalue is null)
 				throw new Exception("variable %s is not defined".format(name));
 			string value = *pvalue;
-			pushFile(new File('$' ~ name.assumeUnique(), asStringLiteral ? toStringLiteral(value) : value));
+			pushFile(new SourceFile('$' ~ name.assumeUnique(), asStringLiteral ? toStringLiteral(value) : value));
 		}
 	}
 
@@ -356,14 +364,14 @@ final class Assembler
 		}
 	}
 
-	void pushFile(File file)
+	void pushFile(SourceFile file)
 	{
 		file.parent = currentFile;
 		currentFile = file;
 	}
 
 	/// For restoring the position of an error
-	void setFile(File file)
+	void setFile(SourceFile file)
 	{
 		currentFile = file;
 	}
@@ -1163,6 +1171,12 @@ final class Assembler
 					case OpcodeArgumentType.Unknown:
 						throw new Exception("Don't know how to assemble OP_" ~ opcodeInfo[instruction.opcode].name);
 
+					case OpcodeArgumentType.ByteLiteral:
+						if (sourceVersion < 4)
+							instruction.arguments[i].bytev = to!ubyte(readInt());
+						else
+							instruction.arguments[i].bytev = to!byte(readInt());
+						break;
 					case OpcodeArgumentType.UByteLiteral:
 						instruction.arguments[i].ubytev = to!ubyte(readUInt());
 						break;
@@ -1324,45 +1338,50 @@ final class Assembler
 
 	void assemble(string mainFilename)
 	{
-		pushFile(new File(mainFilename));
+		auto mainFile = scoped!SourceFile(mainFilename);
+		pushFile(mainFile);
 
 		try
 		{
 			readProgram();
-
-			foreach (ref f; classFixups)
-				if (f.name is null)
-					*f.ptr = null;
-				else
-				{
-					auto cp = f.name in classesByID;
-					if (cp is null)
-					{
-						setFile(f.where.load());
-						throw new Exception("Unknown class refid: " ~ f.name);
-					}
-					*f.ptr = *cp;
-				}
-
-			foreach (ref f; methodFixups)
-				if (f.name is null)
-					*f.ptr = null;
-				else
-				{
-					auto mp = f.name in methodsByID;
-					if (mp is null)
-					{
-						setFile(f.where.load());
-						throw new Exception("Unknown method refid: " ~ f.name);
-					}
-					*f.ptr = *mp;
-				}
+			applyFixups();
 		}
 		catch (Exception e)
 		{
 			e.msg = "\n%s\n%s".format(context(), e.msg);
 			throw e;
 		}
+	}
+
+	void applyFixups()
+	{
+		foreach (ref f; classFixups)
+			if (f.name is null)
+				*f.ptr = null;
+			else
+			{
+				auto cp = f.name in classesByID;
+				if (cp is null)
+				{
+					setFile(f.where.load());
+					throw new Exception("Unknown class refid: " ~ f.name);
+				}
+				*f.ptr = *cp;
+			}
+
+		foreach (ref f; methodFixups)
+			if (f.name is null)
+				*f.ptr = null;
+			else
+			{
+				auto mp = f.name in methodsByID;
+				if (mp is null)
+				{
+					setFile(f.where.load());
+					throw new Exception("Unknown method refid: " ~ f.name);
+				}
+				*f.ptr = *mp;
+			}
 
 		classFixups = null;
 		methodFixups = null;

build_rabcdasm.d

@@ -61,6 +61,16 @@ void test(string code, string extraFlags=null)
 	stderr.writeln(" >>> OK");
 }
 
+void testBug(string description, int bugId, string code)
+{
+	scope(failure)
+	{
+		stderr.writefln("Compiler bug detected: %s ( https://issues.dlang.org/show_bug.cgi?id=%d ).", description, bugId);
+		stderr.writeln("Try again with a different D compiler, compiler version, or build flags (DCFLAGS environment variable)");
+	}
+	test(code);
+}
+
 int main(string[] args)
 {
 	try
@@ -89,6 +99,11 @@ int main(string[] args)
 			void main() {}
 		`);
 
+		stderr.writeln("* Checking for known compiler bugs...");
+		testBug("[REG 2.064] Wrong code with -O on x86_64 for char comparisons", 11508, `
+			import assembler; int main() { foreach (c; "_") if (!Assembler.isWordChar(c)) return 1; return 0; }
+		`);
+
 		bool haveLZMA;
 
 		stderr.writeln("* Checking for LZMA...");

deimos/lzma_/index.d

@@ -223,7 +223,7 @@ struct lzma_index_iter
 		size_t s;
 		lzma_vli v;
 	}
-	InternalData internal[6];
+	InternalData[6] internal;
 }
 
 

disassembler.d

@@ -1,5 +1,5 @@
 /*
- *  Copyright 2010, 2011, 2012, 2013, 2014 Vladimir Panteleev <vladimir@thecybershadow.net>
+ *  Copyright 2010, 2011, 2012, 2013, 2014, 2015 Vladimir Panteleev <vladimir@thecybershadow.net>
  *  This file is part of RABCDAsm.
  *
  *  RABCDAsm is free software: you can redistribute it and/or modify
@@ -535,7 +535,7 @@ final class RefBuilder : ASTraitsVisitor
 				if (obj !in contexts)
 					getContext(refs, obj);
 
-			foreach (obj; contexts.keys.sort)
+			foreach (obj; contexts.keys.sort())
 			{
 				auto context = contexts[obj];
 				auto bname = refs.contextToString(context, false);
@@ -1012,6 +1012,7 @@ final class Disassembler
 	ASProgram as;
 	string name, dir;
 	RefBuilder refs;
+	bool dumpRaw = true;
 
 	void newInclude(StringBuilder mainsb, string filename, void delegate(StringBuilder) callback, bool doInline = true)
 	{
@@ -1051,7 +1052,7 @@ final class Disassembler
 
 		StringBuilder sb = new StringBuilder(dir ~ "/" ~ name ~ ".main.asasm");
 
-		sb ~= "#version 3";
+		sb ~= "#version 4";
 		sb.newLine();
 
 		sb ~= "program";
@@ -1146,17 +1147,33 @@ final class Disassembler
 			static double forceDouble(double d) { static double n; n = d; return n; }
 			if (s != "nan" && s != "inf" && s != "-inf")
 			{
-				foreach_reverse (i; 1..s.length)
-					if (s[i]>='0' && s[i]<='8')
-					{
-						s[i]++;
-						if (forceDouble(to!double(s[0..i+1]))==v)
-							s = s[0..i+1];
-						else
-							s[i]--;
-					}
-				while (s.length>2 && s[$-1]!='.' && forceDouble(to!double(s[0..$-1]))==v)
-					s = s[0..$-1];
+				if (forceDouble(to!double(s)) == v)
+				{
+					foreach_reverse (i; 1..s.length)
+						if (s[i]>='0' && s[i]<='8')
+						{
+							s[i]++;
+							if (forceDouble(to!double(s[0..i+1]))==v)
+								s = s[0..i+1];
+							else
+								s[i]--;
+						}
+					while (s.length>2 && s[$-1]!='.' && forceDouble(to!double(s[0..$-1]))==v)
+						s = s[0..$-1];
+				}
+				else
+				{
+					buf.pos = 0;
+					formattedWrite(&buf, "%.13a", v);
+					s = buf.data();
+					auto n = forceDouble(to!double(s));
+					assert(n == v,
+						"Can't print precise representation of double: %(%02X %) (%.18g) => %s => %(%02X %) (%.18g)".format(
+							cast(ubyte[])cast(void[])[v], v,
+							s,
+							cast(ubyte[])cast(void[])[n], n,
+						));
+				}
 			}
 			sb ~= s;
 		}
@@ -1545,8 +1562,7 @@ final class Disassembler
 
 	void dumpScript(StringBuilder sb, ASProgram.Script script, uint index)
 	{
-		sb ~= "script ; ";
-		sb.write(index);
+		sb ~= "script";
 		sb.indent++; sb.newLine();
 		dumpMethod(sb, script.sinit, "sinit");
 		dumpTraits(sb, script.traits, true);
@@ -1666,7 +1682,8 @@ final class Disassembler
 
 			if (instruction.opcode == Opcode.OP_raw)
 			{
-				sb ~= "; 0x%02X".format(instruction.arguments[0].ubytev);
+				if (dumpRaw)
+					sb ~= "; 0x%02X".format(instruction.arguments[0].ubytev);
 				sb.newLine();
 				continue;
 			}
@@ -1684,6 +1701,9 @@ final class Disassembler
 						case OpcodeArgumentType.Unknown:
 							throw new Exception("Don't know how to disassemble OP_" ~ opcodeInfo[instruction.opcode].name);
 
+						case OpcodeArgumentType.ByteLiteral:
+							sb.write(instruction.arguments[i].bytev);
+							break;
 						case OpcodeArgumentType.UByteLiteral:
 							sb.write(instruction.arguments[i].ubytev);
 							break;

lzma.d

@@ -1,5 +1,5 @@
 /*
- *  Copyright 2012, 2013, 2014 Vladimir Panteleev <vladimir@thecybershadow.net>
+ *  Copyright 2012, 2013, 2014, 2016 Vladimir Panteleev <vladimir@thecybershadow.net>
  *  This file is part of RABCDAsm.
  *
  *  RABCDAsm is free software: you can redistribute it and/or modify
@@ -23,6 +23,7 @@ version(HAVE_LZMA) {} else static assert(0, "LZMA is not available (HAVE_LZMA ve
 import deimos.lzma;
 import std.conv;
 import std.exception;
+import std.string : format;
 
 version (Windows)
 	{ pragma(lib, "liblzma"); }
@@ -40,21 +41,30 @@ static assert(LZMAHeader.sizeof == 13);
 
 ubyte[] lzmaDecompress(LZMAHeader header, in ubyte[] compressedData)
 {
-	enforce(header.decompressedSize > 0, "Decompression with unknown size is unsupported");
-
 	lzma_stream strm;
 	lzmaEnforce(lzma_alone_decoder(&strm, ulong.max), "lzma_alone_decoder");
 	scope(exit) lzma_end(&strm);
 
-	auto outBuf = new ubyte[to!size_t(header.decompressedSize)];
-	strm.next_out  = outBuf.ptr;
-	strm.avail_out = outBuf.length;
+	auto outBuf = new ubyte[1024];
+	size_t pos = 0;
 
 	void decompress(in ubyte[] chunk)
 	{
-		strm.next_in  = chunk.ptr;
-		strm.avail_in = chunk.length;
-		lzmaEnforce!true(lzma_code(&strm, lzma_action.LZMA_RUN), "lzma_code (LZMA_RUN)");
+		strm.next_in   = chunk.ptr;
+		strm.avail_in  = chunk.length;
+
+	again:
+		strm.next_out  = outBuf.ptr    + pos;
+		strm.avail_out = outBuf.length - pos;
+		auto ret = lzma_code(&strm, lzma_action.LZMA_RUN);
+		pos = strm.next_out - outBuf.ptr;
+
+		if (ret == lzma_ret.LZMA_OK && strm.avail_in && !strm.avail_out)
+		{
+			outBuf.length = outBuf.length * 2;
+			goto again;
+		}
+		lzmaEnforce!true(ret, "lzma_code (LZMA_RUN)");
 		enforce(strm.avail_in == 0, "Not all data was read");
 	}
 
@@ -65,7 +75,13 @@ ubyte[] lzmaDecompress(LZMAHeader header, in ubyte[] compressedData)
 
 	lzmaEnforce!true(lzma_code(&strm, lzma_action.LZMA_FINISH), "lzma_code (LZMA_FINISH)");
 
-	enforce(strm.avail_out == 0, "Decompressed size mismatch");
+//	enforce(strm.avail_out == 0,
+//		"Decompressed size mismatch (expected %d/0x%X, got %d/0x%X)".format(
+//			outBuf.length, outBuf.length,
+//			outBuf.length - strm.avail_out, outBuf.length - strm.avail_out,
+//	));
+
+	outBuf = outBuf[0..pos];
 
 	return outBuf;
 }

swffile.d

@@ -1,5 +1,5 @@
 /*
- *  Copyright 2010, 2011, 2012 Vladimir Panteleev <vladimir@thecybershadow.net>
+ *  Copyright 2010, 2011, 2012, 2016 Vladimir Panteleev <vladimir@thecybershadow.net>
  *  This file is part of RABCDAsm.
  *
  *  RABCDAsm is free software: you can redistribute it and/or modify
@@ -20,6 +20,7 @@ module swffile;
 
 import std.conv;
 import std.exception;
+import std.string : format;
 import std.zlib;
 import zlibx;
 version (HAVE_LZMA) import lzma;
@@ -115,7 +116,9 @@ private final class SWFReader
 			else
 				enforce(false, "This version was built without LZMA support");
 		}
-		enforce(swf.header.fileLength == buf.length, "Incorrect file length in file header");
+		//enforce(swf.header.fileLength == buf.length,
+		//	"Incorrect file length in file header (expected %d, got %d)"
+		//	.format(swf.header.fileLength , buf.length));
 		swf.frameSize = readRect();
 		swf.frameRate = readU16();
 		swf.frameCount = readU16();