Update Changelog

This change may also improve performance.

CHANGELOG.md

@@ -1,6 +1,21 @@
 RABCDAsm Changelog
 ==================
 
+RABCDAsm v1.16 (2014.04.21)
+---------------------------
+
+ * Fix handling of TypeName-kind Multinames with null parameters
+ * Fix v1.15 regression in handling very long paths on Windows
+   (DMD 2.066 is required when building from source for this to work)
+
+RABCDAsm v1.15 (2014.01.11)
+---------------------------
+
+ * Fix building on systems with a noexec `/tmp/`
+ * Improve compatibility with 3rd-party players
+   * Don't emit forward references in TypeName-kind Multinames
+ * Improve performance and memory usage
+
 RABCDAsm v1.14 (2013.08.21)
 ---------------------------
 

README.md

@@ -74,6 +74,9 @@ compilation flags (`-O -inline`).
 To be able to manipulate SWF files packed with LZMA compression, you'll need 
 to have the liblzma library and development files installed on your system.
 
+Note: DMD 2.066 (as of this moment not yet released) is required for long path
+support on Windows since RABCDAsm 1.16.
+
   [d2]: http://dlang.org/
   [dmd]: http://www.digitalmars.com/d/download.html
   [gdc]: http://bitbucket.org/goshawk/gdc/

asprogram.d

@@ -1,5 +1,5 @@
 /*
- *  Copyright 2010, 2011, 2012 Vladimir Panteleev <vladimir@thecybershadow.net>
+ *  Copyright 2010, 2011, 2012, 2013, 2014 Vladimir Panteleev <vladimir@thecybershadow.net>
  *  This file is part of RABCDAsm.
  *
  *  RABCDAsm is free software: you can redistribute it and/or modify
@@ -157,6 +157,11 @@ final class ASProgram
 		uint id; // file index
 
 		MethodBody vbody;
+
+		override string toString() const
+		{
+			return name;
+		}
 	}
 
 	static class Metadata
@@ -165,6 +170,7 @@ final class ASProgram
 		string[] keys, values;
 
 		mixin AutoCompare;
+		mixin AutoToString;
 		mixin ProcessAllData;
 	}
 
@@ -232,7 +238,7 @@ final class ASProgram
 
 		Instance instance;
 
-		override string toString()
+		override string toString() const
 		{
 			return instance.name.toString();
 		}
@@ -737,135 +743,79 @@ private final class AStoABC : ASVisitor
 	}
 
 	/// Maintain an unordered set of values; sort/index by usage count
-	struct ConstantPool(T, bool haveNull = true)
+	struct Pool(T, bool byRef, bool haveNull = true)
 	{
-		alias immutable(T) I;
+		static if (byRef)
+			alias void* Key;
+		else
+			alias immutable(T) Key;
+
+		Key toKey(T value)
+		{
+			return cast(Key)value;
+		}
 
 		struct Entry
 		{
 			uint hits;
 			T value;
-			uint index;
+			size_t addIndex, index;
+			Key[] parents;
 
-			mixin AutoCompare;
-
-			R processData(R, string prolog, string epilog, H)(ref H handler) const
-			{
-				mixin(prolog);
-				mixin(addAutoField("hits", true));
-				mixin(addAutoField("value"));
-				mixin(epilog);
-			}
+			mixin AutoToString;
+			mixin ProcessAllData;
 		}
 
-		Entry[immutable(T)] pool;
+		Entry[Key] pool;
 		T[] values;
 
 		bool add(T value) // return true if added
 		{
-			if (haveNull && isNull(value))
+			if ((haveNull || byRef) && isNull(value))
 				return false;
-			auto cp = cast(I)value in pool;
-			if (cp is null)
+			auto p = toKey(value) in pool;
+			if (p is null)
 			{
-				pool[cast(I)value] = Entry(1, value);
+				pool[toKey(value)] = Entry(1, value, pool.length);
 				return true;
 			}
 			else
 			{
-				cp.hits++;
+				p.hits++;
 				return false;
 			}
 		}
 
 		bool notAdded(T value)
 		{
-			auto ep = cast(I)value in pool;
+			static if (haveNull || byRef)
+				if (isNull(value))
+					return false;
+			auto ep = toKey(value) in pool;
 			if (ep)
 				ep.hits++;
-			return !((haveNull && isNull(value)) || ep);
-		}
-
-		void finalize()
-		{
-			auto all = pool.values;
-			all.sort;
-			enum { NullOffset = haveNull ? 1 : 0 }
-			values.length = all.length + NullOffset;
-			foreach (uint i, ref c; all)
-			{
-				pool[cast(I)c.value].index = i + NullOffset;
-				values[i + NullOffset] = c.value;
-			}
-		}
-
-		uint get(T value)
-		{
-			if (haveNull && isNull(value))
-				return 0;
-			return pool[cast(I)value].index;
-		}
-	}
-
-	/// Pair an index with class instances
-	struct ReferencePool(T : Object)
-	{
-		struct Entry
-		{
-			uint hits;
-			void* object;
-			uint addIndex, index;
-			void*[] parents;
-
-			mixin AutoToString;
-			mixin ProcessAllData;
-		}
-
-		Entry[void*] pool;
-		T[] objects;
-
-		bool add(T obj) // return true if added
-		{
-			if (obj is null)
-				return false;
-			auto p = cast(void*)obj;
-			auto rp = p in pool;
-			if (rp is null)
-			{
-				pool[p] = Entry(1, p, to!uint(pool.length));
-				return true;
-			}
-			else
-			{
-				rp.hits++;
-				return false;
-			}
-		}
-
-		bool notAdded(T obj)
-		{
-			auto ep = (cast(void*)obj) in pool;
-			if (ep)
-				ep.hits++;
-			return !(obj is null || ep);
+			return !ep;
 		}
 
+		/// "from" is child of "to", thus "from" must come after "to"
 		void registerDependency(T from, T to)
 		{
-			auto pfrom = (cast(void*)from) in pool;
+			auto pfrom = toKey(from) in pool;
 			assert(pfrom, "Unknown dependency source");
-			auto vto = cast(void*)to;
+			auto vto = toKey(to);
 			auto pto = vto in pool;
 			assert(pto, "Unknown dependency target");
-			assert(!pfrom.parents.contains(vto), "Dependency already set");
+			assert(!pfrom.parents.contains!Key(vto), "Dependency already set");
 			pfrom.parents ~= vto;
 		}
 
-		T[] getPreliminaryObjects()
+		T[] getPreliminaryValues()
 		{
 			return cast(T[])pool.keys;
 		}
 
+		enum { NullOffset = haveNull ? 1 : 0 }
+
 		void finalize()
 		{
 			// create array
@@ -875,7 +825,17 @@ private final class AStoABC : ASVisitor
 				all[i++] = &e;
 
 			// sort
-			sort!q{a.hits > b.hits || (a.hits == b.hits && a.addIndex < b.addIndex)}(all);
+			static if (is(T == double))
+			{
+				static ulong repr(double d) { static assert(double.sizeof == ulong.sizeof); return *cast(ulong*)(&d); }
+				static bool sortPred(Entry* a, Entry* b) { return a.hits > b.hits || (a.hits == b.hits && repr(a.value) < repr(b.value)); }
+			}
+			else
+			static if (byRef)
+				enum sortPred = q{a.hits > b.hits || (a.hits == b.hits && a.addIndex < b.addIndex)};
+			else
+				enum sortPred = q{a.hits > b.hits || (a.hits == b.hits && a.value < b.value)};
+			all.sort!sortPred();
 
 			// topographical sort
 			topSort:
@@ -896,18 +856,29 @@ private final class AStoABC : ASVisitor
 					}
 				}
 
-			objects.length = i;
+			values.length = NullOffset + i;
 			foreach (j, e; all)
-				objects[j] = cast(T)e.object;
+				values[NullOffset + j] = e.value;
 		}
 
-		uint get(T obj)
+		uint get(T value)
 		{
-			assert(obj !is null, "Trying to get index of null object");
-			return pool[cast(void*)obj].index;
+			if (haveNull && isNull(value))
+				return 0;
+			return NullOffset + to!uint(pool[toKey(value)].index);
 		}
 	}
 
+	template ConstantPool(T, bool haveNull=true)
+	{
+		alias Pool!(T, false, haveNull) ConstantPool;
+	}
+
+	template ReferencePool(T : Object)
+	{
+		alias Pool!(T, true, false) ReferencePool;
+	}
+
 	ConstantPool!(long) ints;
 	ConstantPool!(ulong) uints;
 	ConstantPool!(double) doubles;
@@ -998,7 +969,7 @@ private final class AStoABC : ASVisitor
 	{
 		ASProgram.Class[ASProgram.Multiname] classByName;
 
-		ASProgram.Class[] classObjects = classes.getPreliminaryObjects();
+		ASProgram.Class[] classObjects = classes.getPreliminaryValues();
 		foreach (c; classObjects)
 		{
 			assert(!(c.instance.name in classByName), "Duplicate class name " ~ c.instance.name.toString());
@@ -1016,6 +987,18 @@ private final class AStoABC : ASVisitor
 					}
 	}
 
+	void registerMultinameDependencies()
+	{
+		foreach (m; multinames.getPreliminaryValues())
+			if (m.kind == ASType.TypeName)
+			{
+				multinames.registerDependency(m, m.vTypeName.name);
+				foreach (t; m.vTypeName.params)
+					if (t)
+						multinames.registerDependency(m, t);
+			}
+	}
+
 	this(ASProgram as)
 	{
 		super(as);
@@ -1027,6 +1010,7 @@ private final class AStoABC : ASVisitor
 		super.run();
 
 		registerClassDependencies();
+		registerMultinameDependencies();
 
 		ints.finalize();
 		uints.finalize();
@@ -1115,8 +1099,8 @@ private final class AStoABC : ASVisitor
 
 		ASProgram.MethodBody[] bodies;
 
-		abc.methods.length = methods.objects.length;
-		foreach (i, o; methods.objects)
+		abc.methods.length = methods.values.length;
+		foreach (i, o; methods.values)
 		{
 			auto n = &abc.methods[i];
 			n.paramTypes.length = o.paramTypes.length;
@@ -1139,8 +1123,8 @@ private final class AStoABC : ASVisitor
 				bodies ~= o.vbody;
 		}
 
-		abc.instances.length = classes.objects.length;
-		foreach (i, c; classes.objects)
+		abc.instances.length = classes.values.length;
+		foreach (i, c; classes.values)
 		{
 			auto o = c.instance;
 			auto n = &abc.instances[i];
@@ -1156,8 +1140,8 @@ private final class AStoABC : ASVisitor
 			n.traits = convertTraits(o.traits);
 		}
 
-		abc.classes.length = classes.objects.length;
-		foreach (i, o; classes.objects)
+		abc.classes.length = classes.values.length;
+		foreach (i, o; classes.values)
 		{
 			auto n = &abc.classes[i];
 			n.cinit = methods.get(o.cinit);

autodata.d

@@ -1,5 +1,5 @@
 /*
- *  Copyright 2010, 2011, 2012 Vladimir Panteleev <vladimir@thecybershadow.net>
+ *  Copyright 2010, 2011, 2012, 2014 Vladimir Panteleev <vladimir@thecybershadow.net>
  *  This file is part of RABCDAsm.
  *
  *  RABCDAsm is free software: you can redistribute it and/or modify
@@ -222,8 +222,8 @@ struct ToStringDataHandler
 					_AutoDataResult ~= to!string(" ~ name ~ ");
 		";
 */
-		static if (is(typeof(T.init.toString())))
-			enum getMixinSingle = "_AutoDataResult ~= " ~ name ~ ".toString();";
+		static if (is(typeof(T.init is null)))
+			enum getMixinSingle = "_AutoDataResult ~= " ~ name ~ " ? to!string(" ~ name ~ ") : `null`;";
 		else
 			enum getMixinSingle = "_AutoDataResult ~= to!string(" ~ name ~ ");";
 	}

build_rabcdasm.d

@@ -53,10 +53,11 @@ void compile(string program)
 
 void test(string code, string extraFlags=null)
 {
-	const FN = "test.d";
+	const BASE = "build_rabcdasm_buildtest";
+	const FN = BASE ~ ".d";
 	std.file.write(FN, code);
-	scope(exit) remove(FN);
-	enforce(system(format("rdmd --force --compiler=%s %s %s %s", compiler, flags, extraFlags, FN)) == 0, "Test failed");
+	scope(exit) foreach (de; dirEntries(".", BASE ~ "*", SpanMode.shallow)) remove(de.name);
+	enforce(system(format("rdmd --force --compiler=%s -od. %s %s %s", compiler, flags, extraFlags, FN)) == 0, "Test failed");
 	stderr.writeln(" >>> OK");
 }
 

common.d

@@ -1,5 +1,5 @@
 /*
- *  Copyright 2012 Vladimir Panteleev <vladimir@thecybershadow.net>
+ *  Copyright 2012, 2014 Vladimir Panteleev <vladimir@thecybershadow.net>
  *  This file is part of RABCDAsm.
  *
  *  RABCDAsm is free software: you can redistribute it and/or modify
@@ -18,9 +18,10 @@
 
 module common;
 
-import std.path;
-import std.string;
 import std.array;
+import std.path;
+import std.stdio;
+import std.string;
 
 string longPath(string s)
 {
@@ -34,3 +35,56 @@ string longPath(string s)
 	else
 		return s;
 }
+
+File openFile(string fn, string mode)
+{
+	File f;
+	static if (is(typeof(&f.windowsHandleOpen)))
+	{
+		import core.sys.windows.windows;
+
+		import std.exception;
+		import std.utf;
+		import std.windows.syserror;
+
+		string winMode;
+		foreach (c; mode)
+			switch (c)
+			{
+				case 'r':
+				case 'w':
+				case 'a':
+				case '+':
+					winMode ~= c;
+					break;
+				case 'b':
+				case 't':
+					break;
+				default:
+					assert(false, "Unknown character in mode");
+			}
+		DWORD access, creation;
+		bool append;
+		switch (winMode)
+		{
+			case "r" : access = GENERIC_READ                ; creation = OPEN_EXISTING; break;
+			case "r+": access = GENERIC_READ | GENERIC_WRITE; creation = OPEN_EXISTING; break;
+			case "w" : access =                GENERIC_WRITE; creation = OPEN_ALWAYS  ; break;
+			case "w+": access = GENERIC_READ | GENERIC_WRITE; creation = OPEN_ALWAYS  ; break;
+			case "a" : access =                GENERIC_WRITE; creation = OPEN_ALWAYS  ; append = true; break;
+			case "a+": assert(false, "Not implemented"); // requires two file pointers
+			default: assert(false, "Bad file mode: " ~ mode);
+		}
+
+		auto pathW = toUTF16z(longPath(fn));
+		auto h = CreateFileW(pathW, access, FILE_SHARE_READ, null, creation, 0, HANDLE.init);
+		enforce(h != INVALID_HANDLE_VALUE, "Failed to open file \"" ~ fn ~ "\": " ~ sysErrorString(GetLastError()));
+
+		assert(!append, "'a' mode not implemented");
+
+		f.windowsHandleOpen(h, mode);
+	}
+	else
+		f.open(fn, mode);
+	return f;
+}

disassembler.d

@@ -1,5 +1,5 @@
 /*
- *  Copyright 2010, 2011, 2012, 2013 Vladimir Panteleev <vladimir@thecybershadow.net>
+ *  Copyright 2010, 2011, 2012, 2013, 2014 Vladimir Panteleev <vladimir@thecybershadow.net>
  *  This file is part of RABCDAsm.
  *
  *  RABCDAsm is free software: you can redistribute it and/or modify
@@ -18,14 +18,16 @@
 
 module disassembler;
 
-import std.file;
-import std.string;
+import std.algorithm;
 import std.array;
 import std.conv;
-import std.exception;
-import std.algorithm;
-import std.path;
 import std.digest.md;
+import std.exception;
+import std.file;
+import std.format;
+import std.path;
+import std.stdio;
+import std.string;
 import abcfile;
 import asprogram;
 import autodata;
@@ -35,37 +37,20 @@ alias std.array.join join;
 
 final class StringBuilder
 {
-	char[] buf;
-	size_t pos;
+	enum BUF_SIZE = 256*1024;
+
+	static char[] buf;
+	static size_t pos;
+
 	string filename;
+	File file;
 
 	this(string filename)
 	{
+		this.filename = filename;
 		if (exists(longPath(filename)))
 			throw new Exception(filename ~ " exists");
-		this.filename = filename;
-		buf.length = 1024;
-	}
 
-	void opCatAssign(string s)
-	{
-		checkIndent();
-		auto end = pos + s.length;
-		while (buf.length < end)
-			buf.length = buf.length*2;
-		buf[pos..end] = s[];
-		pos = end;
-	}
-
-	void opCatAssign(char c)
-	{
-		if (buf.length < pos+1) // speed hack: no loop, no indent check
-			buf.length = buf.length*2;
-		buf[pos++] = c;
-	}
-
-	void save()
-	{
 		string[] dirSegments = split(filename, "/");
 		for (int l=0; l<dirSegments.length-1; l++)
 		{
@@ -73,7 +58,58 @@ final class StringBuilder
 			if (subdir.length && !exists(longPath(subdir)))
 				mkdir(longPath(subdir));
 		}
-		write(longPath(filename), buf[0..pos]);
+		file = openFile(filename, "wb");
+		assert(!pos, "Opening new file with unflushed buffer");
+	}
+
+	static this()
+	{
+		buf = new char[BUF_SIZE];
+	}
+
+	void put(in char[] s)
+	{
+		checkIndent();
+		auto end = pos + s.length;
+		if (end > buf.length)
+		{
+			flush();
+			end = s.length;
+			while (end > buf.length)
+				buf.length = buf.length*2;
+		}
+		buf[pos..end] = s[];
+		pos = end;
+	}
+
+	void put(char c)
+	{
+		if (pos == buf.length) // speed hack: no indent check
+			flush();
+		buf[pos++] = c;
+	}
+
+	alias put opCatAssign;
+
+	void write(T)(T v)
+	{
+		checkIndent();
+		formattedWrite(this, "%s", v);
+	}
+
+	void flush()
+	{
+		if (pos)
+		{
+			file.rawWrite(buf[0..pos]);
+			pos = 0;
+		}
+	}
+
+	void save()
+	{
+		flush();
+		file.close();
 	}
 
 	int indent;
@@ -473,7 +509,8 @@ final class RefBuilder : ASTraitsVisitor
 			}
 			else
 			{
-				if ((*pset)[priority].length==0 || (*pset)[priority][$-1] != context) // Optimization: don't add contexts identical to the last added
+				static bool rawEqual(T)(T[] arr1, T[] arr2) { return cast(ubyte[])arr1 == cast(ubyte[])arr2; }
+				if ((*pset)[priority].length==0 || !rawEqual((*pset)[priority][$-1], context)) // Optimization: don't add contexts identical to the last added
 					(*pset)[priority] ~= context.dup;
 				return false;
 			}
@@ -793,7 +830,7 @@ final class RefBuilder : ASTraitsVisitor
 			return;
 		}
 
-		auto myContext = context[0..myPos].dup;
+		auto myContext = context[0..myPos];
 		namespaces[ns.kind].add(ns.id, myContext, priority);
 	}
 
@@ -987,6 +1024,7 @@ final class Disassembler
 				base = dirName(base), up++;
 			string rel  = replicate("../", up) ~ full[base.length+1..$];
 
+			mainsb.flush();
 			StringBuilder sb = new StringBuilder(full);
 			callback(sb);
 			sb.save();
@@ -1075,7 +1113,7 @@ final class Disassembler
 		if (v == ABCFile.NULL_INT)
 			sb ~= "null";
 		else
-			sb ~= to!string(v);
+			sb.write(v);
 	}
 
 	void dumpUInt(StringBuilder sb, ulong v)
@@ -1083,7 +1121,16 @@ final class Disassembler
 		if (v == ABCFile.NULL_UINT)
 			sb ~= "null";
 		else
-			sb ~= to!string(v);
+			sb.write(v);
+	}
+
+	static struct StaticBuf(T, size_t size)
+	{
+		T[size] buf;
+		size_t pos;
+		void put(T v) { buf[pos++] = v; }
+		void put(in T[] v) { buf[pos..pos+v.length] = v[]; pos+=v.length; }
+		T[] data() { return buf[0..pos]; }
 	}
 
 	void dumpDouble(StringBuilder sb, double v)
@@ -1092,14 +1139,22 @@ final class Disassembler
 			sb ~= "null";
 		else
 		{
-			string s = format("%.18g", v);
+			StaticBuf!(char, 64) buf;
+			formattedWrite(&buf, "%.18g", v);
+			char[] s = buf.data();
 
 			static double forceDouble(double d) { static double n; n = d; return n; }
 			if (s != "nan" && s != "inf" && s != "-inf")
 			{
 				foreach_reverse (i; 1..s.length)
-					if (s[i]>='0' && s[i]<='8' && forceDouble(to!double(s[0..i] ~ cast(char)(s[i]+1)))==v)
-						s = s[0..i] ~ cast(char)(s[i]+1);
+					if (s[i]>='0' && s[i]<='8')
+					{
+						s[i]++;
+						if (forceDouble(to!double(s[0..i+1]))==v)
+							s = s[0..i+1];
+						else
+							s[i]--;
+					}
 				while (s.length>2 && s[$-1]!='.' && forceDouble(to!double(s[0..$-1]))==v)
 					s = s[0..$-1];
 			}
@@ -1491,7 +1546,7 @@ final class Disassembler
 	void dumpScript(StringBuilder sb, ASProgram.Script script, uint index)
 	{
 		sb ~= "script ; ";
-		sb ~= to!string(index);
+		sb.write(index);
 		sb.indent++; sb.newLine();
 		dumpMethod(sb, script.sinit, "sinit");
 		dumpTraits(sb, script.traits, true);
@@ -1509,12 +1564,12 @@ final class Disassembler
 	void dumpLabel(StringBuilder sb, ref ABCFile.Label label)
 	{
 		sb ~= 'L';
-		sb ~= to!string(label.index);
+		sb.write(label.index);
 		if (label.offset != 0)
 		{
 			if (label.offset > 0)
 				sb ~= '+';
-			sb ~= to!string(label.offset);
+			sb.write(label.offset);
 		}
 	}
 
@@ -1601,7 +1656,7 @@ final class Disassembler
 			{
 				sb.noIndent();
 				sb ~= 'L';
-				sb ~= to!string(ii);
+				sb.write(ii);
 				sb ~= ':';
 				sb.newLine();
 			}
@@ -1629,13 +1684,13 @@ final class Disassembler
 							throw new Exception("Don't know how to disassemble OP_" ~ opcodeInfo[instruction.opcode].name);
 
 						case OpcodeArgumentType.UByteLiteral:
-							sb ~= to!string(instruction.arguments[i].ubytev);
+							sb.write(instruction.arguments[i].ubytev);
 							break;
 						case OpcodeArgumentType.IntLiteral:
-							sb ~= to!string(instruction.arguments[i].intv);
+							sb.write(instruction.arguments[i].intv);
 							break;
 						case OpcodeArgumentType.UIntLiteral:
-							sb ~= to!string(instruction.arguments[i].uintv);
+							sb.write(instruction.arguments[i].uintv);
 							break;
 
 						case OpcodeArgumentType.Int: