Update Changelog

This is to allow files disassembled on one platform to
work on other platforms.

CHANGELOG.md

@@ -1,6 +1,17 @@
 RABCDAsm Changelog
 ==================
 
+RABCDAsm v1.11 (2012.03.15)
+---------------------------
+
+ * Fixed v1.10 hierarchy flattening regression
+ * Improved refid generation for protected namespaces
+ * Improved handling of overflown signed integers
+ * Fixed metadata handling (caused by error in ABC format specification)
+ * Removed path length limitation on Windows
+ * Improved filename sanitization
+ * Updated asasm.hrc
+
 RABCDAsm v1.10 (2012.02.29)
 ---------------------------
 

README.md

@@ -304,6 +304,7 @@ Directives start with a `#`, followed by a word identifying the directive:
     variable *word*.
   * `#unset` *word* - deletes the variable *word*.
   * `#privatens` defines a private namespace alias, as described above.
+  * `#version` specifies the version of the disassembly.
 
 ### Variables
 

abcfile.d

@@ -131,13 +131,8 @@ class ABCFile
 
 	struct Metadata
 	{
-		struct Item
-		{
-			uint key, value;
-		}
-
 		uint name;
-		Item[] items;
+		uint[] keys, values;
 	}
 
 	struct Instance
@@ -1023,6 +1018,11 @@ private final class ABCReader
 
 	/// Note: may return values larger than 0xFFFFFFFF.
 	ulong readU32()
+	out(result)
+	{
+		assert(result <= ABCFile.MAX_UINT);
+	}
+	body
 	{
 		ulong next() { return readU8(); } // force ulong
 
@@ -1042,10 +1042,15 @@ private final class ABCReader
 	}
 
 	long readS32()
+	out(result)
 	{
-		ulong l = readU32();
-		if (l & 0xFFFFFFFF00000000) // preserve unused bits
-			return cast(long)l;
+		assert(result >= ABCFile.MIN_INT && result <= ABCFile.MAX_INT);
+	}
+	body
+	{
+		auto l = readU32();
+		if (l & 0xFFFFFFFF_00000000) // preserve unused bits
+			return l | 0xFFFFFFF0_00000000;
 		else
 			return cast(int)l;
 	}
@@ -1185,12 +1190,11 @@ private final class ABCReader
 	{
 		ABCFile.Metadata r;
 		r.name = readU30();
-		r.items.length = readU30();
-		foreach (ref value; r.items)
-		{
-			value.key = readU30();
-			value.value = readU30();
-		}
+		r.keys.length = r.values.length = readU30();
+		foreach (ref key; r.keys)
+			key = readU30();
+		foreach (ref value; r.values)
+			value = readU30();
 		return r;
 	}
 
@@ -1676,12 +1680,12 @@ private final class ABCWriter
 	void writeMetadata(ref ABCFile.Metadata v)
 	{
 		writeU30(v.name);
-		writeU30(v.items.length);
-		foreach (ref value; v.items)
-		{
-			writeU30(value.key);
-			writeU30(value.value);
-		}
+		assert(v.keys.length == v.values.length);
+		writeU30(v.keys.length);
+		foreach (key; v.keys)
+			writeU30(key);
+		foreach (value; v.values)
+			writeU30(value);
 	}
 
 	void writeInstance(ref ABCFile.Instance v)

asasm.hrc

@@ -4,7 +4,7 @@
 <hrc version="take5" xmlns="http://colorer.sf.net/2003/hrc"
      xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
      xsi:schemaLocation="http://colorer.sf.net/2003/hrc http://colorer.sf.net/2003/hrc.xsd">
-  
+
   <type name="asasm">
     <annotation><documentation>
       ActionScript Assembler Syntax (RABCDAsm variant)
@@ -31,20 +31,24 @@
 
       <!-- Decimal numbers -->
       <regexp match="/\b[0-9]+\b/" region0="def:NumberDec"/>
-      
+
       <!-- Paired brackets -->
       <block start="/(\()/" end="/(\))/" scheme="asasm" region00="def:Symbol" region01="def:PairStart" region10="def:Symbol" region11="def:PairEnd"/>
       <block start="/(\[)/" end="/(\])/" scheme="asasm" region00="def:Symbol" region01="def:PairStart" region10="def:Symbol" region11="def:PairEnd"/>
       <block start="/(\<)/" end="/(\>)/" scheme="asasm" region00="def:Symbol" region01="def:PairStart" region10="def:Symbol" region11="def:PairEnd"/>
 
       <!-- Outliner for refids -->
-      <regexp match="/(\brefid\b)\s*((?{def:StringEdge}&#34;))((?{def:String}[^&#34;]*))((?{def:StringEdge}&#34;))/" region1="def:Keyword" region2="def:PairStart" region3="def:Outlined" region4="def:PairEnd"/> <!-- BUG: does not support escape sequences -->
+      <!--regexp match="/(\brefid\b)\s*((?{def:StringEdge}&#34;))((?{def:String}[^&#34;]*))((?{def:StringEdge}&#34;))/" region1="def:Keyword" region2="def:PairStart" region3="def:Outlined" region4="def:PairEnd"/--> <!-- BUG: does not support escape sequences -->
+      <regexp
+        match="/(\brefid\b)\M\s*&#34;(.*)&#34;$/"
+        region1="def:Keyword"
+        region2="def:Outlined"/>
       <!--block start="/(\brefid\b)\s*(?{def:StringEdge}&#34;)/" end="/(?{def:StringEdge}&#34;)/" scheme="StringContent" region="def:Outlined" inner-region="yes" region00="def:PairStart" region10="def:PairEnd"/-->
 
       <!-- Symbol/keyword highlighting below -->
 
       <keywords region="def:Symbol">
-         <symb name=","/>
+        <symb name=","/>
       </keywords>
 
       <keywords ignorecase="no" region="def:Directive">
@@ -55,6 +59,7 @@
         <word name="#privatens"/>
         <word name="#set"/>
         <word name="#unset"/>
+        <word name="#version"/>
       </keywords>
 
       <keywords ignorecase="no" region="def:Keyword">

asprogram.d

@@ -1,5 +1,5 @@
 /*
- *  Copyright 2010, 2011 Vladimir Panteleev <vladimir@thecybershadow.net>
+ *  Copyright 2010, 2011, 2012 Vladimir Panteleev <vladimir@thecybershadow.net>
  *  This file is part of RABCDAsm.
  *
  *  RABCDAsm is free software: you can redistribute it and/or modify
@@ -159,16 +159,8 @@ final class ASProgram
 
 	static class Metadata
 	{
-		struct Item
-		{
-			string key, value;
-
-			mixin AutoCompare;
-			mixin ProcessAllData;
-		}
-
 		string name;
-		Item[] items;
+		string[] keys, values;
 
 		mixin AutoCompare;
 		mixin ProcessAllData;
@@ -477,12 +469,12 @@ private final class ABCtoAS
 	{
 		auto n = new ASProgram.Metadata();
 		n.name = abc.strings[md.name];
-		n.items.length = md.items.length;
-		foreach (j, ref item; md.items)
-		{
-			n.items[j].key = abc.strings[item.key];
-			n.items[j].value = abc.strings[item.value];
-		}
+		n.keys.length = md.keys.length;
+		foreach (j, key; md.keys)
+			n.keys[j] = abc.strings[key];
+		n.values.length = md.values.length;
+		foreach (j, value; md.values)
+			n.values[j] = abc.strings[value];
 		return n;
 	}
 
@@ -1110,12 +1102,12 @@ private final class AStoABC : ASVisitor
 		{
 			auto n = &abc.metadata[i];
 			n.name = strings.get(m.name);
-			n.items.length = m.items.length;
-			foreach (j, ref item; m.items)
-			{
-				n.items[j].key = strings.get(m.items[j].key);
-				n.items[j].value = strings.get(m.items[j].value);
-			}
+			n.keys.length = m.keys.length;
+			foreach (j, key; m.keys)
+				n.keys[j] = strings.get(key);
+			n.values.length = m.values.length;
+			foreach (j, value; m.values)
+				n.values[j] = strings.get(value);
 		}
 
 		ASProgram.MethodBody[] bodies;
@@ -1458,11 +1450,10 @@ class ASVisitor : ASTraitsVisitor
 		if (metadata)
 		{
 			visitString(metadata.name);
-			foreach (ref item; metadata.items)
-			{
-				visitString(item.key);
-				visitString(item.value);
-			}
+			foreach (key; metadata.keys)
+				visitString(key);
+			foreach (value; metadata.values)
+				visitString(value);
 		}
 	}
 

assembler.d

@@ -25,6 +25,7 @@ import std.path;
 import std.exception;
 import abcfile;
 import asprogram;
+import common;
 
 final class Assembler
 {
@@ -52,7 +53,7 @@ final class Assembler
 
 		static File load(string filename, string[] arguments = null)
 		{
-			return fromFile(filename, cast(string)read(filename), arguments);
+			return fromFile(filename, cast(string)read(longPath(filename)), arguments);
 		}
 
 		static File fromFile(string filename, string data, string[] arguments = null)
@@ -133,6 +134,7 @@ final class Assembler
 
 	string[string] vars;
 	uint[string] privateNamespaces;
+	uint sourceVersion = 1;
 
 	void handlePreprocessor()
 	{
@@ -151,7 +153,7 @@ final class Assembler
 				break;
 			case "get":
 				auto filename = convertFilename(readString());
-				pushFile(File.fromFile(filename, toStringLiteral(cast(string)read(filename))));
+				pushFile(File.fromFile(filename, toStringLiteral(cast(string)read(longPath(filename)))));
 				break;
 			case "set":
 				vars[readWord()] = readString();
@@ -163,6 +165,9 @@ final class Assembler
 				uint index = cast(uint)readUInt();
 				privateNamespaces[readString()] = index;
 				break;
+			case "version":
+				sourceVersion = cast(uint)readUInt();
+				break;
 			default:
 				files[0].pos -= word.length;
 				throw new Exception("Unknown preprocessor declaration: " ~ word);
@@ -756,13 +761,30 @@ final class Assembler
 	{
 		auto metadata = new ASProgram.Metadata;
 		metadata.name = readString();
+		string[] items;
 		while (true)
 			switch (readWord())
 			{
 				case "item":
-					metadata.items ~= ASProgram.Metadata.Item(readString(), readString());
+					items ~= readString();
+					items ~= readString();
 					break;
 				case "end":
+					if (sourceVersion < 2)
+					{
+						metadata.keys   = items[0..$/2];
+						metadata.values = items[$/2..$];
+					}
+					else
+					{
+						metadata.keys  .length = items.length/2;
+						metadata.values.length = items.length/2;
+						foreach (i; 0..items.length/2)
+						{
+							metadata.keys  [i] = items[i*2  ];
+							metadata.values[i] = items[i*2+1];
+						}
+					}
 					return metadata;
 				default:
 					throw new Exception("Expected item or end");

common.d

@@ -0,0 +1,36 @@
+/*
+ *  Copyright 2012 Vladimir Panteleev <vladimir@thecybershadow.net>
+ *  This file is part of RABCDAsm.
+ *
+ *  RABCDAsm is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  RABCDAsm is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with RABCDAsm.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+module common;
+
+import std.path;
+import std.string;
+import std.array;
+
+string longPath(string s)
+{
+	version(Windows)
+	{
+		if (s.startsWith(`\\`))
+			return s;
+		else
+			return `\\?\` ~ s.absolutePath().buildNormalizedPath().replace(`/`, `\`);
+	}
+	else
+		return s;
+}

disassembler.d

@@ -25,9 +25,11 @@ import std.conv;
 import std.exception;
 import std.algorithm;
 import std.path;
+import std.md5;
 import abcfile;
 import asprogram;
 import autodata;
+import common;
 
 alias std.array.join join;
 
@@ -39,7 +41,7 @@ final class StringBuilder
 
 	this(string filename)
 	{
-		if (exists(filename))
+		if (exists(longPath(filename)))
 			throw new Exception(filename ~ " exists");
 		this.filename = filename;
 		buf.length = 1024;
@@ -68,10 +70,10 @@ final class StringBuilder
 		for (int l=0; l<dirSegments.length-1; l++)
 		{
 			auto subdir = join(dirSegments[0..l+1], "/");
-			if (!exists(subdir))
-				mkdir(subdir);
+			if (!exists(longPath(subdir)))
+				mkdir(longPath(subdir));
 		}
-		write(filename, buf[0..pos]);
+		write(longPath(filename), buf[0..pos]);
 	}
 
 	int indent;
@@ -213,6 +215,77 @@ final class RefBuilder : ASTraitsVisitor
 			}
 			mixin(epilog);
 		}
+
+		static bool similar(ref ContextItem i1, ref ContextItem i2)
+		{
+			if (i1.type != i2.type) return false;
+			if (i1.type == ContextItem.Type.String)
+				return i1.str == i2.str;
+			if (i1.multiname.vQName.name != i2.multiname.vQName.name) return false;
+			return nsSimilar(i1.multiname.vQName.ns, i2.multiname.vQName.ns);
+		}
+
+		// truncate=true  -> return partial ContextItem
+		// truncate=false -> return null on partial match
+		static ContextItem[] combine(bool truncate)(ref ContextItem c1, ref ContextItem c2)
+		{
+			if (similar(c1, c2))
+				return [c1];
+
+			if (c1.type != ContextItem.Type.Multiname || c2.type != ContextItem.Type.Multiname)
+				return null;
+			if (c1.multiname.kind != ASType.QName || c2.multiname.kind != ASType.QName)
+				return null;
+
+			auto ns1 = c1.multiname.vQName.ns;
+			auto ns2 = c2.multiname.vQName.ns;
+
+			if (nsSimilar(ns1, ns2) && ns1.name.length && truncate)
+			{
+				auto m = new ASProgram.Multiname;
+				m.kind = ASType.QName;
+				m.vQName.ns = ns1;
+				return [ContextItem(m)];
+			}
+
+			if (c1.multiname.vQName.name && !c2.multiname.vQName.name && truncate)
+			{
+				swap(c1, c2);
+				swap(ns1, ns2);
+			}
+
+			if (!c1.multiname.vQName.name && c2.multiname.vQName.name && nsSimilar(ns1, ns2))
+			{
+				if (truncate)
+				{
+					auto m = new ASProgram.Multiname;
+					m.kind = ASType.QName;
+					m.vQName.ns = ns1;
+					return [ContextItem(m)];
+				}
+				else
+					return [c2];
+			}
+
+			if (ns1.name.length && ns2.name.length)
+			{
+				if (ns1.name.length > ns2.name.length && truncate)
+				{
+					swap(c1, c2);
+					swap(ns1, ns2);
+				}
+
+				auto fullName1 = ns1.name ~ (c1.multiname.vQName.name ? ':' ~ c1.multiname.vQName.name : "");
+				auto fullName2 = ns2.name ~ (c2.multiname.vQName.name ? ':' ~ c2.multiname.vQName.name : "");
+				if (fullName2.startsWith(fullName1 ~ ":"))
+					return [truncate ? c1 : c2];
+			}
+
+			return null;
+		}
+
+		alias combine!true commonRoot;
+		alias combine!false deduplicate;
 	}
 
 	ContextItem[] context; // potential optimization: use array-based stack
@@ -288,8 +361,7 @@ final class RefBuilder : ASTraitsVisitor
 			return *pname;
 		}
 
-		version (Windows)
-			static string[string] filenameMappings;
+		static string[string] filenameMappings;
 
 		string getFilename(U)(U obj, string suffix)
 		{
@@ -297,24 +369,21 @@ final class RefBuilder : ASTraitsVisitor
 			assert(pname, format("Unscanned object: ", obj));
 			auto filename = *pname;
 
-			version (Windows)
+			string[] dirSegments = split(filename, "/");
+			for (int l=0; l<dirSegments.length; l++)
 			{
-				string[] dirSegments = split(filename, "/");
-				for (int l=0; l<dirSegments.length; l++)
+			again:
+				string subpath = join(dirSegments[0..l+1], "/");
+				string subpathl = toLower(subpath);
+				string* canonicalp = subpathl in filenameMappings;
+				if (canonicalp && *canonicalp != subpath)
 				{
-				again:
-					string subpath = join(dirSegments[0..l+1], "/");
-					string subpathl = toLower(subpath);
-					string* canonicalp = subpathl in filenameMappings;
-					if (canonicalp && *canonicalp != subpath)
-					{
-						dirSegments[l] = dirSegments[l] ~ "_"; // not ~=
-						goto again;
-					}
-					filenameMappings[subpathl] = subpath;
+					dirSegments[l] = dirSegments[l] ~ "_"; // not ~=
+					goto again;
 				}
-				filename = join(dirSegments, "/");
+				filenameMappings[subpathl] = subpath;
 			}
+			filename = join(dirSegments, "/");
 
 			return filename ~ "." ~ suffix ~ ".asasm";
 		}
@@ -444,32 +513,15 @@ final class RefBuilder : ASTraitsVisitor
 		if (uninteresting(c1)) return c2;
 		if (uninteresting(c2)) return c1;
 
-		static bool nsSimilar(ASProgram.Namespace ns1, ASProgram.Namespace ns2)
+		ContextItem[] c;
+		while (c.length<c1.length && c.length<c2.length)
 		{
-			if (ns1.kind==ASType.PrivateNamespace || ns2.kind==ASType.PrivateNamespace)
-				return ns1.kind==ns2.kind && ns1.privateIndex==ns2.privateIndex;
-			// ignore ns kind in other cases
-			return ns1.name == ns2.name;
-		}
-
-		static bool similar(ref ContextItem i1, ref ContextItem i2)
-		{
-			if (i1.type != i2.type) return false;
-			if (i1.type == ContextItem.Type.String)
-				return i1.str == i2.str;
-			if (i1.multiname.vQName.name != i2.multiname.vQName.name) return false;
-			return nsSimilar(i1.multiname.vQName.ns, i2.multiname.vQName.ns);
-		}
-
-		int i=0;
-		while (i<c1.length && i<c2.length && similar(c1[i], c2[i])) i++;
-		auto c = c1[0..i];
-		if (i<c1.length && i<c2.length && c1[i].type==ContextItem.Type.Multiname && c2[i].type==ContextItem.Type.Multiname && nsSimilar(c1[i].multiname.vQName.ns, c2[i].multiname.vQName.ns) && c1[i].multiname.vQName.ns.name.length)
-		{
-			auto m = new ASProgram.Multiname;
-			m.kind = ASType.QName;
-			m.vQName.ns = c1[i].multiname.vQName.ns;
-			c ~= ContextItem(m);
+			auto root = ContextItem.commonRoot(c1[c.length], c2[c.length]);
+			assert(root.length <= 1);
+			if (root.length)
+				c ~= root;
+			else
+				break;
 		}
 		return c;
 	}
@@ -568,10 +620,15 @@ final class RefBuilder : ASTraitsVisitor
 	string contextToString(ContextItem[] context, bool filename)
 	{
 		context = ContextItem.expand(this, context, false);
+		if (!context.length)
+			return null;
 
-		foreach_reverse (i, c; context)
-			if (i>0 && c==context[i-1])
-				context = context[0..i] ~ context[i+1..$];
+		foreach_reverse (i; 0..context.length-1)
+		{
+			auto root = ContextItem.deduplicate(context[i], context[i+1]);
+			if (root.length)
+				context = context[0..i] ~ root ~ context[i+2..$];
+		}
 
 		ContextItem.Segment[] segments;
 		foreach (ci; context)
@@ -584,10 +641,10 @@ final class RefBuilder : ASTraitsVisitor
 
 			string result;
 			foreach (c; s)
-				if (c == '.')
+				if (c == '.' || c == ':')
 					result ~= '/';
 				else
-				if (c == ':' || c == '\\' || c == '*' || c == '?' || c == '"' || c == '<' || c == '>' || c == '|' || c < 0x20 || c >= 0x7F || c == ' ' || c == '%')
+				if (c == '\\' || c == '*' || c == '?' || c == '"' || c == '<' || c == '>' || c == '|' || c < 0x20 || c >= 0x7F || c == ' ' || c == '%')
 					result ~= format("%%%02X", c);
 				else
 					result ~= c;
@@ -596,10 +653,24 @@ final class RefBuilder : ASTraitsVisitor
 			if (!pathSegments.length)
 				pathSegments = [""];
 			foreach (ref pathSegment; pathSegments)
+			{
 				if (pathSegment == "")
 					pathSegment = "%";
 
-			return arrayJoin(pathSegments);
+				static const reservedNames = ["CON", "PRN", "AUX", "NUL", "COM1", "COM2", "COM3", "COM4", "COM5", "COM6", "COM7", "COM8", "COM9", "LPT1", "LPT2", "LPT3", "LPT4", "LPT5", "LPT6", "LPT7", "LPT8", "LPT9"];
+				auto pathSegmentU = pathSegment.toUpper();
+				foreach (reservedName; reservedNames)
+					if (pathSegmentU.startsWith(reservedName))
+					{
+						pathSegment = "%" ~ pathSegment;
+						break;
+					}
+
+				if (pathSegment.length > 240)
+					pathSegment = pathSegment[0..200] ~ '-' ~ getDigestString(pathSegment);
+			}
+
+			return arrayJoin(pathSegments, "/");
 		}
 
 		string[] strings = new string[segments.length];
@@ -682,6 +753,8 @@ final class Disassembler
 
 		StringBuilder sb = new StringBuilder(dir ~ "/" ~ name ~ ".main.asasm");
 
+		sb ~= "#version 2";
+		sb.newLine();
 		sb ~= "#include ";
 		dumpString(sb, name ~ ".privatens.asasm");
 		sb.newLine();
@@ -1025,12 +1098,13 @@ final class Disassembler
 		sb ~= "metadata ";
 		dumpString(sb, metadata.name);
 		sb.indent++; sb.newLine();
-		foreach (ref item; metadata.items)
+		assert(metadata.keys.length == metadata.values.length);
+		foreach (i; 0..metadata.keys.length)
 		{
 			sb ~= "item ";
-			dumpString(sb, item.key);
+			dumpString(sb, metadata.keys[i]);
 			sb ~= " ";
-			dumpString(sb, item.value);
+			dumpString(sb, metadata.values[i]);
 			sb.newLine();
 		}
 		sb.indent--; sb ~= "end ; metadata"; sb.newLine();
@@ -1397,6 +1471,14 @@ final class Disassembler
 
 private:
 
+bool nsSimilar(ASProgram.Namespace ns1, ASProgram.Namespace ns2)
+{
+	if (ns1.kind==ASType.PrivateNamespace || ns2.kind==ASType.PrivateNamespace)
+		return ns1.kind==ns2.kind && ns1.privateIndex==ns2.privateIndex;
+	// ignore ns kind in other cases
+	return ns1.name == ns2.name;
+}
+
 bool[256] newLineAfter;
 
 static this()