1 module jaster.serialise.binary; 2 3 private 4 { 5 import std.exception, std.traits, std.bitmanip, std.utf, std.algorithm, std.array, std.range, std.typecons; 6 import jaster.serialise.archive, jaster.serialise.serialiser; 7 import jaster.stream; 8 } 9 10 /++ 11 + Binary format: 12 + 'JSE', 13 + int CRC32 (of all the bytes past this) [Big Endian], 14 + Object Root (Note that the root is special, and doesn't write out a 'Type' field nor a 'Name' field) 15 + 16 + Entry Header: 17 + ubyte Type 18 + ubyte[Varies] TypeSpecificData 19 + 20 + Numeric Entry Data: 21 + T Number [Big Endian] 22 + 23 + Array Entry Data: 24 + Length Length [Big Endian] 25 + ubyte[Length] Data 26 + 27 + Attribute Entry Data: 28 + string(Array) Name 29 + Entry Value 30 + 31 + Object Entry Data: 32 + int Length (of 'Entries' + 'Name' in bytes) 33 + string(Array) Name 34 + Entry[] Entries (includes children, values, and attributes. Always uses an int for the length) 35 + ++/ 36 final class ArchiveBinary : Archive 37 { 38 const MAGIC_NUMBER = cast(const ubyte[])"JSE"; 39 40 private 41 { 42 alias IsRoot = Flag!"isRoot"; 43 alias WriteLength = Flag!"entryLength"; 44 45 enum DataType : ubyte 46 { 47 Bool, 48 Null, 49 UByteArray, 50 ValueArray, 51 Byte, 52 UByte, 53 Short, 54 UShort, 55 Int, 56 UInt, 57 Long, 58 ULong, 59 String, 60 Float, 61 Double, 62 63 Object = 0xFE, 64 Attribute = 0xFF 65 } 66 67 ArchiveObject _obj; 68 } 69 70 this() 71 { 72 this._obj = new ArchiveObject(); 73 } 74 75 public override 76 { 77 const(ubyte[]) saveToMemory() 78 { 79 import std.digest.crc : crc32Of; 80 81 auto stream = new BinaryIO(new MemoryStreamGC()); 82 83 stream.writeBytes(MAGIC_NUMBER); 84 stream.write!int(0); // CRC reserved 85 86 auto start = stream.stream.position; 87 this.saveObject(stream, this.root, IsRoot.yes); 88 auto length = (stream.stream.position - start); 89 90 stream.stream.position = start; 91 auto bytes = stream.readBytes(length); 92 stream.stream.position = start - int.sizeof; 93 auto crc = crc32Of(bytes); // According to the docs, this is little-endian 94 stream.write!int((cast(int[])crc)[0]); 95 96 stream.stream.position = 0; 97 return stream.readBytes(stream.stream.length); 98 } 99 100 void loadFromMemory(const ubyte[] data) 101 { 102 import std.digest.crc : crc32Of; 103 import std.bitmanip : swapEndian; 104 105 auto stream = new BinaryIO(new MemoryStreamGC()); 106 stream.writeBytes(data.dup); 107 stream.stream.position = 0; 108 enforce(stream.readBytes(3) == MAGIC_NUMBER, "Invalid Magic number"); 109 110 auto crc = stream.read!int(); 111 version(BigEndian) 112 crc = crc.swapEndian; // CRC is apparently always in little-endian 113 114 auto start = stream.stream.position; 115 auto length = stream.read!int; 116 117 stream.stream.position = stream.stream.position - int.sizeof; 118 enforce(stream.readBytes(length + int.sizeof).crc32Of == cast(ubyte[4])((&crc)[0..1]), 119 "CRC Mismatch. This is usually a sign that the data has been tampered with, or some kind of transport error occured."); 120 121 stream.stream.position = start; 122 this._obj = this.loadObject(stream, IsRoot.yes); 123 } 124 125 @property 126 ArchiveObject root() 127 { 128 return this._obj; 129 } 130 } 131 132 // ########## 133 // # COMMON # 134 // ########## 135 private 136 { 137 DataType getTypeFor(ArchiveValue value) 138 { 139 if(value.type == typeid(bool)) return DataType.Bool; 140 else if(value.type == typeid(typeof(null))) return DataType.Null; 141 else if(value.type == typeid(ubyte[])) return DataType.UByteArray; 142 else if(value.type == typeid(ArchiveValue[])) return DataType.ValueArray; 143 else if(value.type == typeid(byte)) return DataType.Byte; 144 else if(value.type == typeid(ubyte)) return DataType.UByte; 145 else if(value.type == typeid(short)) return DataType.Short; 146 else if(value.type == typeid(ushort)) return DataType.UShort; 147 else if(value.type == typeid(int)) return DataType.Int; 148 else if(value.type == typeid(uint)) return DataType.UInt; 149 else if(value.type == typeid(long)) return DataType.Long; 150 else if(value.type == typeid(ulong)) return DataType.ULong; 151 else if(value.type == typeid(string)) return DataType.String; 152 else if(value.type == typeid(float)) return DataType.Float; 153 else if(value.type == typeid(double)) return DataType.Double; 154 155 assert(false); 156 } 157 } 158 159 // ##################### 160 // # LOADING FUNCTIONS # 161 // ##################### 162 private 163 { 164 ArchiveValue loadValue(BinaryIO stream, DataType type) 165 { 166 switch(type) with(DataType) 167 { 168 case Bool: 169 return ArchiveValue(cast(bool)stream.readBytes(1)[0]); 170 171 case Null: 172 return ArchiveValue(null); 173 174 case UByteArray: 175 return ArchiveValue(stream.read!(ubyte[])); 176 177 case Byte: 178 return ArchiveValue(stream.read!byte); 179 180 case UByte: 181 return ArchiveValue(stream.read!ubyte); 182 183 case Short: 184 return ArchiveValue(stream.read!short); 185 186 case UShort: 187 return ArchiveValue(stream.read!ushort); 188 189 case Int: 190 return ArchiveValue(stream.read!int); 191 192 case UInt: 193 return ArchiveValue(stream.read!uint); 194 195 case Long: 196 return ArchiveValue(stream.read!long); 197 198 case ULong: 199 return ArchiveValue(stream.read!ulong); 200 201 case String: 202 return ArchiveValue(stream.read!string); 203 204 case Float: 205 return ArchiveValue(stream.read!float); 206 207 case Double: 208 return ArchiveValue(stream.read!double); 209 210 case ValueArray: 211 auto length = stream.readLengthBytes(); 212 ArchiveValue[] values; 213 foreach(i; 0..length) 214 { 215 auto type2 = stream.readBytes(1)[0]; 216 values ~= this.loadValue(stream, cast(DataType)type2); 217 } 218 return ArchiveValue(values); 219 220 default: 221 import std.conv : to; 222 assert(false, type.to!string); 223 } 224 } 225 226 ArchiveObject loadObject(BinaryIO stream, IsRoot isRoot = IsRoot.no) 227 { 228 import std.conv : to; 229 230 auto length = stream.read!uint; 231 auto start = stream.stream.position; 232 auto name = (isRoot) ? "" : stream.read!string; 233 auto obj = new ArchiveObject(name); 234 235 while(true) 236 { 237 enforce(stream.stream.position <= start + length, 238 "Malformed data. Attempted to read past an object's data."); 239 240 if(stream.stream.position == start + length) 241 break; 242 243 auto type = stream.readBytes(1)[0]; 244 switch(type) with(DataType) 245 { 246 case cast(ubyte)Bool: 247 case cast(ubyte)Null: 248 case cast(ubyte)UByteArray: 249 case cast(ubyte)ValueArray: 250 case cast(ubyte)Byte: 251 case cast(ubyte)UByte: 252 case cast(ubyte)Short: 253 case cast(ubyte)UShort: 254 case cast(ubyte)Int: 255 case cast(ubyte)UInt: 256 case cast(ubyte)Long: 257 case cast(ubyte)ULong: 258 case cast(ubyte)String: 259 case cast(ubyte)Float: 260 case cast(ubyte)Double: 261 obj.addValue(this.loadValue(stream, cast(DataType)type)); 262 break; 263 264 case cast(ubyte)Object: 265 obj.addChild(this.loadObject(stream)); 266 break; 267 268 case cast(ubyte)Attribute: 269 auto attribName = stream.read!string; 270 obj.setAttribute(attribName, this.loadValue(stream, cast(DataType)stream.readBytes(1)[0])); 271 break; 272 273 default: 274 throw new Exception("Malformed data. Unknown entry data type: " ~ type.to!string); 275 } 276 } 277 278 return obj; 279 } 280 } 281 282 // #################### 283 // # SAVING FUNCTIONS # 284 // #################### 285 private 286 { 287 void saveEntry(BinaryIO stream, DataType type, void delegate() saver, WriteLength hasLength, IsRoot isRoot = IsRoot.no) 288 { 289 if(!isRoot) 290 stream.write!ubyte(cast(ubyte)type); 291 292 if(hasLength) 293 stream.write!int(0); // Length, reserved 294 295 auto start = stream.stream.position; 296 saver(); 297 298 if(hasLength) 299 { 300 auto length = (stream.stream.position - start); 301 assert(stream.stream.position >= start); 302 303 stream.stream.position = start - int.sizeof; 304 stream.write!int(cast(int)length); 305 stream.stream.position = stream.stream.length; 306 } 307 } 308 309 void saveObject(BinaryIO stream, ArchiveObject obj, IsRoot isRoot = IsRoot.no) 310 { 311 this.saveEntry(stream, DataType.Object, () 312 { 313 if(!isRoot) 314 stream.write(obj.name); 315 316 foreach(attrib; obj.attributes) 317 this.saveAttribute(stream, attrib); 318 319 foreach(value; obj.values) 320 this.saveValue(stream, value); 321 322 foreach(child; obj.children) 323 this.saveObject(stream, child); 324 }, WriteLength.yes, isRoot); 325 } 326 327 void saveAttribute(BinaryIO stream, ArchiveObject.Attribute attrib) 328 { 329 this.saveEntry(stream, DataType.Attribute, () 330 { 331 stream.write(attrib.name); 332 this.saveValue(stream, attrib.value); 333 }, WriteLength.no); 334 } 335 336 void saveValue(BinaryIO stream, ArchiveValue value) 337 { 338 auto type = this.getTypeFor(value); 339 this.saveEntry(stream, type, () 340 { 341 assert(type != DataType.Object); 342 assert(type != DataType.Attribute); 343 final switch(type) with(DataType) 344 { 345 case Object: 346 case Attribute: 347 case Null: break; 348 case Bool: stream.write(cast(byte)value.get!bool); break; 349 case UByteArray: stream.writeBytes(value.get!(ubyte[])); break; 350 case Byte: stream.write(value.get!byte); break; 351 case UByte: stream.write(value.get!ubyte); break; 352 case Short: stream.write(value.get!short); break; 353 case UShort: stream.write(value.get!ushort); break; 354 case Int: stream.write(value.get!int); break; 355 case UInt: stream.write(value.get!uint); break; 356 case Long: stream.write(value.get!long); break; 357 case ULong: stream.write(value.get!ulong); break; 358 case String: stream.write(value.get!string); break; 359 case Float: stream.write(value.get!float); break; 360 case Double: stream.write(value.get!double); break; 361 362 case ValueArray: 363 auto arr = value.get!(ArchiveValue[]); 364 stream.writeLengthBytes(arr.length); 365 366 foreach(val; arr) 367 this.saveValue(stream, val); 368 break; 369 } 370 }, WriteLength.no); 371 } 372 } 373 } 374 /// 375 version(Jasterialise_Unittests) 376 unittest 377 { 378 import fluent.asserts; 379 380 static struct B 381 { 382 @MainValue 383 bool toBOrNotToB; 384 } 385 386 static struct A 387 { 388 @Attribute 389 string name; 390 391 @MainValue 392 int age; 393 394 B[] muhB; 395 } 396 397 auto archive = new ArchiveBinary(); 398 Serialiser.serialise(A("Bob Ross", 60, [B(false), B(true)]), archive.root); 399 400 ubyte[] data = 401 [ 402 // Header 403 cast(ubyte)'J', 'S', 'E', 404 0xEB, 0x3C, 0x4F, 0xC1, // CRC 405 406 // Root object header 407 0x00, 0x00, 0x00, 0x2E, 408 409 // Object Entry 410 0xFE, 411 0x00, 0x00, 0x00, 0x29, 412 0x01, 'A', 413 414 // Attribute Entry 415 0xFF, 416 0x04, 'n', 'a', 'm', 'e', 417 418 // String Entry 419 0x0C, 420 0x08, 'B', 'o', 'b', ' ', 'R', 'o', 's', 's', 421 422 // Int Entry 423 0x08, 424 0x00, 0x00, 0x00, 0x3C, 425 426 // Object Entry 427 0xFE, 428 0x00, 0x00, 0x00, 0x04, 429 0x01, 'B', 430 431 // Bool Entry 432 0x00, 433 0x00, 434 435 // Object Entry 436 0xFE, 437 0x00, 0x00, 0x00, 0x04, 438 0x01, 'B', 439 440 // Bool Entry 441 0x00, 442 0x01 443 ]; 444 archive.saveToMemory().should.equal(data); 445 446 // Here for manual debugging, and to retrieve the CRC for when the format changes. 447 // import std.file; 448 // write("Test.bin", archive.saveToMemory()); 449 450 archive.loadFromMemory(data); 451 auto root = archive.root; 452 453 root["A"].expectAttributeAs!string("name").should.equal("Bob Ross"); 454 root["A"].children[0].name.should.equal("B"); 455 root["A"].children[0].getValueAs!bool(0).should.equal(false); 456 root["A"].children[1].getValueAs!bool(0).should.equal(true); 457 }