1 /******************************************************************************
2
3 LZO1X-1 (Mini LZO) compressor/uncompressor
4
5 Copyright:
6 Copyright (c) 2009-2016 dunnhumby Germany GmbH.
7 All rights reserved.
8
9 License:
10 Boost Software License Version 1.0. See LICENSE_BOOST.txt for details.
11 Alternatively, this file may be distributed under the terms of the Tango
12 3-Clause BSD License (see LICENSE_BSD.txt for details).
13
14 ******************************************************************************/
15
16 module ocean.io.compress.Lzo;
17
18
19 import ocean.io.compress.lzo.c.lzo1x: lzo1x_1_compress,
20 lzo1x_decompress, lzo1x_decompress_safe,
21 lzo1x_max_compressed_length, lzo_init,
22 Lzo1x1WorkmemSize, LzoStatus;
23
24 import ocean.io.compress.lzo.LzoCrc;
25
26 import ocean.io.compress.CompressException;
27
28 import ocean.core.Enforce: enforce;
29
30 import ocean.meta.types.Qualifiers;
31
32 import ocean.core.Verify;
33
34 version (unittest) import ocean.core.Test;
35
36 /******************************************************************************
37
38 Lzo class
39
40 ******************************************************************************/
41
42 class Lzo
43 {
44 alias LzoCrc.crc32 crc32;
45
46 /**************************************************************************
47
48 Working memory buffer for lzo1x_1_compress()
49
50 **************************************************************************/
51
52 private void[] workmem;
53
54 /**************************************************************************
55
56 Static constructor
57
58 Throws:
59 CompressException if the library pouts
60
61 **************************************************************************/
62
63 static this ( )
64 {
65 enforce!(CompressException)(!lzo_init(), "Lzo kaputt");
66 }
67
68 /**************************************************************************
69
70 Constructor
71
72 **************************************************************************/
73
74 public this ( )
75 {
76 this.workmem = new ubyte[Lzo1x1WorkmemSize];
77 }
78
79 /**************************************************************************
80
81 Compresses src data. dst must have a length of at least
82 maxCompressedLength(src.length).
83
84 Params:
85 src = data to compress
86 dst = compressed data destination buffer
87
88 Returns:
89 length of compressed data in dst
90
91 Throws:
92 CompressionException on error
93
94 **************************************************************************/
95
96 size_t compress ( in void[] src, void[] dst )
97 {
98 verify(dst.length >= this.maxCompressedLength(src.length),
99 typeof (this).stringof ~ ".compress: dst buffer too short");
100
101 size_t len;
102
103 this.checkStatus(lzo1x_1_compress(cast (const(ubyte)*) src.ptr, src.length, cast (ubyte*) dst.ptr, &len, this.workmem.ptr));
104
105 return len;
106 }
107
108 /**************************************************************************
109
110 Uncompresses src data. dst must have at least the length of the
111 uncompressed data, which must be memorized at compression time.
112
113 Note: dst overflow checking is NOT done!
114
115 Params:
116 src = data to uncompress
117 dst = uncompressed data destination buffer
118
119 Returns:
120 length of uncompressed data in dst
121
122 Throws:
123 CompressionException on error
124
125 **************************************************************************/
126
127 size_t uncompress ( in void[] src, void[] dst )
128 {
129 size_t len;
130
131 this.checkStatus(lzo1x_decompress(cast (const(ubyte)*) src.ptr, src.length, cast (ubyte*) dst.ptr, &len));
132
133 return len;
134 }
135
136 /**************************************************************************
137
138 Uncompresses src data, checking for dst not to overflow.
139
140 Params:
141 src = data to uncompress
142 dst = uncompressed data destination buffer
143
144 Returns:
145 length of uncompressed data in dst
146
147 Throws:
148 CompressionException on error
149
150 **************************************************************************/
151
152 size_t decompressSafe ( in void[] src, void[] dst )
153 {
154 size_t len = dst.length;
155
156 this.checkStatus(lzo1x_decompress_safe(cast (const(ubyte)*) src.ptr, src.length, cast (ubyte*) dst.ptr, &len));
157
158 return len;
159 }
160
161 /******************************************************************************
162
163 Calculates the maximum compressed length of data which has a length of
164 uncompressed_length.
165
166 Note: Surprisingly, this is more than uncompressed_length but that's the
167 worst case for completely uncompressable data.
168
169 Parameters:
170 uncompressed_length = length of data to compressed
171
172 Returns:
173 maximum compressed length of data
174
175 ******************************************************************************/
176
177 static size_t maxCompressedLength ( size_t uncompressed_length )
178 {
179 return lzo1x_max_compressed_length(uncompressed_length);
180 }
181
182 /**************************************************************************
183
184 Checks if status indicates an error.
185
186 Params:
187 status = LZO library function return status
188
189 Throws:
190 resulting 32-bit CRC value
191
192 **************************************************************************/
193
194 static void checkStatus ( LzoStatus status )
195 {
196 switch (status)
197 {
198 case LzoStatus.Error:
199 throw new CompressException(typeof (this).stringof ~ ": Error");
200
201 case LzoStatus.OutOfMemory:
202 throw new CompressException(typeof (this).stringof ~ ": Out Of Memory");
203
204 case LzoStatus.NotCompressible:
205 throw new CompressException(typeof (this).stringof ~ ": Not Compressible");
206
207 case LzoStatus.InputOverrun:
208 throw new CompressException(typeof (this).stringof ~ ": Input Overrun");
209
210 case LzoStatus.OutputOverrun:
211 throw new CompressException(typeof (this).stringof ~ ": Output Overrun");
212
213 case LzoStatus.LookBehindOverrun:
214 throw new CompressException(typeof (this).stringof ~ ": Look Behind Overrun");
215
216 case LzoStatus.EofNotFound:
217 throw new CompressException(typeof (this).stringof ~ ": Eof Not Found");
218
219 case LzoStatus.InputNotConsumed:
220 throw new CompressException(typeof (this).stringof ~ ": Input Not Consumed");
221
222 case LzoStatus.NotYetImplemented:
223 throw new CompressException(typeof (this).stringof ~ ": Not Yet Implemented");
224
225 default:
226 return;
227 }
228 }
229 }
230
231 /// Simple compress / decompress example.
232 unittest
233 {
234 // LZO instance. (Usually a single one can be shared globally.)
235 auto lzo = new Lzo;
236
237 // Typical function to decompress a buffer that was received over the
238 // network. (i.e. where the buffer and the expected length of the
239 // decompressed data are both received from an external source.)
240 void decompress ( in void[] src, size_t expected_decompressed_length,
241 ref void[] dst )
242 {
243 dst.length = expected_decompressed_length;
244 assumeSafeAppend(dst);
245 auto decompressed_length = lzo.decompressSafe(src, dst);
246
247 // Check that the length of the uncompressed data is what we expected.
248 // (It is important to verify this, when the data was received
249 // externally.)
250 enforce(decompressed_length == expected_decompressed_length);
251 }
252
253 auto original = "compress this!";
254
255 // Compress.
256 auto compressed = new void[lzo.maxCompressedLength(original.length)];
257 compressed.length = lzo.compress(original, compressed);
258
259 // Decompress.
260 void[] decompressed;
261 decompress(compressed, original.length, decompressed);
262
263 // We should have the original string back now.
264 test!("==")(decompressed, original);
265 }
266
267 /******************************************************************************
268
269 Unit test
270
271 Add -debug=GcDisabled to the compiler command line to disable the garbage
272 collector.
273
274 Note: The unit test requires a file named "lzotest.dat" in the current
275 working directory at runtime. This file provides the data to be compressed
276 for testing and performance measurement.
277
278 ******************************************************************************/
279
280 version (unittest):
281
282 // Uncomment the next line to see UnitTest output
283 // version = UnitTestVerbose;
284
285 import ocean.io.device.File;
286 import ocean.time.StopWatch;
287
288 debug (GcDisabled) import ocean.core.Memory;
289
290 import ocean.text.util.MetricPrefix;
291
292 import core.stdc.signal: signal, SIGINT;
293
294 /******************************************************************************
295
296 Rounds x to the nearest integer value
297
298 Params:
299 x = value to round
300
301 Returns:
302 nearest integer value of x
303
304 ******************************************************************************/
305
306 extern (C) int lrintf ( float x );
307
308 /******************************************************************************
309
310 Terminator structure
311
312 ******************************************************************************/
313
314 struct Terminator
315 {
316 static:
317
318 /**************************************************************************
319
320 Termination flag
321
322 **************************************************************************/
323
324 bool terminated = false;
325
326 /**************************************************************************
327
328 Signal handler; raises the termination flag
329
330 **************************************************************************/
331
332 extern (C) void terminate ( int code ) nothrow @nogc
333 {
334 Terminator.terminated = true;
335 }
336 }
337
338 version (UnitTestVerbose) import ocean.io.Stdout;
339
340 unittest
341 {
342 debug (GcDisabled)
343 {
344 version (UnitTestVerbose) Stdout.formatln("LZO unittest: garbage collector disabled");
345 GC.disable();
346 }
347
348 StopWatch swatch;
349
350 MetricPrefix pre_comp_sz, pre_uncomp_sz,
351 pre_comp_tm, pre_uncomp_tm, pre_crc_tm;
352
353 version (UnitTestVerbose) Stdout.formatln("LZO unittest: loading test data from file \"lzotest.dat\"");
354
355 // FLAKEY: Avoid IO in unittests and specially fixed file names
356 File file;
357
358 try file = new File("lzotest.dat");
359 catch (Exception e)
360 return;
361
362 scope data = new void[file.length];
363 scope uncompressed = new void[file.length];
364 scope compressed = new void[Lzo.maxCompressedLength(data.length)];
365 scope lzo = new Lzo;
366
367 file.read(data);
368
369 file.close();
370
371 version (UnitTestVerbose) Stdout.formatln("LZO unittest: loaded {} bytes of test data, compressing...", data.length);
372
373 swatch.start();
374
375 compressed.length = lzo.compress(data, compressed);
376
377 ulong compr_us = swatch.microsec;
378
379 size_t uncompr_len = lzo.uncompress(compressed, uncompressed);
380
381 ulong uncomp_us = swatch.microsec;
382
383 uint crc32_data = lzo.crc32(data);
384
385 ulong crc_us = swatch.microsec;
386
387 test (uncompr_len == data.length, "uncompressed data length mismatch");
388 test (uncompressed == data, "uncompressed data mismatch");
389 test (lzo.crc32(uncompressed) == crc32_data, "uncompressed data CRC-32 mismatch");
390
391 crc_us -= uncomp_us;
392 uncomp_us -= compr_us;
393
394 pre_comp_sz.bin(compressed.length);
395 pre_uncomp_sz.bin(uncompressed.length);
396
397 pre_comp_tm.dec(compr_us, -2);
398 pre_uncomp_tm.dec(uncomp_us, -2);
399 pre_crc_tm.dec(crc_us, -2);
400
401 version (UnitTestVerbose)
402 Stdout.formatln("LZO unittest results:\n\t"
403 ~ "uncompressed length: {} {}B\t({} bytes)\n\t"
404 ~ "compressed length: {} {}B\t({} bytes)\n\t"
405 ~ "compression ratio: {}%\n\t"
406 ~ "compression time: {} {}s\t({} µs)\n\t"
407 ~ "uncompression time: {} {}s\t({} µs)\n\t"
408 ~ "CRC-32 time: {} {}s\t({} µs)\n\t"
409 ~ "CRC-32 (uncompr'd): 0x{:X8}\n"
410 ~ "\n"
411 ~ "LZO unittest: Looping for memory leak detection; "
412 ~ "watch memory usage and press Ctrl+C to quit",
413 pre_uncomp_sz.scaled, pre_uncomp_sz.prefix, uncompressed.length,
414 pre_comp_sz.scaled, pre_comp_sz.prefix, compressed.length,
415 lrintf((compressed.length * 100.f) / uncompressed.length),
416 pre_comp_tm.scaled, pre_comp_tm.prefix, compr_us,
417 pre_uncomp_tm.scaled, pre_uncomp_tm.prefix, uncomp_us,
418 pre_crc_tm.scaled, pre_crc_tm.prefix, crc_us,
419 crc32_data);
420
421 auto prev_sigint_handler = signal(SIGINT, &Terminator.terminate);
422
423 scope (exit) signal(SIGINT, prev_sigint_handler);
424
425 while (!Terminator.terminated)
426 {
427 compressed.length = Lzo.maxCompressedLength(data.length);
428
429 compressed.length = lzo.compress(data, compressed);
430
431 lzo.uncompress(compressed, uncompressed);
432 }
433 }