ocean.io.compress.Lzo source code

1 /******************************************************************************
2 
3     LZO1X-1 (Mini LZO) compressor/uncompressor
4 
5     Copyright:
6         Copyright (c) 2009-2016 dunnhumby Germany GmbH.
7         All rights reserved.
8 
9     License:
10         Boost Software License Version 1.0. See LICENSE_BOOST.txt for details.
11         Alternatively, this file may be distributed under the terms of the Tango
12         3-Clause BSD License (see LICENSE_BSD.txt for details).
13 
14  ******************************************************************************/
15 
16 module ocean.io.compress.Lzo;
17 
18 
19 import ocean.io.compress.lzo.c.lzo1x: lzo1x_1_compress,
20                                       lzo1x_decompress, lzo1x_decompress_safe,
21                                       lzo1x_max_compressed_length, lzo_init,
22                                       Lzo1x1WorkmemSize, LzoStatus;
23 
24 import ocean.io.compress.lzo.LzoCrc;
25 
26 import ocean.io.compress.CompressException;
27 
28 import ocean.core.Enforce: enforce;
29 
30 import ocean.meta.types.Qualifiers;
31 
32 import ocean.core.Verify;
33 
34 version (unittest) import ocean.core.Test;
35 
36 /******************************************************************************
37 
38     Lzo class
39 
40  ******************************************************************************/
41 
42 class Lzo
43 {
44     alias LzoCrc.crc32 crc32;
45 
46     /**************************************************************************
47 
48         Working memory buffer for lzo1x_1_compress()
49 
50      **************************************************************************/
51 
52     private void[] workmem;
53 
54     /**************************************************************************
55 
56         Static constructor
57 
58         Throws:
59             CompressException if the library pouts
60 
61      **************************************************************************/
62 
63     static this ( )
64     {
65         enforce!(CompressException)(!lzo_init(), "Lzo kaputt");
66     }
67 
68     /**************************************************************************
69 
70         Constructor
71 
72      **************************************************************************/
73 
74     public this ( )
75     {
76         this.workmem = new ubyte[Lzo1x1WorkmemSize];
77     }
78 
79     /**************************************************************************
80 
81         Compresses src data. dst must have a length of at least
82         maxCompressedLength(src.length).
83 
84         Params:
85             src = data to compress
86             dst = compressed data destination buffer
87 
88         Returns:
89             length of compressed data in dst
90 
91         Throws:
92             CompressionException on error
93 
94      **************************************************************************/
95 
96     size_t compress ( in void[] src, void[] dst )
97     {
98         verify(dst.length >= this.maxCompressedLength(src.length),
99                 typeof (this).stringof ~ ".compress: dst buffer too short");
100 
101         size_t len;
102 
103         this.checkStatus(lzo1x_1_compress(cast (const(ubyte)*) src.ptr, src.length, cast (ubyte*) dst.ptr, &len, this.workmem.ptr));
104 
105         return len;
106     }
107 
108     /**************************************************************************
109 
110         Uncompresses src data. dst must have at least the length of the
111         uncompressed data, which must be memorized at compression time.
112 
113         Note: dst overflow checking is NOT done!
114 
115         Params:
116             src = data to uncompress
117             dst = uncompressed data destination buffer
118 
119         Returns:
120             length of uncompressed data in dst
121 
122         Throws:
123             CompressionException on error
124 
125      **************************************************************************/
126 
127     size_t uncompress ( in void[] src, void[] dst )
128     {
129         size_t len;
130 
131         this.checkStatus(lzo1x_decompress(cast (const(ubyte)*) src.ptr, src.length, cast (ubyte*) dst.ptr, &len));
132 
133         return len;
134     }
135 
136     /**************************************************************************
137 
138         Uncompresses src data, checking for dst not to overflow.
139 
140         Params:
141             src = data to uncompress
142             dst = uncompressed data destination buffer
143 
144         Returns:
145             length of uncompressed data in dst
146 
147         Throws:
148             CompressionException on error
149 
150      **************************************************************************/
151 
152     size_t decompressSafe ( in void[] src, void[] dst )
153     {
154         size_t len = dst.length;
155 
156         this.checkStatus(lzo1x_decompress_safe(cast (const(ubyte)*) src.ptr, src.length, cast (ubyte*) dst.ptr, &len));
157 
158         return len;
159     }
160 
161     /******************************************************************************
162 
163         Calculates the maximum compressed length of data which has a length of
164         uncompressed_length.
165 
166         Note: Surprisingly, this is more than uncompressed_length but that's the
167               worst case for completely uncompressable data.
168 
169         Parameters:
170             uncompressed_length = length of data to compressed
171 
172         Returns:
173             maximum compressed length of data
174 
175      ******************************************************************************/
176 
177     static size_t maxCompressedLength ( size_t uncompressed_length )
178     {
179         return lzo1x_max_compressed_length(uncompressed_length);
180     }
181 
182     /**************************************************************************
183 
184         Checks if status indicates an error.
185 
186         Params:
187             status = LZO library function return status
188 
189         Throws:
190             resulting 32-bit CRC value
191 
192      **************************************************************************/
193 
194     static void checkStatus ( LzoStatus status )
195     {
196         switch (status)
197         {
198             case LzoStatus.Error:
199                 throw new CompressException(typeof (this).stringof ~ ": Error");
200 
201             case LzoStatus.OutOfMemory:
202                 throw new CompressException(typeof (this).stringof ~ ": Out Of Memory");
203 
204             case LzoStatus.NotCompressible:
205                 throw new CompressException(typeof (this).stringof ~ ": Not Compressible");
206 
207             case LzoStatus.InputOverrun:
208                 throw new CompressException(typeof (this).stringof ~ ": Input Overrun");
209 
210             case LzoStatus.OutputOverrun:
211                 throw new CompressException(typeof (this).stringof ~ ": Output Overrun");
212 
213             case LzoStatus.LookBehindOverrun:
214                 throw new CompressException(typeof (this).stringof ~ ": Look Behind Overrun");
215 
216             case LzoStatus.EofNotFound:
217                 throw new CompressException(typeof (this).stringof ~ ": Eof Not Found");
218 
219             case LzoStatus.InputNotConsumed:
220                 throw new CompressException(typeof (this).stringof ~ ": Input Not Consumed");
221 
222             case LzoStatus.NotYetImplemented:
223                 throw new CompressException(typeof (this).stringof ~ ": Not Yet Implemented");
224 
225             default:
226                 return;
227         }
228     }
229 }
230 
231 /// Simple compress / decompress example.
232 unittest
233 {
234     // LZO instance. (Usually a single one can be shared globally.)
235     auto lzo = new Lzo;
236 
237     // Typical function to decompress a buffer that was received over the
238     // network. (i.e. where the buffer and the expected length of the
239     // decompressed data are both received from an external source.)
240     void decompress ( in void[] src, size_t expected_decompressed_length,
241         ref void[] dst )
242     {
243         dst.length = expected_decompressed_length;
244         assumeSafeAppend(dst);
245         auto decompressed_length = lzo.decompressSafe(src, dst);
246 
247         // Check that the length of the uncompressed data is what we expected.
248         // (It is important to verify this, when the data was received
249         // externally.)
250         enforce(decompressed_length == expected_decompressed_length);
251     }
252 
253     auto original = "compress this!";
254 
255     // Compress.
256     auto compressed = new void[lzo.maxCompressedLength(original.length)];
257     compressed.length = lzo.compress(original, compressed);
258 
259     // Decompress.
260     void[] decompressed;
261     decompress(compressed, original.length, decompressed);
262 
263     // We should have the original string back now.
264     test!("==")(decompressed, original);
265 }
266 
267 /******************************************************************************
268 
269     Unit test
270 
271     Add -debug=GcDisabled to the compiler command line to disable the garbage
272     collector.
273 
274     Note: The unit test requires a file named "lzotest.dat" in the current
275     working directory at runtime. This file provides the data to be compressed
276     for testing and performance measurement.
277 
278  ******************************************************************************/
279 
280 version (unittest):
281 
282 // Uncomment the next line to see UnitTest output
283 // version = UnitTestVerbose;
284 
285 import ocean.io.device.File;
286 import ocean.time.StopWatch;
287 
288 debug (GcDisabled) import ocean.core.Memory;
289 
290 import ocean.text.util.MetricPrefix;
291 
292 import core.stdc.signal: signal, SIGINT;
293 
294 /******************************************************************************
295 
296     Rounds x to the nearest integer value
297 
298     Params:
299         x = value to round
300 
301     Returns:
302         nearest integer value of x
303 
304  ******************************************************************************/
305 
306 extern (C) int lrintf ( float x );
307 
308 /******************************************************************************
309 
310     Terminator structure
311 
312  ******************************************************************************/
313 
314 struct Terminator
315 {
316     static:
317 
318     /**************************************************************************
319 
320         Termination flag
321 
322      **************************************************************************/
323 
324     bool terminated = false;
325 
326     /**************************************************************************
327 
328         Signal handler; raises the termination flag
329 
330      **************************************************************************/
331 
332     extern (C) void terminate ( int code ) nothrow @nogc
333     {
334         Terminator.terminated = true;
335     }
336 }
337 
338 version (UnitTestVerbose) import ocean.io.Stdout;
339 
340 unittest
341 {
342     debug (GcDisabled)
343     {
344         version (UnitTestVerbose) Stdout.formatln("LZO unittest: garbage collector disabled");
345         GC.disable();
346     }
347 
348     StopWatch swatch;
349 
350     MetricPrefix pre_comp_sz, pre_uncomp_sz,
351                  pre_comp_tm, pre_uncomp_tm, pre_crc_tm;
352 
353     version (UnitTestVerbose) Stdout.formatln("LZO unittest: loading test data from file \"lzotest.dat\"");
354 
355     // FLAKEY: Avoid IO in unittests and specially fixed file names
356     File file;
357 
358     try file = new File("lzotest.dat");
359     catch (Exception e)
360     return;
361 
362     scope data         = new void[file.length];
363     scope uncompressed = new void[file.length];
364     scope compressed   = new void[Lzo.maxCompressedLength(data.length)];
365     scope lzo          = new Lzo;
366 
367     file.read(data);
368 
369     file.close();
370 
371     version (UnitTestVerbose) Stdout.formatln("LZO unittest: loaded {} bytes of test data, compressing...", data.length);
372 
373     swatch.start();
374 
375     compressed.length = lzo.compress(data, compressed);
376 
377     ulong compr_us = swatch.microsec;
378 
379     size_t uncompr_len = lzo.uncompress(compressed, uncompressed);
380 
381     ulong uncomp_us = swatch.microsec;
382 
383     uint crc32_data = lzo.crc32(data);
384 
385     ulong crc_us = swatch.microsec;
386 
387     test (uncompr_len == data.length,            "uncompressed data length mismatch");
388     test (uncompressed == data,                  "uncompressed data mismatch");
389     test (lzo.crc32(uncompressed) == crc32_data, "uncompressed data CRC-32 mismatch");
390 
391     crc_us    -= uncomp_us;
392     uncomp_us -= compr_us;
393 
394     pre_comp_sz.bin(compressed.length);
395     pre_uncomp_sz.bin(uncompressed.length);
396 
397     pre_comp_tm.dec(compr_us, -2);
398     pre_uncomp_tm.dec(uncomp_us, -2);
399     pre_crc_tm.dec(crc_us, -2);
400 
401     version (UnitTestVerbose)
402         Stdout.formatln("LZO unittest results:\n\t"
403                  ~ "uncompressed length: {} {}B\t({} bytes)\n\t"
404                  ~ "compressed length:   {} {}B\t({} bytes)\n\t"
405                  ~ "compression ratio:   {}%\n\t"
406                  ~ "compression time:    {} {}s\t({} µs)\n\t"
407                  ~ "uncompression time:  {} {}s\t({} µs)\n\t"
408                  ~ "CRC-32 time:         {} {}s\t({} µs)\n\t"
409                  ~ "CRC-32 (uncompr'd):  0x{:X8}\n"
410                  ~ "\n"
411                  ~ "LZO unittest: Looping for memory leak detection; "
412                  ~ "watch memory usage and press Ctrl+C to quit",
413                    pre_uncomp_sz.scaled, pre_uncomp_sz.prefix, uncompressed.length,
414                    pre_comp_sz.scaled, pre_comp_sz.prefix, compressed.length,
415                    lrintf((compressed.length * 100.f) / uncompressed.length),
416                    pre_comp_tm.scaled, pre_comp_tm.prefix, compr_us,
417                    pre_uncomp_tm.scaled, pre_uncomp_tm.prefix, uncomp_us,
418                    pre_crc_tm.scaled, pre_crc_tm.prefix, crc_us,
419                    crc32_data);
420 
421     auto prev_sigint_handler = signal(SIGINT, &Terminator.terminate);
422 
423     scope (exit) signal(SIGINT, prev_sigint_handler);
424 
425     while (!Terminator.terminated)
426     {
427         compressed.length = Lzo.maxCompressedLength(data.length);
428 
429         compressed.length =  lzo.compress(data, compressed);
430 
431         lzo.uncompress(compressed, uncompressed);
432     }
433 }