1 /*******************************************************************************
2
3 LZO1X-1 (Mini LZO) compressor/uncompressor generating/accepting chunks of
4 compressed data with a length and checksum header
5
6 Usage example:
7
8 ---
9 import $(TITLE);
10
11 void[] lzo_chunk;
12 void[] uncompressed_chunk;
13
14 void run ( )
15 {
16 scope lzo = new LzoChunk;
17
18 // preallocate lzo_chunk and uncompressed_chunk (optional)
19
20 lzo_chunk = new void[LzoChunk.maxChunkLength(4096)];
21
22 uncompressed_chunk = new void[4096];
23
24 char[] data;
25
26 // populate data with data to compress...
27
28 lzo.compress(data, lzo_chunk);
29
30 // lzo_chunk now holds an LZO chunk with compressed data
31
32 lzo.uncompress(lzo_chunk, uncompressed);
33
34 // uncompressed_chunk now holds data, restored from lzo_chunk
35 }
36 ---
37
38 Copyright:
39 Copyright (c) 2009-2016 dunnhumby Germany GmbH.
40 All rights reserved.
41
42 License:
43 Boost Software License Version 1.0. See LICENSE_BOOST.txt for details.
44 Alternatively, this file may be distributed under the terms of the Tango
45 3-Clause BSD License (see LICENSE_BSD.txt for details).
46
47 ******************************************************************************/
48
49 module ocean.io.compress.lzo.LzoChunk;
50
51
52 import ocean.io.compress.Lzo;
53
54 import ocean.io.compress.lzo.LzoHeader;
55
56 import ocean.io.compress.CompressException;
57
58 import ocean.core.Enforce;
59
60 version (unittest) import ocean.core.Test;
61
62 /*******************************************************************************
63
64 LzoChunk compressor/decompressor
65
66 Chunk data layout if size_t has a width of 32-bit:
67
68 ---
69 void[] chunk
70
71 chunk[0 .. 16] // header
72 chunk[16 .. $] // compressed data
73 ---
74
75 Header data layout
76
77 ---
78 chunk[0 .. 4] // length of chunk[4 .. $] (or compressed data length
79 // + header length - 4)
80 chunk[4 .. 8] // 32-bit CRC value of following header elements and
81 // compressed data (chunk[8 .. $]), calculated using
82 // lzo_crc32()
83 chunk[8 .. 12] // chunk/compression type code (signed 32-bit integer)
84 chunk[12 .. 16] // length of uncompressed data
85 ---
86
87 ******************************************************************************/
88
89 class LzoChunk ( bool LengthInline = true )
90 {
91 /***************************************************************************
92
93 Lzo instance
94
95 **************************************************************************/
96
97 private Lzo lzo;
98
99 /***************************************************************************
100
101 Flag set to true if the lzo instance is a reference (ie was passed from
102 the outside, is *not* owned by this instance, and should not be deleted
103 in the destructor).
104
105 **************************************************************************/
106
107 bool lzo_is_reference;
108
109 /***************************************************************************
110
111 Constructor - creates a new lzo instance internally.
112
113 **************************************************************************/
114
115 public this ( )
116 {
117 this.lzo = new Lzo;
118 }
119
120 /***************************************************************************
121
122 Constructor - sets this instance to use an lzo object passed externally
123 (this allows multiple instances to use the same lzo object).
124
125 Params:
126 lzo = lzo object to use
127
128 **************************************************************************/
129
130 public this ( Lzo lzo )
131 {
132 this.lzo = lzo;
133 this.lzo_is_reference = true;
134 }
135
136 /***************************************************************************
137
138 Compresses a data chunk
139
140 Params:
141 data = data to compress
142 compressed = LZO compressed data chunk with header
143 (output ref parameter)
144
145 Returns:
146 this instance
147
148 **************************************************************************/
149
150 public typeof (this) compress ( T ) ( void[] data, ref T[] compressed )
151 {
152 static assert (T.sizeof == 1);
153
154 LzoHeader!(LengthInline) header;
155
156 size_t end;
157
158 header.uncompressed_length = data.length;
159 header.type = header.type.LZO1X;
160
161 compressed.length = this.maxChunkLength(data.length);
162
163 end = header.length + this.lzo.compress(data, header.strip(compressed));
164
165 compressed.length = end;
166
167 header.write(compressed);
168
169 return this;
170 }
171
172 /***************************************************************************
173
174 Uncompresses a LZO chunk
175
176 Params:
177 compressed = LZO compressed data chunk to uncompress
178 data = uncompressed data (output ref parameter)
179
180 Returns:
181 this instance
182
183 FIXME: - Add assertion for chunk length
184
185 **************************************************************************/
186
187 public typeof (this) uncompress ( T ) ( void[] compressed, ref T[] data )
188 {
189 static assert (T.sizeof == 1);
190
191 LzoHeader!(LengthInline) header;
192
193 void[] buf = header.read(compressed);
194
195 data.length = header.uncompressed_length;
196
197 enforce!(CompressException)(header.type == header.type.LZO1X, "Not LZO1X");
198
199 this.lzo.uncompress(buf, data);
200
201 return this;
202 }
203
204 /***************************************************************************
205
206 Static method alias, to be used as
207
208 ---
209 static size_t maxCompressedLength ( size_t uncompressed_length );
210 ---
211
212 Calculates the maximum compressed length of data which has a length of
213 uncompressed_length.
214
215 Note: Surprisingly, this is more than uncompressed_length but that's the
216 worst case for completely uncompressable data.
217
218 Parameters:
219 uncompressed_length = length of data to compressed
220
221 Returns:
222 maximum compressed length of data
223
224 **************************************************************************/
225
226 alias Lzo.maxCompressedLength maxCompressedLength;
227
228 /***************************************************************************
229
230 Calculates the maximum chunk length from input data which has a length
231 of uncompressed_length.
232
233 Note: Surprisingly, this is more than uncompressed_length but that's the
234 worst case for completely uncompressable data.
235
236 Parameters:
237 uncompressed_length = length of data to compressed
238
239 Returns:
240 maximum compressed length of data
241
242 **************************************************************************/
243
244 static size_t maxChunkLength ( size_t uncompressed_length )
245 {
246 return maxCompressedLength(uncompressed_length) + LzoHeader!().length;
247 }
248 }
249
250 version (unittest):
251
252 /*******************************************************************************
253
254 Unit test
255
256 Add -debug=GcDisabled to the compiler command line to disable the garbage
257 collector.
258
259 Note: The unit test requires a file named "lzotest.dat" in the current
260 working directory at runtime. This file provides the data to be compressed
261 for testing and performance measurement.
262
263 ******************************************************************************/
264
265 import ocean.io.device.File;
266 import ocean.time.StopWatch;
267
268 debug (GcDisabled) import ocean.core.Memory;
269
270 import ocean.text.util.MetricPrefix;
271
272 import core.stdc.signal: signal, SIGINT;
273
274 /*******************************************************************************
275
276 Rounds x to the nearest integer value
277
278 Params:
279 x = value to round
280
281 Returns:
282 nearest integer value of x
283
284 ******************************************************************************/
285
286 extern (C) int lrintf ( float x );
287
288 /*******************************************************************************
289
290 Terminator structure
291
292 ******************************************************************************/
293
294 struct Terminator
295 {
296 static:
297
298 /***************************************************************************
299
300 Termination flag
301
302 **************************************************************************/
303
304 bool terminated = false;
305
306 /***************************************************************************
307
308 Signal handler; raises the termination flag
309
310 **************************************************************************/
311
312 extern (C) void terminate ( int code ) nothrow @nogc
313 {
314 terminated = true;
315 }
316 }
317
318 version (UnitTestVerbose) import ocean.io.Stdout;
319
320 unittest
321 {
322 // Uncomment the next line to see UnitTest output
323 // version = UnitTestVerbose;
324
325 debug (GcDisabled)
326 {
327 version (UnitTestVerbose)
328 Stdout.formatln("LzoChunk unittest: garbage collector disabled");
329 GC.disable();
330 }
331
332 StopWatch swatch;
333
334 MetricPrefix pre_comp_sz, pre_uncomp_sz,
335 pre_comp_tm, pre_uncomp_tm, pre_crc_tm;
336
337 version (UnitTestVerbose)
338 Stdout.formatln(
339 `LzoChunk unittest: loading test data from file "lzotest.dat\"`);
340
341 // FLAKEY: Avoid IO in unittests and specially fixed file names
342 File file;
343
344 try file = new File("lzotest.dat");
345 catch (Exception e)
346 return;
347
348 scope data = new void[file.length];
349 scope compressed = new void[LzoChunk!().maxChunkLength(data.length)];
350 scope uncompressed = new void[file.length];
351
352 scope lzo = new LzoChunk!();
353
354 file.read(data);
355
356 file.close();
357
358 swatch.start();
359
360 lzo.compress(data, compressed);
361
362 ulong compr_us = swatch.microsec;
363
364 lzo.uncompress(compressed, uncompressed);
365
366 ulong uncomp_us = swatch.microsec;
367
368 uncomp_us -= compr_us;
369
370 test (data.length == uncompressed.length, "data lengthmismatch");
371 test (data == uncompressed, "data mismatch");
372
373 pre_comp_sz.bin(compressed.length);
374 pre_uncomp_sz.bin(data.length);
375
376 pre_comp_tm.dec(compr_us, -2);
377 pre_uncomp_tm.dec(uncomp_us, -2);
378
379 version (UnitTestVerbose)
380 {
381 Stdout.formatln("LzoChunk unittest results:\n\t"
382 ~ "uncompressed length: {} {}B\t({} bytes)\n\t"
383 ~ "compressed length: {} {}B\t({} bytes)\n\t"
384 ~ "compression ratio: {}%\n\t"
385 ~ "compression time: {} {}s\t({} µs)\n\t"
386 ~ "uncompression time: {} {}s\t({} µs)\n\t"
387 ~ "\n"
388 ~ "LzoChunk unittest: Looping for memory leak detection; "
389 ~ "watch memory usage and press Ctrl+C to quit",
390 pre_uncomp_sz.scaled, pre_uncomp_sz.prefix, data.length,
391 pre_comp_sz.scaled, pre_comp_sz.prefix, compressed.length,
392 lrintf((compressed.length * 100.f) / data.length),
393 pre_comp_tm.scaled, pre_comp_tm.prefix, compr_us,
394 pre_uncomp_tm.scaled, pre_uncomp_tm.prefix, uncomp_us
395 );
396 }
397
398 auto prev_sigint_handler = signal(SIGINT, &Terminator.terminate);
399
400 scope (exit) signal(SIGINT, prev_sigint_handler);
401
402 while (!Terminator.terminated)
403 {
404 lzo.compress(data, compressed).uncompress(compressed, uncompressed);
405 }
406 }