Update zopfli to current google state.

sailfishos-mirror · Dec 28, 2017 · 1a8e39b · 1a8e39b
1 parent 0880e0b
commit 1a8e39b
Show file tree

Hide file tree

Showing 21 changed files with 1,086 additions and 751 deletions.
diff --git a/Makefile b/Makefile
@@ -17,7 +17,7 @@ yarn.o: yarn.c yarn.h
 
 try.o: try.c try.h
 
-deflate.o: $(ZOPFLI)deflate.c $(ZOPFLI)deflate.h $(ZOPFLI)blocksplitter.h $(ZOPFLI)lz77.h $(ZOPFLI)squeeze.h $(ZOPFLI)tree.h $(ZOPFLI)zopfli.h $(ZOPFLI)cache.h $(ZOPFLI)hash.h $(ZOPFLI)util.h
+deflate.o: $(ZOPFLI)deflate.c $(ZOPFLI)deflate.h $(ZOPFLI)blocksplitter.h $(ZOPFLI)lz77.h $(ZOPFLI)squeeze.h $(ZOPFLI)tree.h $(ZOPFLI)zopfli.h $(ZOPFLI)cache.h $(ZOPFLI)hash.h $(ZOPFLI)util.h $(ZOPFLI)symbols.h
 	$(CC) $(CFLAGS) -c $(ZOPFLI)deflate.c
 
 blocksplitter.o: $(ZOPFLI)blocksplitter.c $(ZOPFLI)blocksplitter.h $(ZOPFLI)deflate.h $(ZOPFLI)lz77.h $(ZOPFLI)squeeze.h $(ZOPFLI)tree.h $(ZOPFLI)util.h $(ZOPFLI)zopfli.h $(ZOPFLI)cache.h $(ZOPFLI)hash.h
@@ -26,7 +26,7 @@ blocksplitter.o: $(ZOPFLI)blocksplitter.c $(ZOPFLI)blocksplitter.h $(ZOPFLI)defl
 tree.o: $(ZOPFLI)tree.c $(ZOPFLI)tree.h $(ZOPFLI)katajainen.h $(ZOPFLI)util.h
 	$(CC) $(CFLAGS) -c $(ZOPFLI)tree.c
 
-lz77.o: $(ZOPFLI)lz77.c $(ZOPFLI)lz77.h $(ZOPFLI)util.h $(ZOPFLI)cache.h $(ZOPFLI)hash.h $(ZOPFLI)zopfli.h
+lz77.o: $(ZOPFLI)lz77.c $(ZOPFLI)lz77.h $(ZOPFLI)util.h $(ZOPFLI)cache.h $(ZOPFLI)hash.h $(ZOPFLI)zopfli.h $(ZOPFLI)symbols.h
 	$(CC) $(CFLAGS) -c $(ZOPFLI)lz77.c
 
 cache.o: $(ZOPFLI)cache.c $(ZOPFLI)cache.h $(ZOPFLI)util.h
@@ -38,7 +38,7 @@ hash.o: $(ZOPFLI)hash.c $(ZOPFLI)hash.h $(ZOPFLI)util.h
 util.o: $(ZOPFLI)util.c $(ZOPFLI)util.h
 	$(CC) $(CFLAGS) -c $(ZOPFLI)util.c
 
-squeeze.o: $(ZOPFLI)squeeze.c $(ZOPFLI)squeeze.h $(ZOPFLI)blocksplitter.h $(ZOPFLI)deflate.h $(ZOPFLI)tree.h $(ZOPFLI)util.h $(ZOPFLI)zopfli.h $(ZOPFLI)lz77.h $(ZOPFLI)cache.h $(ZOPFLI)hash.h
+squeeze.o: $(ZOPFLI)squeeze.c $(ZOPFLI)squeeze.h $(ZOPFLI)blocksplitter.h $(ZOPFLI)deflate.h $(ZOPFLI)tree.h $(ZOPFLI)util.h $(ZOPFLI)zopfli.h $(ZOPFLI)lz77.h $(ZOPFLI)cache.h $(ZOPFLI)hash.h $(ZOPFLI)symbols.h
 	$(CC) $(CFLAGS) -c $(ZOPFLI)squeeze.c
 
 katajainen.o: $(ZOPFLI)katajainen.c $(ZOPFLI)katajainen.h

diff --git a/zopfli/CONTRIBUTING.md b/zopfli/CONTRIBUTING.md
@@ -0,0 +1,24 @@
+Want to contribute? Great! First, read this page (including the small print at the end).
+
+### Before you contribute
+Before we can use your code, you must sign the
+[Google Individual Contributor License Agreement](https://developers.google.com/open-source/cla/individual?csw=1)
+(CLA), which you can do online. The CLA is necessary mainly because you own the
+copyright to your changes, even after your contribution becomes part of our
+codebase, so we need your permission to use and distribute your code. We also
+need to be sure of various other things—for instance that you'll tell us if you
+know that your code infringes on other people's patents. You don't have to sign
+the CLA until after you've submitted your code for review and a member has
+approved it, but you must do it before we can put your code into our codebase.
+Before you start working on a larger contribution, you should get in touch with
+us first through the issue tracker with your idea so that we can help out and
+possibly guide you. Coordinating up front makes it much easier to avoid
+frustration later on.
+
+### Code reviews
+All submissions, including submissions by project members, require review. We
+use Github pull requests for this purpose.
+
+### The small print
+Contributions made by corporations are covered by a different agreement than
+the one above, the Software Grant and Corporate Contributor License Agreement.
diff --git a/zopfli/CONTRIBUTORS b/zopfli/CONTRIBUTORS
@@ -1,7 +1,9 @@
 Mark Adler
 Jyrki Alakuijala
 Frédéric Kayser
+Jeffrey Lim
 Daniel Reed
 Huzaifa Sidhpurwala
 Péter Szabó
 Lode Vandevenne
+Derek Buitenhuis
diff --git a/zopfli/README b/zopfli/README
@@ -25,8 +25,15 @@ zopfli_bin.c is separate from the library and contains an example program to
 create very well compressed gzip files. Currently the makefile builds this
 program with the library statically linked in.
 
-To build the binary, use "make". To build the library as a shared Linux library,
-use "make libzopfli". The source code of Zopfli is under src/zopfli.
+The source code of Zopfli is under src/zopfli. Build instructions:
+
+To build zopfli, compile all .c source files under src/zopfli to a single binary
+with C, and link to the standard C math library, e.g.:
+gcc src/zopfli/*.c -O2 -W -Wall -Wextra -Wno-unused-function -ansi -pedantic -lm -o zopfli
+
+A makefile is provided as well, but only for linux. Use "make" to build the
+binary, "make libzopfli" to build it as a shared library. For other platforms,
+please use the build instructions above instead.
 
 Zopfli Compression Algorithm was created by Lode Vandevenne and Jyrki
 Alakuijala, based on an algorithm by Jyrki Alakuijala.
diff --git a/zopfli/src/zopfli/blocksplitter.c b/zopfli/src/zopfli/blocksplitter.c
@@ -24,7 +24,6 @@ Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
 #include <stdlib.h>
 
 #include "deflate.h"
-#include "lz77.h"
 #include "squeeze.h"
 #include "tree.h"
 #include "util.h"
@@ -39,9 +38,10 @@ typedef double FindMinimumFun(size_t i, void* context);
 /*
 Finds minimum of function f(i) where is is of type size_t, f(i) is of type
 double, i is in range start-end (excluding end).
+Outputs the minimum value in *smallest and returns the index of this value.
 */
 static size_t FindMinimum(FindMinimumFun f, void* context,
-                          size_t start, size_t end) {
+                          size_t start, size_t end, double* smallest) {
   if (end - start < 1024) {
     double best = ZOPFLI_LARGE_FLOAT;
     size_t result = start;
@@ -53,6 +53,7 @@ static size_t FindMinimum(FindMinimumFun f, void* context,
         result = i;
       }
     }
+    *smallest = best;
     return result;
   } else {
     /* Try to find minimum faster by recursively checking multiple points. */
@@ -88,6 +89,7 @@ static size_t FindMinimum(FindMinimumFun f, void* context,
       pos = p[besti];
       lastbest = best;
     }
+    *smallest = lastbest;
     return pos;
 #undef NUM
   }
@@ -103,16 +105,13 @@ dists: ll77 distances
 lstart: start of block
 lend: end of block (not inclusive)
 */
-static double EstimateCost(const unsigned short* litlens,
-                           const unsigned short* dists,
+static double EstimateCost(const ZopfliLZ77Store* lz77,
                            size_t lstart, size_t lend) {
-  return ZopfliCalculateBlockSize(litlens, dists, lstart, lend, 2);
+  return ZopfliCalculateBlockSizeAutoType(lz77, lstart, lend);
 }
 
 typedef struct SplitCostContext {
-  const unsigned short* litlens;
-  const unsigned short* dists;
-  size_t llsize;
+  const ZopfliLZ77Store* lz77;
   size_t start;
   size_t end;
 } SplitCostContext;
@@ -125,8 +124,7 @@ type: FindMinimumFun
 */
 static double SplitCost(size_t i, void* context) {
   SplitCostContext* c = (SplitCostContext*)context;
-  return EstimateCost(c->litlens, c->dists, c->start, i) +
-      EstimateCost(c->litlens, c->dists, i, c->end);
+  return EstimateCost(c->lz77, c->start, i) + EstimateCost(c->lz77, i, c->end);
 }
 
 static void AddSorted(size_t value, size_t** out, size_t* outsize) {
@@ -147,9 +145,8 @@ static void AddSorted(size_t value, size_t** out, size_t* outsize) {
 /*
 Prints the block split points as decimal and hex values in the terminal.
 */
-static void PrintBlockSplitPoints(const unsigned short* litlens,
-                                  const unsigned short* dists,
-                                  size_t llsize, const size_t* lz77splitpoints,
+static void PrintBlockSplitPoints(const ZopfliLZ77Store* lz77,
+                                  const size_t* lz77splitpoints,
                                   size_t nlz77points) {
   size_t* splitpoints = 0;
   size_t npoints = 0;
@@ -158,8 +155,8 @@ static void PrintBlockSplitPoints(const unsigned short* litlens,
   index values. */
   size_t pos = 0;
   if (nlz77points > 0) {
-    for (i = 0; i < llsize; i++) {
-      size_t length = dists[i] == 0 ? 1 : litlens[i];
+    for (i = 0; i < lz77->size; i++) {
+      size_t length = lz77->dists[i] == 0 ? 1 : lz77->litlens[i];
       if (lz77splitpoints[npoints] == i) {
         ZOPFLI_APPEND_DATA(pos, &splitpoints, &npoints);
         if (npoints == nlz77points) break;
@@ -186,7 +183,7 @@ static void PrintBlockSplitPoints(const unsigned short* litlens,
 Finds next block to try to split, the largest of the available ones.
 The largest is chosen to make sure that if only a limited amount of blocks is
 requested, their sizes are spread evenly.
-llsize: the size of the LL77 data, which is the size of the done array here.
+lz77size: the size of the LL77 data, which is the size of the done array here.
 done: array indicating which blocks starting at that position are no longer
     splittable (splitting them increases rather than decreases cost).
 splitpoints: the splitpoints found so far.
@@ -196,15 +193,15 @@ lend: output variable, giving end of block.
 returns 1 if a block was found, 0 if no block found (all are done).
 */
 static int FindLargestSplittableBlock(
-    size_t llsize, const unsigned char* done,
+    size_t lz77size, const unsigned char* done,
     const size_t* splitpoints, size_t npoints,
     size_t* lstart, size_t* lend) {
   size_t longest = 0;
   int found = 0;
   size_t i;
   for (i = 0; i <= npoints; i++) {
     size_t start = i == 0 ? 0 : splitpoints[i - 1];
-    size_t end = i == npoints ? llsize - 1 : splitpoints[i];
+    size_t end = i == npoints ? lz77size - 1 : splitpoints[i];
     if (!done[start] && end - start > longest) {
       *lstart = start;
       *lend = end;
@@ -216,9 +213,7 @@ static int FindLargestSplittableBlock(
 }
 
 void ZopfliBlockSplitLZ77(const ZopfliOptions* options,
-                          const unsigned short* litlens,
-                          const unsigned short* dists,
-                          size_t llsize, size_t maxblocks,
+                          const ZopfliLZ77Store* lz77, size_t maxblocks,
                           size_t** splitpoints, size_t* npoints) {
   size_t lstart, lend;
   size_t i;
@@ -227,35 +222,31 @@ void ZopfliBlockSplitLZ77(const ZopfliOptions* options,
   unsigned char* done;
   double splitcost, origcost;
 
-  if (llsize < 10) return;  /* This code fails on tiny files. */
+  if (lz77->size < 10) return;  /* This code fails on tiny files. */
 
-  done = (unsigned char*)malloc(llsize);
+  done = (unsigned char*)malloc(lz77->size);
   if (!done) exit(-1); /* Allocation failed. */
-  for (i = 0; i < llsize; i++) done[i] = 0;
+  for (i = 0; i < lz77->size; i++) done[i] = 0;
 
   lstart = 0;
-  lend = llsize;
+  lend = lz77->size;
   for (;;) {
     SplitCostContext c;
 
     if (maxblocks > 0 && numblocks >= maxblocks) {
       break;
     }
 
-    c.litlens = litlens;
-    c.dists = dists;
-    c.llsize = llsize;
+    c.lz77 = lz77;
     c.start = lstart;
     c.end = lend;
     assert(lstart < lend);
-    llpos = FindMinimum(SplitCost, &c, lstart + 1, lend);
+    llpos = FindMinimum(SplitCost, &c, lstart + 1, lend, &splitcost);
 
     assert(llpos > lstart);
     assert(llpos < lend);
 
-    splitcost = EstimateCost(litlens, dists, lstart, llpos) +
-        EstimateCost(litlens, dists, llpos, lend);
-    origcost = EstimateCost(litlens, dists, lstart, lend);
+    origcost = EstimateCost(lz77, lstart, lend);
 
     if (splitcost > origcost || llpos == lstart + 1 || llpos == lend) {
       done[lstart] = 1;
@@ -265,7 +256,7 @@ void ZopfliBlockSplitLZ77(const ZopfliOptions* options,
     }
 
     if (!FindLargestSplittableBlock(
-        llsize, done, *splitpoints, *npoints, &lstart, &lend)) {
+        lz77->size, done, *splitpoints, *npoints, &lstart, &lend)) {
       break;  /* No further split will probably reduce compression. */
     }
 
@@ -275,7 +266,7 @@ void ZopfliBlockSplitLZ77(const ZopfliOptions* options,
   }
 
   if (options->verbose) {
-    PrintBlockSplitPoints(litlens, dists, llsize, *splitpoints, *npoints);
+    PrintBlockSplitPoints(lz77, *splitpoints, *npoints);
   }
 
   free(done);
@@ -290,25 +281,22 @@ void ZopfliBlockSplit(const ZopfliOptions* options,
   size_t* lz77splitpoints = 0;
   size_t nlz77points = 0;
   ZopfliLZ77Store store;
+  ZopfliHash hash;
+  ZopfliHash* h = &hash;
 
-  ZopfliInitLZ77Store(&store);
-
-  s.options = options;
-  s.blockstart = instart;
-  s.blockend = inend;
-#ifdef ZOPFLI_LONGEST_MATCH_CACHE
-  s.lmc = 0;
-#endif
+  ZopfliInitLZ77Store(in, &store);
+  ZopfliInitBlockState(options, instart, inend, 0, &s);
+  ZopfliAllocHash(ZOPFLI_WINDOW_SIZE, h);
 
   *npoints = 0;
   *splitpoints = 0;
 
   /* Unintuitively, Using a simple LZ77 method here instead of ZopfliLZ77Optimal
   results in better blocks. */
-  ZopfliLZ77Greedy(&s, in, instart, inend, &store);
+  ZopfliLZ77Greedy(&s, in, instart, inend, &store, h);
 
   ZopfliBlockSplitLZ77(options,
-                       store.litlens, store.dists, store.size, maxblocks,
+                       &store, maxblocks,
                        &lz77splitpoints, &nlz77points);
 
   /* Convert LZ77 positions to positions in the uncompressed input. */
@@ -326,7 +314,9 @@ void ZopfliBlockSplit(const ZopfliOptions* options,
   assert(*npoints == nlz77points);
 
   free(lz77splitpoints);
+  ZopfliCleanBlockState(&s);
   ZopfliCleanLZ77Store(&store);
+  ZopfliCleanHash(h);
 }
 
 void ZopfliBlockSplitSimple(const unsigned char* in,

diff --git a/zopfli/src/zopfli/blocksplitter.h b/zopfli/src/zopfli/blocksplitter.h
@@ -30,21 +30,17 @@ ones that enhance it.
 
 #include <stdlib.h>
 
+#include "lz77.h"
 #include "zopfli.h"
 
 
 /*
 Does blocksplitting on LZ77 data.
 The output splitpoints are indices in the LZ77 data.
-litlens: lz77 lit/lengths
-dists: lz77 distances
-llsize: size of litlens and dists
 maxblocks: set a limit to the amount of blocks. Set to 0 to mean no limit.
 */
 void ZopfliBlockSplitLZ77(const ZopfliOptions* options,
-                          const unsigned short* litlens,
-                          const unsigned short* dists,
-                          size_t llsize, size_t maxblocks,
+                          const ZopfliLZ77Store* lz77, size_t maxblocks,
                           size_t** splitpoints, size_t* npoints);
 
 /*

diff --git a/zopfli/src/zopfli/cache.c b/zopfli/src/zopfli/cache.c
@@ -31,6 +31,12 @@ void ZopfliInitCache(size_t blocksize, ZopfliLongestMatchCache* lmc) {
   lmc->dist = (unsigned short*)malloc(sizeof(unsigned short) * blocksize);
   /* Rather large amount of memory. */
   lmc->sublen = (unsigned char*)malloc(ZOPFLI_CACHE_LENGTH * 3 * blocksize);
+  if(lmc->sublen == NULL) {
+    fprintf(stderr,
+        "Error: Out of memory. Tried allocating %lu bytes of memory.\n",
+        ZOPFLI_CACHE_LENGTH * 3 * blocksize);
+    exit (EXIT_FAILURE);
+  }
 
   /* length > 0 and dist 0 is invalid combination, which indicates on purpose
   that this cache value is not filled in yet. */