Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
pigz version 2.1.1
  • Loading branch information
madler committed Jan 25, 2010
1 parent c58a800 commit 7635551
Show file tree
Hide file tree
Showing 5 changed files with 66 additions and 40 deletions.
6 changes: 3 additions & 3 deletions Makefile
Expand Up @@ -13,16 +13,16 @@ pigzt: pigzt.o yarnt.o
cc -o pigzt pigzt.o yarnt.o -lpthread -lz

pigzt.o: pigz.c yarn.h
cc -Wall -pedantic -O3 -DDEBUG -c -o pigzt.o pigz.c
cc -Wall -O3 -DDEBUG -g -c -o pigzt.o pigz.c

yarnt.o: yarn.c yarn.h
cc -Wall -pedantic -O3 -DDEBUG -c -o yarnt.o yarn.c
cc -Wall -O3 -DDEBUG -g -c -o yarnt.o yarn.c

pigzn: pigzn.o
cc -o pigzn pigzn.o -lz

pigzn.o: pigz.c
cc -Wall -pedantic -O3 -DDEBUG -DNOTHREAD -c -o pigzn.o pigz.c
cc -Wall -O3 -DDEBUG -DNOTHREAD -g -c -o pigzn.o pigz.c

clean:
rm -f *.o pigz pigzn pigzt
2 changes: 1 addition & 1 deletion README
@@ -1,4 +1,4 @@
pigz 2.1 (24 Oct 2008) by Mark Adler
pigz 2.2 (xx Nov 2008) by Mark Adler

pigz, which stands for Parallel Implementation of GZip, is a fully functional
replacement for gzip that exploits multiple processors and multiple cores to
Expand Down
51 changes: 32 additions & 19 deletions pigz.c
@@ -1,6 +1,6 @@
/* pigz.c -- parallel implementation of gzip
* Copyright (C) 2007, 2008 Mark Adler
* Version 2.1 24 Oct 2008 Mark Adler
* Version 2.11 28 Oct 2008 Mark Adler
*/

/*
Expand Down Expand Up @@ -87,9 +87,11 @@
Remove _LARGEFILE64_SOURCE, _FILE_OFFSET_BITS is enough
Detect file-too-large error and report, blame build
Replace check combination routines with those from zlib
2.11 28 Oct 2008 Fix a bug for files with an integer number of blocks
Update for yarn 1.1 (yarn_prefix and yarn_abort)
*/

#define VERSION "pigz 2.1\n"
#define VERSION "pigz 2.11\n"

/* To-do:
- add --rsyncable (or -R) [use my own algorithm, set min/max block size]
Expand All @@ -113,12 +115,12 @@
Each partial raw deflate stream is terminated by an empty stored block
(using the Z_SYNC_FLUSH option of zlib), in order to end that partial bit
stream at a byte boundary. That allows the partial streams to be
concantenated simply as sequences of bytes. This adds a very small four to
concatenated simply as sequences of bytes. This adds a very small four to
five byte overhead to the output for each input chunk.
The default input block size is 128K, but can be changed with the -b option.
The number of compress threads is set by default to 8, which can be changed
using -p option. Specifiying -p 1 avoids the use of threads entirely.
using the -p option. Specifying -p 1 avoids the use of threads entirely.
The input blocks, while compressed independently, have the last 32K of the
previous block loaded as a preset dictionary to preserve the compression
Expand Down Expand Up @@ -249,7 +251,6 @@
# include "yarn.h" /* thread, launch(), join(), join_all(), */
/* lock, new_lock(), possess(), twist(), wait_for(),
release(), peek_lock(), free_lock(), yarn_name */
char *yarn_prefix = "pigz"; /* prefix for yarn error messages */
#endif

/* for local functions and globals */
Expand Down Expand Up @@ -295,7 +296,6 @@ local int rsync; /* true for rsync blocking */
local int procs; /* maximum number of compression threads (>= 1) */
local int dict; /* true to initialize dictionary in each thread */
local size_t size; /* uncompressed input size per thread (>= 32K) */
local struct timeval start; /* starting time of day for tracing */

/* saved gzip/zip header data for decompression, testing, and listing */
local time_t stamp; /* time stamp from gzip header */
Expand All @@ -317,6 +317,9 @@ local int bail(char *why, char *what)

#ifdef DEBUG

/* starting time of day for tracing */
local struct timeval start;

/* trace log */
local struct log {
struct timeval when; /* time of entry */
Expand Down Expand Up @@ -666,7 +669,7 @@ local void put_trailer(unsigned long ulen, unsigned long clen,
}
}

/* compute check value depeding on format */
/* compute check value depending on format */
#define CHECK(a,b,c) (form == 1 ? adler32(a,b,c) : crc32(a,b,c))

#ifndef NOTHREAD
Expand Down Expand Up @@ -1124,6 +1127,8 @@ local void parallel_compress(void)
next = get_space(&in_pool);
next->len = readn(ind, next->buf, next->pool->size);
more = next->len != 0;
if (!more)
drop_space(next); /* won't be using it */
if (dict && more) {
use_space(job->in); /* hold as dictionary for next loop */
prev = job->in;
Expand Down Expand Up @@ -1286,20 +1291,24 @@ local void load_read(void *dummy)
{
size_t len;

Trace(("-- launched decompress read thread"));
do {
possess(load_state);
wait_for(load_state, TO_BE, 1);
in_len = len = readn(ind, in_which ? in_buf : in_buf2, BUF);
Trace(("-- decompress read thread read %lu bytes", len));
twist(load_state, TO, 0);
} while (len == BUF);
Trace(("-- exited decompress read thread"));
}

#endif

/* load an input buffer, and set in_next and in_left to that data, update
in_tot, return in_left (only called when in_left has gone to zero) --
in_eof is set to true when in_left has gone to zero and there is no
more input to be had */
/* load() is called when in_left has gone to zero in order to provide more
input data: load the input buffer with BUF or less bytes (less if at end of
file) from the file ind, set in_next to point to the in_left bytes read,
update in_tot, and return in_left -- in_eof is set to true when in_left has
gone to zero and there is no more data left to read from ind */
local size_t load(void)
{
/* if already detected end of file, do nothing */
Expand Down Expand Up @@ -1351,7 +1360,7 @@ local size_t load(void)
if (in_left < BUF) {
in_short = 1;

/* if we got bupkis, now's the time to mark eof */
/* if we got bupkis, now is the time to mark eof */
if (in_left == 0)
in_eof = 1;
}
Expand All @@ -1373,7 +1382,7 @@ local void in_init(void)
#endif
}

/* buffered reading macros for decompresion and listing */
/* buffered reading macros for decompression and listing */
#define GET() (in_eof || (in_left == 0 && load() == 0) ? EOF : \
(in_left--, *in_next++))
#define GET2() (tmp2 = GET(), tmp2 + (GET() << 8))
Expand Down Expand Up @@ -1651,8 +1660,8 @@ local size_t compressed_suffix(char *nm)
}

/* listing file name lengths for -l and -lv */
#define NAMEMAX1 48 /* name display limit at vebosity 1 */
#define NAMEMAX2 16 /* name display limit at vebosity 2 */
#define NAMEMAX1 48 /* name display limit at verbosity 1 */
#define NAMEMAX2 16 /* name display limit at verbosity 2 */

/* print gzip or lzw file information */
local void show_info(int method, unsigned long check, off_t len, int cont)
Expand Down Expand Up @@ -1866,7 +1875,7 @@ local unsigned char out_buf[OUTSIZE];
local unsigned char out_copy[OUTSIZE];
local size_t out_len;

/* outb threads locks */
/* outb threads states */
local lock *outb_write_more = NULL;
local lock *outb_check_more;

Expand Down Expand Up @@ -2735,7 +2744,7 @@ local int option(char *arg)
get & 1 ? "b" : (get & 2 ? "p" : "s"));
arg++;

/* a single dash will be interpeted as stdin */
/* a single dash will be interpreted as stdin */
if (*arg == 0)
return 1;

Expand Down Expand Up @@ -2860,9 +2869,13 @@ int main(int argc, char **argv)

/* prepare for interrupts and logging */
signal(SIGINT, cut_short);
gettimeofday(&start, NULL);
#ifndef NOTHREAD
yarn_prefix = "pigz"; /* prefix for yarn error messages */
yarn_abort = cut_short; /* call on thread error */
#endif
#ifdef DEBUG
log_init();
gettimeofday(&start, NULL); /* starting time for log entries */
log_init(); /* initialize logging */
#endif

/* set all options to defaults */
Expand Down
28 changes: 17 additions & 11 deletions yarn.c
@@ -1,6 +1,6 @@
/* yarn.c -- generic thread operations implemented using pthread functions
* Copyright (C) 2008 Mark Adler
* Version 1.0 19 Oct 2008 Mark Adler
* Version 1.1 26 Oct 2008 Mark Adler
* For conditions of distribution and use, see copyright notice in yarn.h
*/

Expand All @@ -9,6 +9,12 @@
implementations with other thread libraries. See yarn.h for the description
of these operations. */

/* Version history:
1.0 19 Oct 2008 First version
1.1 26 Oct 2008 No need to set the stack size -- remove
Add yarn_abort() function for clean-up on error exit
*/

/* for thread portability */
#define _POSIX_PTHREAD_SEMANTICS
#define _REENTRANT
Expand All @@ -19,7 +25,7 @@
#include <pthread.h> /* pthread_t, pthread_create(), pthread_join(), */
/* pthread_attr_t, pthread_attr_init(), pthread_attr_destroy(),
PTHREAD_CREATE_JOINABLE, pthread_attr_setdetachstate(),
pthread_attr_setstacksize(), pthread_self(), pthread_equal(),
pthread_self(), pthread_equal(),
pthread_mutex_t, PTHREAD_MUTEX_INITIALIZER, pthread_mutex_init(),
pthread_mutex_lock(), pthread_mutex_unlock(), pthread_mutex_destroy(),
pthread_cond_t, PTHREAD_COND_INITIALIZER, pthread_cond_init(),
Expand All @@ -31,17 +37,19 @@

/* constants */
#define local static /* for non-exported functions and globals */
#define STACK_SIZE 524288UL /* stack size for every thread */

/* error handling globals */
extern char *yarn_prefix;
/* error handling external globals, resettable by application */
char *yarn_prefix = "yarn";
void (*yarn_abort)(int) = NULL;


/* immediately exit -- use for errors that shouldn't ever happen */
local void fail(int err)
{
fprintf(stderr, "%s: %s (%d) -- aborting\n",
yarn_prefix,
fprintf(stderr, "%s: %s (%d) -- aborting\n", yarn_prefix,
err == ENOMEM ? "out of memory" : "internal pthread error", err);
if (yarn_abort != NULL)
yarn_abort(err);
exit(err == ENOMEM || err == EAGAIN ? err : EINVAL);
}

Expand Down Expand Up @@ -238,16 +246,15 @@ local void *ignition(void *arg)
}

/* not all POSIX implementations create threads as joinable by default, so that
is made explicit here -- we also set the stack size to make sure it is
adequate */
is made explicit here */
thread *launch(void (*probe)(void *), void *payload)
{
int ret;
thread *th;
struct capsule *capsule;
pthread_attr_t attr;

/* construct the requested probe and argument for the ignition() routine
/* construct the requested call and argument for the ignition() routine
(allocated instead of automatic so that we're sure this will still be
there when ignition() actually starts up -- ignition() will free this
allocation) */
Expand All @@ -263,7 +270,6 @@ thread *launch(void (*probe)(void *), void *payload)
th = my_malloc(sizeof(struct thread_s));
if ((ret = pthread_attr_init(&attr)) ||
(ret = pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE)) ||
(ret = pthread_attr_setstacksize(&attr, STACK_SIZE)) ||
(ret = pthread_create(&(th->id), &attr, ignition, capsule)) ||
(ret = pthread_attr_destroy(&attr)))
fail(ret);
Expand Down
19 changes: 13 additions & 6 deletions yarn.h
@@ -1,6 +1,6 @@
/* yarn.h -- generic interface for thread operations
* Copyright (C) 2008 Mark Adler
* Version 1.0 19 Oct 2008 Mark Adler
* Version 1.1 26 Oct 2008 Mark Adler
*/

/*
Expand Down Expand Up @@ -44,7 +44,7 @@
thread *thread; identifier for launched thread, used by join
void probe(void *); pointer to function "probe", run when thread starts
void *payload; single argument to the probe function
void *payload; single argument passed to the probe function
lock *lock; a lock with a value -- used for exclusive access to
an object and to synchronize threads waiting for
changes to an object
Expand All @@ -64,6 +64,9 @@
called from the main thread, and should only be called after any calls
of join() have completed)
destruct(thread) - terminate the thread in mid-execution and join it
(depending on the implementation, the termination may not be immediate,
but may wait for the thread to execute certain thread or file i/o
operations)
-- Lock functions --
Expand Down Expand Up @@ -97,13 +100,17 @@
-- Error control --
yarn_name - an external char pointer to a string that will be the prefix for
any error messages that these routines generate before exiting -- this
global must be provided by some other module and linked with this one,
there is no default
yarn_name - a char pointer to a string that will be the prefix for any error
messages that these routines generate before exiting -- if not changed
by the application, "yarn" will be used
yarn_abort - an external function that will be executed when there is an
internal yarn error, due to out of memory or misuse -- this function
may exit to abort the application, or if it returns, the yarn error
handler will exit (set to NULL by default for no action)
*/

extern char *yarn_prefix;
extern void (*yarn_abort)(int);

void yarn_mem(void *(*)(size_t), void (*)(void *));

Expand Down

0 comments on commit 7635551

Please sign in to comment.