iozone.c

Go to the documentation of this file.

/************************************************************************/
/*  Original Author:                        */
/*      William Norcott (wnorcott@us.oracle.com)        */
/*              4 Dunlap Drive                      */
/*              Nashua, NH 03060                    */
/*                                      */
/************************************************************************/
/*      Enhancements by:                        */
/*      Don Capps (capps@iozone.org)                */
/*      7417 Crenshaw                       */
/*      Plano, TX 75025                     */
/*                                      */
/************************************************************************/
/*  Copyright 1991, 1992, 1994, 1998, 2000, 2001  William D. Norcott    */
/************************************************************************/
/*                                  */
/* Iozone is based on the original work done by William Norrcot. It has */
/* been enhanced so that it provides a more complete filesystem     */
/* characterization.                            */
/* Its purpose is to provide automated filesystem characterization. */
/* Enhancements have been made by:                  */
/*                                  */
/* Don Capps                 capps@iozone.org           */ 
/*                                  */
/* Iozone can perform single stream and multi stream I/O        */
/* also it now performs read, write, re-read, re-write,         */
/* read backwards, read/write random, re-read record,           */
/* pread, re-pread, re-pwrite, preadv, re-preadv, pwritev,      */
/* stride read, and re-pwritev,mmap, POSIX async I/O, NFS               */
/* cluster testing and much more.                   */
/*                                  */
/* The frontend now uses getopt() and the user can control many more    */
/* of the actions.                          */
/*                                  */
/*                                  */
/************************************************************************/
/* THIS SOFTWARE IS PROVIDED BY DON CAPPS AND THE IOZONE CREW "AS IS    */
/* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED    */
/* TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A      */
/* PARTICULAR PURPOSE ARE DISCLAIMED.                   */
/*                                  */
/* IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY      */
/* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL   */
/* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE    */
/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS        */
/* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER */
/* IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR      */
/* OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE.       */
/************************************************************************/

/************************************************************************/
/* For the beginner...                              */
/*                                  */
/* 1. make linux  (linux, hpux, convex, hpux_no_ansi)           */
/* 2. type ./iozone -Ra                         */
/*                                  */
/*  Hint: Type make    (it will give you a list of valid targets)   */
/*                                  */
/************************************************************************/


/* The version number */
#define THISVERSION "        Version $Revision$"

#if defined(linux)
  #define _GNU_SOURCE
#endif
/* Include for Cygnus development environment for Windows */
#if defined (Windows)
#include <windows.h>
#include <errno.h>
#else
#if defined(linux) || defined(solaris) || defined(macosx) || defined(__AIX__) || defined(FreeBSD) || defined(_HPUX_SOURCE)
#include <errno.h>
#else
extern  int errno;   /* imported for errors */
extern  int h_errno; /* imported for errors */
#endif
#endif


#include <sys/types.h>
#include <sys/stat.h>
#if defined (__LP64__) || defined(OSF_64) || defined(__alpha__) || defined(__arch64__) || defined(_LP64) || defined(__s390x__) || defined(__AMD64__)
#define MODE "\tCompiled for 64 bit mode."
#define _64BIT_ARCH_
#else
#define MODE "\tCompiled for 32 bit mode."
#endif

#ifndef NO_THREADS
#include <pthread.h>
#endif

#if defined(HAVE_ANSIC_C) && defined(linux)
#include <stdlib.h>
#include <sys/wait.h>
#endif

#ifdef HAVE_PROTO
#include "proto.h"
#else
int atoi();
int close();
int unlink();
int main();
void record_command_line();
#if !defined(linux)
int wait();
#endif
int fsync();
void srand48();
long lrand48();
void create_list();
void Poll();
void print_header();
void Kill();
long long l_min();
long long l_max();
long long mythread_create();
int gen_new_buf();
void touch_dedup();
void init_by_array64(unsigned long long *, unsigned long long );
unsigned long long genrand64_int64(void);
#endif

#include <fcntl.h>

char *help[] = {
"    Usage: iozone [-s filesize_Kb] [-r record_size_Kb] [-f [path]filename] [-h]",
"                  [-i test] [-E] [-p] [-a] [-A] [-z] [-Z] [-m] [-M] [-t children]",
"                  [-l min_number_procs] [-u max_number_procs] [-v] [-R] [-x] [-o]",
"                  [-d microseconds] [-F path1 path2...] [-V pattern] [-j stride]",
"                  [-T] [-C] [-B] [-D] [-G] [-I] [-H depth] [-k depth] [-U mount_point]",
"                  [-S cache_size] [-O] [-L cacheline_size] [-K] [-g maxfilesize_Kb]",
"                  [-n minfilesize_Kb] [-N] [-Q] [-P start_cpu] [-e] [-c] [-b Excel.xls]",
"                  [-J milliseconds] [-X write_telemetry_filename] [-w] [-W]",
"                  [-Y read_telemetry_filename] [-y minrecsize_Kb] [-q maxrecsize_Kb]",
"                  [-+u] [-+m cluster_filename] [-+d] [-+x multiplier] [-+p # ]",
"                  [-+r] [-+t] [-+X] [-+Z] [-+w percent dedupable] [-+y percent_interior_dedup]",
"                  [-+C percent_dedup_within]",
" ",
"           -a  Auto mode",
"           -A  Auto2 mode",
"           -b Filename  Create Excel worksheet file",
"           -B  Use mmap() files",
"           -c  Include close in the timing calculations",
"           -C  Show bytes transferred by each child in throughput testing",
"           -d #  Microsecond delay out of barrier",
"           -D  Use msync(MS_ASYNC) on mmap files",
"           -e  Include flush (fsync,fflush) in the timing calculations",
"           -E  Run extension tests",
"           -f filename  to use",
"           -F filenames  for each process/thread in throughput test",
"           -g #  Set maximum file size (in Kbytes) for auto mode (or #m or #g)",
"           -G  Use msync(MS_SYNC) on mmap files",
"           -h  help",
"           -H #  Use POSIX async I/O with # async operations",
"           -i #  Test to run (0=write/rewrite, 1=read/re-read, 2=random-read/write",
"                 3=Read-backwards, 4=Re-write-record, 5=stride-read, 6=fwrite/re-fwrite",
"                 7=fread/Re-fread, 8=random_mix, 9=pwrite/Re-pwrite, 10=pread/Re-pread",
"                 11=pwritev/Re-pwritev, 12=preadv/Re-preadv)",
"           -I  Use VxFS VX_DIRECT, O_DIRECT,or O_DIRECTIO for all file operations",
"           -j #  Set stride of file accesses to (# * record size)",
"           -J #  milliseconds of compute cycle before each I/O operation",
"           -k #  Use POSIX async I/O (no bcopy) with # async operations",
"           -K  Create jitter in the access pattern for readers",
"           -l #  Lower limit on number of processes to run",
"           -L #  Set processor cache line size to value (in bytes)",
"           -m  Use multiple buffers",
"           -M  Report uname -a output",
"           -n #  Set minimum file size (in Kbytes) for auto mode (or #m or #g)",
"           -N  Report results in microseconds per operation",
"           -o  Writes are synch (O_SYNC)",
"           -O  Give results in ops/sec.",
"           -p  Purge on",
"           -P #  Bind processes/threads to processors, starting with this cpu",
"           -q #  Set maximum record size (in Kbytes) for auto mode (or #m or #g)",
"           -Q  Create offset/latency files",
"           -r #  record size in Kb",
"              or -r #k .. size in Kb",
"              or -r #m .. size in Mb",
"              or -r #g .. size in Gb",
"           -R  Generate Excel report",
"           -s #  file size in Kb",
"              or -s #k .. size in Kb",
"              or -s #m .. size in Mb",
"              or -s #g .. size in Gb",
"           -S #  Set processor cache size to value (in Kbytes)",
"           -t #  Number of threads or processes to use in throughput test",
"           -T  Use POSIX pthreads for throughput tests",
"           -u #  Upper limit on number of processes to run",
"           -U  Mount point to remount between tests",
"           -v  version information",
"           -V #  Verify data pattern write/read",
"           -w  Do not unlink temporary file",
"           -W  Lock file when reading or writing",
"           -x  Turn off stone-walling",
"           -X filename  Write telemetry file. Contains lines with (offset reclen compute_time) in ascii",
"           -y #  Set minimum record size (in Kbytes) for auto mode (or #m or #g)",
"           -Y filename  Read  telemetry file. Contains lines with (offset reclen compute_time) in ascii",
"           -z  Used in conjunction with -a to test all possible record sizes",
"           -Z  Enable mixing of mmap I/O and file I/O",
"           -+E Use existing non-Iozone file for read-only testing",
"           -+K Sony special. Manual control of test 8.",
"           -+m  Cluster_filename   Enable Cluster testing",
"           -+d  File I/O diagnostic mode. (To troubleshoot a broken file I/O subsystem)",
"           -+u  Enable CPU utilization output (Experimental)",
"           -+x # Multiplier to use for incrementing file and record sizes",
"           -+p # Percentage of mix to be reads",
"           -+r Enable O_RSYNC|O_SYNC for all testing.",
"           -+t Enable network performance test. Requires -+m ",
"           -+n No retests selected.",
"           -+k Use constant aggregate data set size.",
"           -+q Delay in seconds between tests.",
"           -+l Enable record locking mode.",
"           -+L Enable record locking mode, with shared file.",
"           -+B Sequential mixed workload.",
#if defined(O_DSYNC)
"           -+D Enable O_DSYNC mode.",
#endif
#ifndef NO_MADVISE
"           -+A #  Enable madvise. 0 = normal, 1=random, 2=sequential",
"                                  3=dontneed, 4=willneed",
#endif
"           -+N Do not truncate existing files on sequential writes.",
"           -+S # Dedup-able data is limited to sharing within each numerically",
"                 identified file set",
"           -+V Enable shared file. No locking.",
#if defined(Windows)
"           -+U Windows Unbufferd I/O API (Very Experimental)",
#endif
"           -+X Enable short circuit mode for filesystem testing ONLY",
"               ALL Results are NOT valid in this mode.",
"           -+Z Enable old data set compatibility mode. WARNING.. Published",
"               hacks may invalidate these results and generate bogus, high",
"               values for results.",
"           -+w ## Percent of dedup-able data in buffers.",
"           -+y ## Percent of dedup-able within & across files in buffers.",
"           -+C ## Percent of dedup-able within & not across files in buffers.",
"           -+H Hostname    Hostname of the PIT server.",
"           -+P Service     Service  of the PIT server.",
"           -+z Enable latency histogram logging.",
"" };

char *head1[] = {
  "       'Iozone' Filesystem Benchmark Program",
  " ",
THISVERSION,
  MODE,
  " ",
  "       Original Author: William Norcott (wnorcott@us.oracle.com)",
  "               4 Dunlap Drive",
  "               Nashua, NH 03060",
  " ",
  "       Enhancements: Don Capps (capps@iozone.org)",
  "           7417 Crenshaw",
  "           Plano, TX 75025",
  " ",
  "  Copyright 1991, 1992, 1994, 1998, 1999, 2002   William D. Norcott",
  " ",
  "  License to freely use and distribute this software is hereby granted ",
  "  by the author, subject to the condition that this copyright notice ",
  "  remains intact.  The author retains the exclusive right to publish ",
  "  derivative works based on this work, including, but not limited to, ",
  "  revised versions of this work",
  " ",
  "  Other contributors:",
  " ",
  "  Don Capps       (Network Appliance)    capps@iozone.org",
  " ",
  ""};

/******************************************************************

    INCLUDE FILES (system-dependent)

******************************************************************/
#include <sys/mman.h>
#include <stdio.h>
#include <signal.h>
#include <unistd.h>

#include <fcntl.h>
#if !defined(__FreeBSD__) && !defined(__OpenBSD__) && !defined(__APPLE__) && !defined(__DragonFly__)
#include <malloc.h>
#endif
#if defined(__FreeBSD__) || defined(__OpenBSD__) || defined(__APPLE__) || defined(__DragonFly__)
#include <stdlib.h>
#include <string.h>
#endif

#if defined (__FreeBSD__) || defined(__OpenBSD__) || defined(__bsdi__) || defined(__APPLE__) || defined(__DragonFly__)
#ifndef O_SYNC
#define O_SYNC O_FSYNC
#endif
#endif

#if defined (__FreeBSD__)
#ifndef O_RSYNC
#define O_RSYNC O_FSYNC
#endif
#endif

#if ((defined(solaris) && defined(__LP64__)) || defined(__s390x__))
/* If we are building for 64-bit Solaris, all functions that return pointers
 * must be declared before they are used; otherwise the compiler will assume
 * that they return ints and the top 32 bits of the pointer will be lost,
 * causing segmentation faults.  The following includes take care of this.
 * It should be safe to add these for all other OSs too, but we're only
 * doing it for Solaris now in case another OS turns out to be a special case.
 */
#include <strings.h>
#include <stdlib.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <arpa/inet.h>

#endif
#if ( defined(solaris) && defined(studio11) )
#include <strings.h>
#include <stdlib.h>
#endif

#if defined(OSFV5) || defined(linux)
#include <string.h>
#endif

#if defined(linux)
#include <sys/socket.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#endif

#ifndef MAP_FAILED
#define MAP_FAILED -1
#endif

#ifdef generic
typedef long long off64_t;
#endif

#if defined(__DragonFly__)
#define __off64_t_defined
typedef off_t off64_t;
#endif


#ifndef solaris
#ifndef off64_t
#ifndef _OFF64_T
#ifndef __AIX__
#ifndef __off64_t_defined
#ifndef SCO_Unixware_gcc
#ifndef UWIN
#ifndef __DragonFly__
typedef long long off64_t;
#endif
#endif
#endif
#endif
#endif
#endif
#endif
#endif

#ifdef __AIX__
#include <fcntl.h>
#endif

#ifdef VXFS
#include <sys/fs/vx_ioctl.h>
#endif

#ifdef unix
#if defined (__APPLE__) || defined(__FreeBSD__) || defined(__DragonFly__) \
   || defined(_SUA_)
#include <sys/time.h>
#endif
#include <sys/times.h>
#include <sys/file.h>
#include <sys/resource.h>
#ifndef NULL
#define NULL 0
#endif

#ifndef nolimits
#include <limits.h>
#endif
#endif

#ifdef HAVE_ANSIC_C
#define VOLATILE volatile
#else 
#define VOLATILE 
#endif

#include <sys/time.h>

#ifdef SHARED_MEM
#include <sys/shm.h>
#endif

#if defined(bsd4_2) && !defined(MS_SYNC)
#define MS_SYNC 0
#define MS_ASYNC 0
#endif

#if defined(bsd4_4) || defined(__DragonFly__)
#define MAP_ANONYMOUS MAP_ANON
#endif

#if defined(SCO_Unixware_gcc) || defined(solaris) || defined(UWIN) || defined(SCO)
#define MAP_FILE (0)
#endif

#if defined(IRIX) || defined(IRIX64) || defined(Windows) || defined(bsd4_2) || defined(bsd4_4) || defined(SCO) || defined(Solaris) || defined(SCO_Unixware_gcc)
long long page_size = 4096;
#define GOT_PAGESIZE 1
#elif defined(NBPG)
long long page_size = NBPG;
#define GOT_PAGESIZE 1
#elif defined(old_linux)
#include <asm/page.h>
long long page_size = PAGE_SIZE;
#define GOT_PAGESIZE 1
#elif !defined(GOT_PAGESIZE)
long long page_size = 4096; /* Used when all else fails */
#endif

#ifdef HAVE_PREAD
#ifdef HAVE_PREADV
#define PVECMAX 16

#ifdef _HPUX_SOURCE
#define PER_VECTOR_OFFSET
#include <sys/puio.h>
struct piovec piov[PVECMAX];
#else
#include <sys/uio.h>
struct iovec piov[PVECMAX];
#define piov_base iov_base
#define piov_len iov_len
#endif

#endif
#endif

#define DEDUPSEED 0x2719362


/*
 * In multi thread/process throughput mode each child keeps track of
 * statistics and communicates them through various flavors of
 * shared memory, and via messages.
 */
struct child_stats {
    long long flag;     /* control space */
    long long flag1;    /* pad */
    float walltime;         /* child elapsed time */
    float cputime;      /* child CPU time */
    float throughput;   /* Throughput in either kb/sec or ops/sec */
    float actual;       /* Either actual kb read or # of ops performed */
} VOLATILE *child_stat;

/*
 * Used for cpu time statistics.
 */
struct runtime {
    float   walltime;
    float   cputime;
    float   cpuutil;
};

#ifdef __convex_spp
#include <sys/cnx_ail.h>
#endif

#include <sys/socket.h>
#include <netinet/in.h>
#include <netdb.h>


/* 
 * Messages the controlling process sends to children.
 * Internal representation that is arch specific.
 * This is used when using the network distributed mode.
 */
struct client_command {
    char c_host_name[100];
    char c_pit_hostname[40];
    char c_pit_service[8];
    char c_client_name[100];
    char c_working_dir[200];
    char c_file_name[200];
    char c_path_dir[200];
    char c_execute_name[200];
    char c_write_traj_filename[200];
    char c_read_traj_filename[200];
    int c_oflag;
    int c_mfflag;
    int c_unbuffered;
    int c_noretest;
    int c_notruncate;
    int c_read_sync;
    int c_jflag;
    int c_async_flag;
    int c_k_flag;
    int c_h_flag;
    int c_mflag;
    int c_pflag;
    int c_stride_flag;
    int c_verify;
    int c_sverify;
    int c_odsync;
    int c_diag_v;
    int c_dedup;
    int c_dedup_interior;
    int c_dedup_compress;
    int c_dedup_mseed;
    int c_hist_summary;
    int c_op_rate;
    int c_op_rate_flag;
    int c_Q_flag;
    int c_L_flag;
    int c_OPS_flag;
    int c_mmapflag;
    int c_mmapasflag;
    int c_mmapnsflag;
    int c_mmapssflag;
    int c_no_copy_flag;
    int c_include_close;
    int c_include_flush;
    int c_disrupt_flag;
    int c_compute_flag;
    int c_xflag;
    int c_MS_flag;
    int c_mmap_mix;
    int c_Kplus_flag;
    int c_stop_flag;
    int c_w_traj_flag;
    int c_r_traj_flag;
    int c_direct_flag;
    int c_cpuutilflag;
    int c_seq_mix;
    int c_client_number;
    int c_command;
    int c_testnum;
    int c_no_unlink;
    int c_no_write;
    int c_file_lock;
    int c_rec_lock;
    int c_Kplus_readers;
    int c_multiplier;
    int c_share_file;
    int c_pattern;
    int c_version;
    int c_base_time;
    int c_num_child;
    int c_pct_read;
    int c_advise_op;
    int c_advise_flag;
    int c_restf;
    int c_mygen;
    long long c_stride;
    long long c_rest_val;
    long long c_delay;
    long long c_purge;
    long long c_fetchon;
    long long c_numrecs64;
    long long c_reclen;
    long long c_child_flag;
    long long c_delay_start;
    long long c_depth;
    float c_compute_time;
};  

/*
 * All data in this is in string format for portability in a 
 * hetrogeneous environment.
 *
 * Messages that the master will send to the clients
 * over the socket. This provides neutral format
 * so that heterogeneous clusters will work.
 * This is used when using the network distributed mode.
 * WARNING !!! This data structure MUST not be bigger
 * than 1448 bytes or fragmentation will kick your butt.
 */
struct client_neutral_command {
    char c_host_name[100];
    char c_pit_hostname[40];
    char c_pit_service[8];
    char c_client_name[100];
    char c_working_dir[200];
    char c_file_name[200];
    char c_path_dir[200];
    char c_execute_name[200];
    char c_write_traj_filename[200];
    char c_read_traj_filename[200];
    char c_oflag[2];
    char c_mfflag[2];
    char c_unbuffered[2];
    char c_noretest[2];
    char c_notruncate[2];
    char c_read_sync[2];
    char c_jflag[2];
    char c_async_flag[2];
    char c_k_flag[2];
    char c_h_flag[2];
    char c_mflag[2];
    char c_pflag[2];
    char c_stride_flag[2];
    char c_verify[2];
    char c_sverify[2];
    char c_odsync[2];
    char c_diag_v[2];
    char c_dedup[4];
    char c_dedup_interior[4];
    char c_dedup_compress[4];
    char c_dedup_mseed[4];
    char c_hist_summary[4];
    char c_op_rate[4];
    char c_op_rate_flag[2];
    char c_Q_flag[2];
    char c_L_flag[2];
    char c_OPS_flag[2];
    char c_mmapflag[2];
    char c_mmapasflag[2];
    char c_mmapnsflag[2];
    char c_mmapssflag[2];
    char c_no_copy_flag[2];
    char c_include_close[2];
    char c_include_flush[2];
    char c_disrupt_flag[2];
    char c_compute_flag[2];
    char c_stop_flag[2];
    char c_xflag[2];
    char c_MS_flag[2];
    char c_mmap_mix[2];
    char c_Kplus_flag[2];
    char c_w_traj_flag[2];      /* small int */
    char c_r_traj_flag[2];      /* small int */
    char c_direct_flag[2];      /* small int */
    char c_cpuutilflag[2];      /* small int */
    char c_seq_mix[2];      /* small int */
    char c_stride[10];      /* small long long */
    char c_rest_val[10];        /* small long long */
    char c_purge[10];       /* very small long long */
    char c_fetchon[10];         /* very small long long */
    char c_multiplier[10];      /* small int */
    char c_share_file[10];      /* small int */
    char c_file_lock[10];       /* small int */
    char c_rec_lock[10];        /* small int */
    char c_Kplus_readers[10];   /* small int */
    char c_client_number[20];   /* int */
    char c_command[20];         /* int */
    char c_testnum[20];         /* int */
    char c_no_unlink[4];        /* int */
    char c_no_write[4];         /* int */
    char c_pattern[20];         /* int */
    char c_version[20];         /* int */
    char c_base_time[20];       /* int */
    char c_num_child[20];       /* int */
    char c_pct_read[6];         /* small int */
    char c_advise_op[4];        /* small int */
    char c_advise_flag[4];      /* small int */
    char c_restf[4];        /* small int */
    char c_mygen[20];       /* long */
    char c_depth[20];       /* small long long */
    char c_child_flag[40];      /* small long long */
    char c_delay[80];       /* long long */
    char c_numrecs64[80];       /* long long */
    char c_reclen[80];      /* long long */
    char c_delay_start[80];     /* long long */
    char c_compute_time[80];    /* float */
};  

/* 
 * Messages the clients will send to the master.
 * Internal representation on each client and the master.
 * This is used when using the network distributed mode.
 */
struct master_command {
    char m_host_name[100];
    char m_client_name[100];
    char m_stop_flag; 
    int m_client_number;
    int m_client_error;
    int m_child_port;
    int m_child_async_port;
    int m_command;
    int m_testnum;
    int m_version;
    int m_mygen;
    float m_throughput;
    float m_cputime;
    float m_walltime;
    float m_actual;
    long long m_child_flag;
};  

/*
 * Messages that the clients will send to the master
 * over the socket. This provides neutral format
 * so that heterogeneous clusters will work.
 * This is used when using the network distributed mode.
 */
struct master_neutral_command {
    char m_host_name[100];
    char m_client_name[100];
    char m_client_number[20];   /* int */
    char m_client_error[20];    /* int */
    char m_stop_flag[4];        /* char +space */
    char m_child_port[20];      /* int */
    char m_child_async_port[20];    /* int */
    char m_command[20];     /* int */
    char m_testnum[20];     /* int */
    char m_version[20];     /* int */
    char m_mygen[20];       /* int */
    char m_throughput[80];      /* float */
    char m_cputime[80];     /* float */
    char m_walltime[80];        /* float */
    char m_actual[80];      /* float */
    char m_child_flag[80];      /* long long */
};  


/*
 * Possible values for the commands sent to the master
 */
#define R_CHILD_JOIN        1
#define R_STAT_DATA         2
#define R_FLAG_DATA         3

/*
 * Possible values for the master's commands sent to a client
 *
 * The R_FLAG_DATA is also used by the master to tell the 
 * client to update its flags.
 */
#define R_JOIN_ACK        4
#define R_STOP_FLAG       5
#define R_TERMINATE       6
#define R_DEATH           7


/* These are the defaults for the processor. They can be 
 * over written by the command line options.
 */
#define CACHE_LINE_SIZE 32
#define CACHE_SIZE ( 1024 * 1024 )


#define MEG (1024 * 1024)

/*
 * For stride testing use a prime number to avoid stripe
 * wrap hitting the same spindle.
 */
#define STRIDE 17



/************************************************************************/
/*                                  */
/*    DEFINED CONSTANTS                         */
/*                                  */
/* Never add a comment to the end of a #define. Some compilers will     */
/* choke and fail the compile.                      */
/************************************************************************/

/*
 * Size of buffer for capturing the machine's name.
 */
#define IBUFSIZE 100
/*
 * How many I/Os before a non-uniform access.
 */
#define DISRUPT 100

/*
 * Set the crossover size. This is where the small transfers
 * are skipped to save time. There is an option to 
 * disable the skipping.
 */
#define LARGE_REC 65536

/* Default number of kilobytes in file */
#define KILOBYTES 512           

/* Default number of bytes in a record */
#define RECLEN 1024         

/* Default size of file in bytes*/
#define FILESIZE (KILOBYTES*1024)   

/* Default number of records */
#define NUMRECS FILESIZE/RECLEN     

#ifdef __bsdi__
/* At 8 Meg switch to large records */
#define CROSSOVER (8*1024)      
/*maximum buffer size*/
#define MAXBUFFERSIZE (8*1024*1024)     
#else
/* At 16 Meg switch to large records */
#define CROSSOVER (16*1024)     
/* Maximum buffer size*/
#define MAXBUFFERSIZE (16*1024*1024)        
#endif

/* Maximum number of children. Threads/procs/clients */
#define MAXSTREAMS  256     

/* Minimum buffer size */
#define MINBUFFERSIZE 128
/* If things ran way too fast */
#define TOOFAST 10
/* Set the maximum number of types of tests */
#define MAXTESTS 12
/* Default fill pattern for verification */
#define PATTERN get_pattern();
#define PATTERN1 0xBB
/* Used for Excel internal tables */
#define MAX_X 100           
/* Used for Excel internal tables */
#define MAX_Y 512           

#define USAGE  "\tUsage: For usage information type iozone -h \n\n"


/* Maximum number of characters in filename */
#define MAXNAMESIZE 1000                

/*
 * Define the typical output that the user will see on their
 * screen.
 */
#ifdef NO_PRINT_LLD
#ifdef HAVE_PREAD
#include <sys/times.h>
#if defined(HAVE_PREAD) && defined(HAVE_PREADV)
#define CONTROL_STRING1 "%16ld%8ld%8ld%8ld%8ld%8ld%8ld%8ld%8ld %8ld %8ld%8ld%8ld%9ld%9ld%8ld%10ld%9ld%10ld%9ld%10ld%10ld%9ld\n"
#define CONTROL_STRING2 "%16s%8s%8s%8s%8s%10s%8s%8s%8s %8s %8s%9s%9s%8s%9s%8s%9s%7s%10s%10s%10s%9s%9s\n"
#define CONTROL_STRING3 "%16s%8s%8s%8s%8s%10s%8s%8s%8s %8s %8s%9s%9s%8s%9s\n"
#define CONTROL_STRING4 "%16s%8s%8s%8s%8s%10s\n"
#else
#define CONTROL_STRING1 "%16ld%8ld%8ld%8ld%8ld%8ld%8ld%8ld%8ld %8ld %8ld%8ld%8ld%9ld%9ld%8ld%10ld%9ld%10ld\n"
#define CONTROL_STRING2 "%16s%8s%8s%8s%8s%10s%8s%8s%8s %8s %8s%9s%9s%8s%9s%8s%9s%7s%10s\n"
#define CONTROL_STRING3 "%16s%8s%8s%8s%8s%10s%8s%8s%8s %8s %8s\n"
#define CONTROL_STRING4 "%16s%8s%8s%8s%8s%10s\n"
#endif
#else
#define CONTROL_STRING1 "%16ld%8ld%8ld%8ld%8ld%8ld%8ld%8ld%8ld %8ld %8ld%8ld%8ld%9ld%9ld\n"
#define CONTROL_STRING2 "%16s%8s%8s%8s%8s%10s%8s%8s%8s %8s %8s%9s%9s%8s%9s\n"
#define CONTROL_STRING3 "%16s%8s%8s%8s%8s%10s%8s%8s%8s %8s %8s%9s%9s%8s%9s\n"
#define CONTROL_STRING4 "%16s%8s%8s%8s%8s%10s\n"
#endif
#endif

#ifndef NO_PRINT_LLD
#ifdef HAVE_PREAD
#include <sys/times.h>
#if defined(HAVE_PREAD) && defined(HAVE_PREADV)
#define CONTROL_STRING1 "%16lld%8ld%8ld%8ld%8ld%8ld%8ld%8ld%8ld %8ld %8ld%8ld%8ld%9ld%9ld%8ld%10ld%9ld%10ld%9ld%10ld%10ld%9ld\n"
#define CONTROL_STRING2 "%16s%8s%8s%8s%8s%10s%8s%8s%8s %8s %8s%9s%9s%8s%9s%8s%9s%7s%10s%10s%10s%9s%9s\n"
#define CONTROL_STRING3 "%16s%8s%8s%8s%8s%10s%8s%8s%8s %8s %8s%9s%9s%8s%9s\n"
#define CONTROL_STRING4 "%16s%8s%8s%8s%8s%10s\n"
#else
#define CONTROL_STRING1 "%16lld%8ld%8ld%8ld%8ld%8ld%8ld%8ld%8ld %8ld %8ld%8ld%8ld%9ld%9ld%8ld%10ld%9ld%10ld\n"
#define CONTROL_STRING2 "%16s%8s%8s%8s%8s%10s%8s%8s%8s %8s %8s%9s%9s%8s%9s%8s%9s%7s%10s\n"
#define CONTROL_STRING3 "%16s%8s%8s%8s%8s%10s%8s%8s%8s %8s %8s%9s%9s%8s%9s\n"
#define CONTROL_STRING4 "%16s%8s%8s%8s%8s%10s\n"
#endif
#else
#define CONTROL_STRING1 "%16lld%8ld%8ld%8ld%8ld%8ld%8ld%8ld %8ld %8ld%8ld%8ld%8ld%9ld%9ld\n"
#define CONTROL_STRING2 "%16s%8s%8s%8s%8s%10s%8s%8s%8s %8s %8s%9s%9s%8s%9s\n"
#define CONTROL_STRING3 "%16s%8s%8s%8s%8s%10s%8s%8s%8s %8s %8s%9s%9s%8s%9s\n"
#define CONTROL_STRING4 "%16s%8s%8s%8s%8s%10s\n"
#endif
#endif

/* 
    For 'auto mode', these defines determine the number of iterations
    to perform for both the file size and the record length.
*/ 

/* Start with 64 kbyte minimum file size by default */
#define KILOBYTES_START       64
/* Default maximum file size. This is 512 Mbytes */
#define KILOBYTES_END (1024*512)
/* Default starting record size */
#define RECLEN_START  4096
/* Default maximum record size */
#define RECLEN_END    (MAXBUFFERSIZE)
/* Multiplier for each itteration on file and record size */
#define MULTIPLIER    2

/*
 * Assign numeric values to each of the tests.
 */
#define WRITER_TEST     0
#define READER_TEST     1
#define RANDOM_RW_TEST      2
#define REVERSE_TEST        3
#define REWRITE_REC_TEST    4
#define STRIDE_READ_TEST    5
#define FWRITER_TEST        6
#define FREADER_TEST        7
#define RANDOM_MIX_TEST     8
#ifdef HAVE_PREAD
#define PWRITER_TEST        9
#define PREADER_TEST        10  
#endif /* HAVE_PREAD */
#ifdef HAVE_PREADV
#define PWRITEV_TEST        11
#define PREADV_TEST     12
#endif /* HAVE_PREADV */

#define WRITER_MASK     (1 << WRITER_TEST)
#define READER_MASK     (1 << READER_TEST)
#define RANDOM_RW_MASK      (1 << RANDOM_RW_TEST)
#define RANDOM_MIX_MASK     (1 << RANDOM_MIX_TEST)
#define REVERSE_MASK        (1 << REVERSE_TEST)
#define REWRITE_REC_MASK    (1 << REWRITE_REC_TEST)
#define STRIDE_READ_MASK    (1 << STRIDE_READ_TEST)
#define FWRITER_MASK        (1 << FWRITER_TEST)
#define FREADER_MASK        (1 << FREADER_TEST)
#ifdef HAVE_PREAD
#define PWRITER_MASK        (1 << PWRITER_TEST)
#define PREADER_MASK        (1 << PREADER_TEST)
#endif /* HAVE_PREAD */
#ifdef HAVE_PREADV
#define PWRITEV_MASK        (1 << PWRITEV_TEST)
#define PREADV_MASK     (1 << PREADV_TEST)
#endif /* HAVE_PREADV */

/*
 * child_stat->flag values and transitions
 */
/* Parent initializes children to HOLD */
#define CHILD_STATE_HOLD    0   
/* Child tells master when it's READY */
#define CHILD_STATE_READY   1   
/* Parent tells child to BEGIN */
#define CHILD_STATE_BEGIN   2   
/* Child tells parent that it's DONE */
#define CHILD_STATE_DONE    3   

#define MERSENNE

/******************************************************************/
/*                                */
/*    FUNCTION DECLARATIONS                   */
/*                                */
/******************************************************************/
char *initfile();
/*int pit_gettimeofday( struct timeval *, struct timezone *, char *, char *);*/
int pit_gettimeofday( );
static int openSckt( const char *, const char *, unsigned int );
static void pit( int, struct timeval *);
void mmap_end();
void alloc_pbuf();
void auto_test();       /* perform automatic test series  */
void show_help();       /* show development help          */
static double time_so_far();    /* time since start of program    */
#ifdef unix
static double utime_so_far();   /* user time              */
static double stime_so_far();   /* system time            */
static double clk_tck();    /* Get clocks/tick        */
static double cputime_so_far();
#else
#define cputime_so_far()    time_so_far()
#endif
static double time_so_far1();   /* time since start of program    */
void get_resolution();
void get_rusage_resolution();
void signal_handler();      /* clean up if user interrupts us */
void begin();           /* The main worker in the app     */
void fetchit();         /* Prime on chip cache        */
void purgeit();         /* Purge on chip cache        */
void throughput_test();     /* Multi process throughput       */
void multi_throughput_test();   /* Multi process throughput       */
void prepage();         /* Pre-fault user buffer      */
void get_date();
int get_pattern();      /* Set pattern based on version   */
#ifdef HAVE_ANSIC_C
float do_compute(float);    /* compute cycle simulation       */
#else
float do_compute();     /* compute cycle simulation       */
#endif
void write_perf_test();     /* write/rewrite test         */
void fwrite_perf_test();    /* fwrite/refwrite test       */
void fread_perf_test();     /* fread/refread test         */
void read_perf_test();      /* read/reread test       */
void mix_perf_test();       /* read/reread test       */
void random_perf_test();    /* random read/write test     */
void reverse_perf_test();   /* reverse read test          */
void rewriterec_perf_test();    /* rewrite record test        */
void read_stride_perf_test();   /* read with stride test      */
#ifdef HAVE_PREAD
void pread_perf_test();     /* pread/re-pread test        */
void pwrite_perf_test();    /* pwrite/re-pwrite test      */
#endif /* HAVE_PREAD */
#ifdef HAVE_PREADV
void preadv_perf_test();    /* preadv/re-preadv test      */
void pwritev_perf_test();   /* pwritev/re-pwritev test    */
#endif /* HAVE_PREADV */
void store_dvalue();        /* Store doubles array        */
void dump_excel();
void dump_throughput();
int sp_start_child_send();
int sp_start_master_listen();
#ifdef HAVE_ANSIC_C
#if defined (HAVE_PREAD) && defined(_LARGEFILE64_SOURCE)
ssize_t pwrite64(); 
ssize_t pread64(); 
#endif
#if !defined(linux)
char *getenv();
char *inet_ntoa();
int system();
#endif
void my_nap();
void my_unap();
int thread_exit();
#ifdef ASYNC_IO
size_t async_write();
void async_release();
int async_read();
int async_read_no_copy();
size_t async_write_no_copy();
void end_async();
void async_init();
#else
size_t async_write();
size_t async_write_no_copy();
void async_release();
#endif
void do_float();
int create_xls();
void close_xls();
void do_label();
int mylockf(int, int, int);
int mylockr(int,int, int, off64_t, off64_t);
int rand(void);
void srand(unsigned int);
int get_client_info(void);
void exit(int);
void find_remote_shell(char *);
void find_external_mon(char *,char *);
void start_monitor(char *);
void stop_monitor(char *);
void takeoff_cache();
void del_cache();
void fill_area(long long *, long long *, long long);
void fill_buffer(char *,long long ,long long ,char, long long );
void store_value(off64_t);
void store_times(double, double);
static double cpu_util(double, double);
void dump_cputimes(void);
void purge_buffer_cache(void);
char *alloc_mem(long long,int);
void *(thread_rwrite_test)(void *);
void *(thread_write_test)(void *);
void *(thread_fwrite_test)(void *);
void *(thread_fread_test)(void *);
void *(thread_read_test)(void*);
#ifdef HAVE_PREAD
void *(thread_pread_test)(void*);
void *(thread_pwrite_test)(void*);
#endif
void *(thread_cleanup_test)(void*);
void *(thread_cleanup_quick)(void*);
void *(thread_ranread_test)(void *);
void *(thread_mix_test)(void *);
void *(thread_ranwrite_test)(void *);
void *(thread_rread_test)(void *);
void *(thread_reverse_read_test)(void *);
void *(thread_stride_read_test)(void *);
void *(thread_set_base)(void *);
void *(thread_join)(long long, void *);
void disrupt(int);
#if defined(Windows)
void disruptw(HANDLE);
#endif
long long get_traj(FILE *, long long *, float *, long);
void create_temp(off64_t, long long );
FILE *open_w_traj(void);
FILE *open_r_traj(void);
void traj_vers(void);
void r_traj_size(void);
long long w_traj_size(void);
void init_file_sizes();
off64_t get_next_file_size(off64_t);
void add_file_size(off64_t);
void init_file_sizes( off64_t,  off64_t);
off64_t get_next_record_size(off64_t);
void add_record_size(off64_t);
void init_record_sizes( off64_t,  off64_t);
void del_record_sizes( void );
void hist_insert(double );
void dump_hist(char *,int );
void do_speed_check(int);
#else
void do_speed_check();
#if !defined(linux)
char *getenv();
char *inet_ntoa();
int system();
#endif
void my_nap();
void my_unap();
int thread_exit();
void close_xls();
void do_label();
int create_xls();
void do_float();
#ifdef ASYNC_IO
void async_release();
size_t async_write();
size_t async_write_no_copy();
int async_read();
int async_read_no_copy();
#endif
int mylockf();
int mylockr();
int rand();
void srand();
int get_client_info();
void exit();
void find_remote_shell();
void traj_vers();
void r_traj_size();
long long w_traj_size();
FILE *open_w_traj();
FILE *open_r_traj();
void create_temp();
void fill_buffer();
char *alloc_mem();
void *(thread_rwrite_test)();
void *(thread_write_test)();
void *(thread_fwrite_test)();
void *(thread_fread_test)();
void *(thread_read_test)();
void *(thread_cleanup_test)();
void *(thread_ranread_test)();
void *(thread_mix_test)();
void *(thread_ranwrite_test)();
void *(thread_rread_test)();
void *(thread_reverse_read_test)();
void *(thread_stride_read_test)();
void *(thread_set_base)();
void *(thread_join)();
void disrupt();
long long get_traj();
void init_file_sizes();
off64_t get_next_file_size();
void add_file_size();
void init_record_sizes();
off64_t get_next_record_size();
void add_record_size();
void dump_cputimes();
static double cpu_util();
void del_record_sizes();
void hist_insert();
void dump_hist();
#endif

#ifdef _LARGEFILE64_SOURCE
#define I_LSEEK(x,y,z)  lseek64(x,(off64_t)(y),z)
#define I_OPEN(x,y,z)   open64(x,(int)(y),(int)(z))
#define I_CREAT(x,y)    creat64(x,(int)(y))
#define I_FOPEN(x,y)    fopen64(x,y)
#define I_STAT(x,y)     stat64(x,y)
#ifdef HAVE_PREAD
#define I_PREAD(a,b,c,d)    pread64(a,b,(size_t)(c),(off64_t)(d))
#define I_PWRITE(a,b,c,d)   pwrite64(a,b,(size_t)(c),(off64_t)(d))
#endif
#define I_MMAP(a,b,c,d,e,f)     mmap64((void *)(a),(size_t)(b),(int)(c),(int)(d),(int)(e),(off64_t)(f))
#else
#define I_LSEEK(x,y,z)  lseek(x,(off_t)(y),z)
#define I_OPEN(x,y,z)   open(x,(int)(y),(int)(z))
#define I_CREAT(x,y)    creat(x,(int)(y))
#define I_FOPEN(x,y)    fopen(x,y)
#define I_STAT(x,y)     stat(x,y)
#ifdef HAVE_PREAD
#define I_PREAD(a,b,c,d)    pread(a,b,(size_t)(c),(off_t)(d))
#define I_PWRITE(a,b,c,d)   pwrite(a,b,(size_t)(c),(off_t)(d))
#endif
#define I_MMAP(a,b,c,d,e,f)     mmap((void *)(a),(size_t)(b),(int)(c),(int)(d),(int)(e),(off_t)(f))
#endif


/************************************************************************/
/* The list of tests to be called.                  */
/************************************************************************/
void (*func[])() = { 
            write_perf_test, 
            read_perf_test,
            random_perf_test,
            reverse_perf_test,
            rewriterec_perf_test,
            read_stride_perf_test,
            fwrite_perf_test,
            fread_perf_test,
            mix_perf_test 
#ifdef HAVE_PREAD
            ,
            pwrite_perf_test,
            pread_perf_test
#ifdef HAVE_PREADV
            ,
            pwritev_perf_test,
            preadv_perf_test
#endif /* HAVE_PREADV */
#endif /* HAVE_PREAD */
    };

/*
char *test_output[] = {"                ",
              "                  ",
              "                ",
              "        ",
              "        ",
              "        ",
              "                  ",
              "                ",
              "                 ",
              "                 ",
              "                   ",
              "                   \n" };
*/
char *test_output[] = {"                ",
              "                  ",
              "                ",
              "        ",
              "        ",
              "        ",
              "                  ",
              "                ",
              "",
              "                 ",
              "                   ",
              "                   ",
              "                   ",
              "                   ",
              "                   ",
              "                   \n" };
long long test_soutput[] = {2,2,2,1,1,1,2,2,2,2,2,2,2,2};


/******************************************************************/
/*                                */
/*    GLOBAL VARIABLES                            */
/*                                */
/*******************************************************************/

/*
 * Set the size of the shared memory segment for the children
 * to put their results.
 */
#define SHMSIZE  ((( sizeof(struct child_stats) * MAXSTREAMS) )+4096 )
/*
 * Pointer to the shared memory segment.
 */
VOLATILE struct child_stats *shmaddr;
double totaltime,total_time, temp_time ,total_kilos;
off64_t report_array[MAX_X][MAX_Y];
double report_darray[MAX_X][MAXSTREAMS];
double time_res,cputime_res;
long long throughput_array[MAX_X];  /* Filesize & record size are constants */
short current_x, current_y;
long long orig_size;
long long max_x, max_y;
unsigned long long goodkilos;
off64_t kilobytes64 = (off64_t)KILOBYTES;
long long goodrecl;
off64_t offset = 0;               /*offset for random I/O */
off64_t offset64 = 0;               /*offset for random I/O */
off64_t filebytes64;
off64_t r_range[100];
off64_t s_range[100];
int t_range[100];
int t_count = 0;
int r_count,s_count;
char *barray[MAXSTREAMS];
char *haveshm;
extern int optind;
long long onetime, auto_mode, sfd, multi_buffer;
int fd;
int sp_msfd,sp_mrfd,sp_csfd,sp_crfd;
int begin_proc,num_processors,ioz_processor_bind;
long long res_prob,rec_prob;
char silent,read_sync;
char master_iozone, client_iozone,distributed;
int bif_fd,s_count;
int bif_row,bif_column;
int dedup_mseed = 1;
int hist_summary;
int op_rate;
int op_rate_flag;
char aflag, Eflag, hflag, Rflag, rflag, sflag;
char diag_v,sent_stop,dedup,dedup_interior,dedup_compress;
char *dedup_ibuf;
char *dedup_temp;
char bif_flag;
int rlocking;
int share_file;
int ecount;
char gflag,nflag;
char yflag,qflag;
#ifdef Windows
char *build_name = "Windows";
#else
char *build_name = NAME;
#endif
char imon_start[256],imon_stop[256]; 
char imon_sync;
char trflag; 
char cpuutilflag;
char seq_mix;
long base_time;
long long mint, maxt; 
long long w_traj_ops, r_traj_ops, w_traj_fsize,r_traj_fsize;
long long r_traj_ops_completed,r_traj_bytes_completed;
long long w_traj_ops_completed,w_traj_bytes_completed;
int w_traj_items, r_traj_items;
char fflag, Uflag,uflag,lflag,include_tflag; 
struct runtime runtimes [MAX_X] [MAX_Y];    /* in parallel with report_array[][] */
long long include_test[50];
long long include_mask;
char RWONLYflag, NOCROSSflag;       /*auto mode 2 - kcollins 8-21-96*/
char mfflag;
long long status, x, y, childids[MAXSTREAMS+1], myid, num_child;
int pct_read,speed_code;
#ifndef NO_THREADS
pthread_t p_childids[MAXSTREAMS+1];
#endif
off64_t next64;
char wol_opened, rol_opened;
FILE *wqfd,*rwqfd,*rqfd,*rrqfd;

extern char *optarg;
#ifndef __AIX__
long long ret;
#else
short ret;
#endif
struct size_entry {
    struct size_entry *next;
    off64_t size;
};
struct size_entry *size_list=0;
struct size_entry *rec_size_list=0;
off64_t maximum_file_size;
off64_t minimum_file_size;

char bif_filename [MAXNAMESIZE];           /* name of biff file      */
char filename [MAXNAMESIZE];               /* name of temporary file */
char mountname [MAXNAMESIZE];              /* name of device         */
char dummyfile [MAXSTREAMS][MAXNAMESIZE];  /* name of dummy file     */
char dummyfile1 [MAXNAMESIZE];             /* name of dummy file     */
char *filearray[MAXSTREAMS];           /* array of file names    */
char tfile[] = "iozone";
char *buffer,*buffer1, *mbuffer,*mainbuffer;
FILE *pi,*r_traj_fd,*w_traj_fd;
VOLATILE char *pbuffer;
char *default_filename="iozone.tmp"; /*default name of temporary file*/
VOLATILE char stoptime;
char Cflag;
char use_thread = 0;
long long debug1=0;     
long long debug=0;
unsigned long cache_size=CACHE_SIZE;
unsigned long cache_line_size=CACHE_LINE_SIZE;
long long *pstatus;
off64_t min_file_size = KILOBYTES_START;
off64_t max_file_size = KILOBYTES_END;
long long min_rec_size = RECLEN_START;
long long max_rec_size = RECLEN_END;
long long orig_min_rec_size = RECLEN_START;
long long orig_max_rec_size = RECLEN_END;
long long xover = CROSSOVER;
char *throughput_tests[] = {"Initial write","Rewrite","Read","Re-read",
    "Reverse Read","Stride read","Random read","Mixed workload","Random write","Pwrite","Pread","Fwrite","Fread"};
char command_line[1024] = "\0";
#ifdef unix
double sc_clk_tck;
#endif

int argcsave;
char **argvsave;
char splash[80][80];
int splash_line;
char client_filename[256];
char remote_shell[256];
int client_error;

char pit_hostname[40];
char pit_service[8];
int junk;

/* 
 * Host ports used to listen, and handle errors.
 */
#define HOST_LIST_PORT 20000
#define HOST_ESEND_PORT (HOST_LIST_PORT+MAXSTREAMS)
#define HOST_ASEND_PORT (HOST_ESEND_PORT+MAXSTREAMS)
int controlling_host_port = HOST_LIST_PORT;

/* 
 * Childs ports used to listen, and handle errors.
 */
#define CHILD_ESEND_PORT (HOST_ASEND_PORT+MAXSTREAMS)
#define CHILD_LIST_PORT (CHILD_ESEND_PORT+MAXSTREAMS)

/* Childs async message port. Used for stop flag and terminate */
#define CHILD_ALIST_PORT (CHILD_LIST_PORT+MAXSTREAMS)

/* Ports for the network speed code */
#define SP_CHILD_LISTEN_PORT 31000
#define SP_CHILD_ESEND_PORT (SP_CHILD_LISTEN_PORT+10)
#define SP_MASTER_LISTEN_PORT (SP_CHILD_ESEND_PORT+10)
#define SP_MASTER_ESEND_PORT (SP_MASTER_LISTEN_PORT+10)
#define SP_MASTER_RESULTS_PORT (SP_MASTER_ESEND_PORT+10)


#define THREAD_WRITE_TEST 1
#define THREAD_REWRITE_TEST 2
#define THREAD_READ_TEST 3
#define THREAD_REREAD_TEST 4
#define THREAD_STRIDE_TEST 5
#define THREAD_RANDOM_READ_TEST 6
#define THREAD_RANDOM_WRITE_TEST 7
#define THREAD_REVERSE_READ_TEST 8
#define THREAD_RANDOM_MIX_TEST 9
#define THREAD_PWRITE_TEST 10
#define THREAD_PREAD_TEST 11
#define THREAD_FWRITE_TEST 12
#define THREAD_FREAD_TEST 13
#define THREAD_CLEANUP_TEST 14

/*
 * Child states that the master is tracking.
 * The master uses these to determine how to shutdown
 * the clients when some fool hits control-C.
 */
#define C_STATE_ZERO 1
#define C_STATE_WAIT_WHO 2
#define C_STATE_WAIT_BARRIER 3


int c_port,a_port; /* port number */
int child_port; /* Virtualized due to fork */
int child_async_port; /* Virtualized due to fork */
int client_listen_pid; /* Virtualized due to fork */
int master_join_count; /* How many children have joined */
int l_sock,l_async_sock; /* Sockets for listening */
char master_rcv_buf[4096]; /* Master's receive buffer */
int master_listen_pid; /* Pid of the master's async listener proc */
char master_send_buf[4096]; /* Master's send buffer */
char child_rcv_buf[4096]; /* Child's receive buffer */
char child_async_rcv_buf[4096]; /* Child's async recieve buffer */
char child_send_buf[4096]; /* Child's send buffer */
int child_send_socket; /* Child's send socket */
int child_listen_socket; /* Child's listener socket */
int child_listen_socket_async; /* Child's async listener socket */
int master_send_socket; /* Needs to be an array. One for each child*/
int master_send_sockets[MAXSTREAMS]; /* Needs to be an array. One for each child*/
int master_send_async_sockets[MAXSTREAMS]; /* Needs to be an array. One for each child*/
int master_listen_port; /* Master's listener port number */
int master_listen_socket; /* Master's listener socket */
int clients_found; /* Number of clients found in the client file */
FILE *newstdin, *newstdout, *newstderr; /* used for debug in cluster mode.*/
char toutput[20][20]; /* Used to help format the output */
int toutputindex; /* Index to the current output line */
int cdebug = 0; /* Use to turn on child/client debugging in cluster mode. */
int mdebug = 0; /* Use to turn on master debug in cluster mode */
int aggflag; /* Used to indicate constant aggregate data set size */
struct sockaddr_in child_sync_sock, child_async_sock;

/*
 * Change this whenever you change the message format of master or client.
 */
int proto_version = 25;

/******************************************************************************/
/* Tele-port zone. These variables are updated on the clients when one is     */
/* using cluster mode. (-+m)                                                  */
/* Do not touch these unless you have become one with the universe !!         */
/******************************************************************************/
char controlling_host_name[100];
struct child_ident {
    char child_name[100];
    char workdir[200];
    char execute_path[200];
    char file_name[200];
    int state;
    int child_number;
    int child_port;
    int child_async_port;
    int master_socket_num;
    int master_async_socket_num;
}child_idents[MAXSTREAMS];
int Kplus_readers;
char write_traj_filename [MAXNAMESIZE];     /* name of write telemetry file */
char read_traj_filename [MAXNAMESIZE];    /* name of read telemetry file  */
char oflag,jflag,k_flag,h_flag,mflag,pflag,unbuffered,Kplus_flag;
char noretest;
char notruncate;   /* turn off truncation of files */
char async_flag,stride_flag,mmapflag,mmapasflag,mmapssflag,mmapnsflag,mmap_mix;
char verify = 1;
int restf;
char sverify = 1;
char odsync = 0;
char Q_flag,OPS_flag;
char L_flag=0;
char no_copy_flag,include_close,include_flush;
char disrupt_flag,compute_flag,xflag,Z_flag, X_flag;
int no_unlink = 0;
int no_write = 0;
int r_traj_flag,w_traj_flag;
int mygen;
char MS_flag;
int advise_op,advise_flag;
int direct_flag;
int current_client_number;
long long chid;
int file_lock;
unsigned int pattern;
long long stride = STRIDE;
long long delay,purge,fetchon;
off64_t  numrecs64 = (off64_t)NUMRECS;
long long reclen = RECLEN;
long long delay_start,depth;
VOLATILE char *stop_flag;       /* Used to stop all children */
float compute_time;
int multiplier = MULTIPLIER;
long long rest_val;
#if defined(Windows)
    HANDLE hand;
#endif

/******************************************************************************/
/* End of Tele-port zone.                                                     */
/******************************************************************************/


/* 
 * Prototypes
 * Sort of... Full prototypes break non-ansi C compilers. No protos is 
 * a bit sloppy, so the compromise is this.
 */
void child_send();
int start_child_listen();
int start_child_listen_async();
void start_child_listen_loop();
void child_listen();
void child_listen_async();
void stop_child_send();
void stop_child_listen();
void cleanup_comm();
void master_send();
int start_master_send();
int start_master_listen();
int check_filename();
void master_listen();
void stop_master_send();
void stop_master_listen();
long long start_child_proc();
int parse_client_line();
void wait_dist_join();
void tell_children_begin();
void start_master_listen_loop();
void wait_for_master_go();
void tell_master_ready();
void stop_master_listen_loop();
void tell_master_stats();
void become_client();
int pick_client();
long long start_child_proc();
int start_master_send();
void child_listen();
int start_child_listen();
void stop_master_send();
void stop_master_listen();
void stop_child_send();
void stop_child_listen();
void master_send();
void child_send();
void master_listen();
int start_master_listen();
void child_remove_files();
void terminate_child_async();
void distribute_stop();
void send_stop();
void cleanup_children();


/****************************************************************/
/*                              */
/*   MAIN ()                            */
/*                              */
/****************************************************************/

int
main(argc,argv) 
int argc;
char **argv;
{

    long long fileindx,i,tval;
    long long ind;
    int ret;
    FILE *pi;
    char reply[IBUFSIZE];
    unsigned char inp_pat;
    time_t time_run;
    char *port,*m,*subarg;
    int num_child1;
    int cret;
    int anwser,bind_cpu;
    char *evalue;


    anwser=bind_cpu=0;
    /* Used to make fread/fwrite do something better than their defaults */
    setvbuf( stdout, NULL, _IONBF, (size_t) NULL );
    setvbuf( stderr, NULL, _IONBF, (size_t) NULL );
    
    /* Save the master's name */
    gethostname(controlling_host_name,100);

    /* Let user activate mdebug or cdebug via environmental variables */
    evalue = (char *)NULL;
    evalue=(char *)getenv("CDEBUG");
    if(evalue)
        cdebug=atoi(evalue);
    evalue = (char *)NULL;
    evalue=(char *)getenv("MDEBUG");
    if(evalue)
        mdebug=atoi(evalue);

    srand(time(0));
    mygen=rand(); /* Pick a random generation number */

    /* Try to find the actual VM page size, if possible */
#if defined (solaris) || defined (_HPUX_SOURCE) || defined (linux) || defined(IRIX) || defined (IRIX64)
#ifndef __convex_spp
    page_size=getpagesize();
#endif
#endif
    /* Try to find the actual number of ticks per second */
#ifdef unix
    sc_clk_tck = clk_tck();
#endif
    for(ind=0;ind<MAXSTREAMS;ind++)
        filearray[ind]=(char *)tfile;

    /* base_time=(long)time_so_far(); */
    myid=(long long)getpid();   /* save the master's PID */
    /* get_resolution();         Get clock resolution */
    time_run = time(0);     /* Start a timer */
    (void)find_external_mon(imon_start, imon_stop);

    /*
     * Save the splash screen for later display. When in distributed network
     * mode this output does not get displayed on the clients.
     */
    sprintf(splash[splash_line++],"\tIozone: Performance Test of File I/O\n");
        sprintf(splash[splash_line++],"\t%s\n\t%s\n", THISVERSION,MODE);
        sprintf(splash[splash_line++],"\t\tBuild: %s \n\n",build_name);
        sprintf(splash[splash_line++],"\tContributors:William Norcott, Don Capps, Isom Crawford, Kirby Collins\n");
    sprintf(splash[splash_line++],"\t             Al Slater, Scott Rhine, Mike Wisner, Ken Goss\n");
        sprintf(splash[splash_line++],"\t             Steve Landherr, Brad Smith, Mark Kelly, Dr. Alain CYR,\n");
        sprintf(splash[splash_line++],"\t             Randy Dunlap, Mark Montague, Dan Million, Gavin Brebner,\n");
        sprintf(splash[splash_line++],"\t             Jean-Marc Zucconi, Jeff Blomberg, Benny Halevy, Dave Boone,\n");
        sprintf(splash[splash_line++],"\t             Erik Habbinga, Kris Strecker, Walter Wong, Joshua Root,\n");
        sprintf(splash[splash_line++],"\t             Fabrice Bacchella, Zhenghua Xue, Qin Li, Darren Sawyer.\n");
        sprintf(splash[splash_line++],"\t             Ben England.\n\n");
    sprintf(splash[splash_line++],"\tRun began: %s\n",ctime(&time_run));
    argcsave=argc;
    argvsave=argv;

        signal(SIGINT, signal_handler);     /* handle user interrupt */
        signal(SIGTERM, signal_handler);    /* handle kill from shell */

        /********************************************************/
        /* Allocate and align buffer with beginning of the  */
        /* on chip data cache.                  */
        /********************************************************/

        buffer = (char *) alloc_mem((long long)(MAXBUFFERSIZE + (2 * cache_size)),(int)0);
    if(buffer == 0) {
            perror("Memory allocation failed:");
            exit(1);
        }

#ifdef _64BIT_ARCH_
        buffer = (char *) ((long long )(buffer + cache_size ) & 
        ~(cache_size-1));
#else
        buffer = (char *) ((long)(buffer + cache_size ) & 
        ~((long)cache_size-1));
#endif
    mainbuffer = buffer;

    /* de-dup input buf */
        buffer1 = (char *) alloc_mem((long long)(MAXBUFFERSIZE + (2 * cache_size)),(int)0);
    if(buffer1 == 0) {
            perror("Memory allocation failed:");
            exit(1);
        }

#ifdef _64BIT_ARCH_
        buffer1 = (char *) ((long long )(buffer1 + cache_size ) & 
        ~(cache_size-1));
#else
        buffer1 = (char *) ((long)(buffer1 + cache_size ) & 
        ~((long)cache_size-1));
#endif
    dedup_ibuf = buffer1;
    touch_dedup(buffer1, MAXBUFFERSIZE);

#ifdef FOOB
    /* de-dup temp buf */
        buffer1 = (char *) alloc_mem((long long)(MAXBUFFERSIZE + (2 * cache_size)),(int)0);
    if(buffer1 == 0) {
            perror("Memory allocation failed:");
            exit(1);
        }

#ifdef _64BIT_ARCH_
        buffer1 = (char *) ((long long )(buffer1 + cache_size ) & 
        ~(cache_size-1));
#else
        buffer1 = (char *) ((long)(buffer1 + cache_size ) & 
        ~((long)cache_size-1));
#endif
#endif
    dedup_temp = mainbuffer;

    fetchon++;  /* By default, prefetch the CPU cache lines associated with the buffer */
    strcpy(filename,default_filename);  /* Init default filename */
    sprintf(dummyfile[0],"%s.DUMMY",default_filename);
    if(argc <=1){
        printf(USAGE);
        exit(255);
    }
    auto_mode = 0;      /* Default is to disable auto mode */
    inp_pat = PATTERN;  /* Init default pattern for verification */
    /* Fill the entire pattern variable with the same character */
    pattern = ((inp_pat << 24) | (inp_pat << 16) | (inp_pat << 8) | inp_pat);

    /*
     * Parse all of the options that the user specified.
     */
    while((cret = getopt(argc,argv,"ZQNIBDGCTOMREWovAxamwphcezKJ:j:k:V:r:t:s:f:F:d:l:u:U:S:L:H:+:P:i:b:X:Y:g:n:y:q: ")) != EOF){
        switch(cret){
        case 'k':   /* Async I/O with no bcopys */
            depth = (long long)(atoi(optarg));
            if(depth <0)
                depth=0;
            /*
            if(depth > 60)
                depth=60;
            */
#ifdef NO_PRINT_LLD
            sprintf(splash[splash_line++],"\tPOSIX Async I/O (no bcopy). Depth %ld \n",depth);
#else
            sprintf(splash[splash_line++],"\tPOSIX Async I/O (no bcopy). Depth %lld \n",depth);
#endif
            no_copy_flag=1;
            async_flag++;
            k_flag++;
            break;
        case 'T':   /* Switch to POSIX thread based */
#ifndef NO_THREADS
            use_thread++;
#else
            printf("\tThreads not supported in this version\n");
            exit(2);
#endif
            break;
        case 'H':   /* Use POSIX async_io */
            h_flag++;
            depth = (long long)(atoi(optarg));
            if(depth <0)
                depth=0;
            /*
             * Hmmm. many systems fail is strange ways when the maximum
             * number of async I/Os per user or proc is exceeded.
             */
            /*
            if(depth > 60)
                depth=60;
            */
#ifdef NO_PRINT_LLD
            sprintf(splash[splash_line++],"\tPOSIX async I/O (with bcopy). Depth %ld\n",depth);
#else
            sprintf(splash[splash_line++],"\tPOSIX async I/O (with bcopy). Depth %lld\n",depth);
#endif
            async_flag++;
            break;
        case 'I':   /* Use VXFS direct advisory or O_DIRECT from Linux or AIX , or O_DIRECTIO for TRU64  or Solaris directio */
#ifdef VXFS
            direct_flag++;
            sprintf(splash[splash_line++],"\tVxFS advanced feature SET_CACHE, VX_DIRECT enabled\n");
            break;
#endif
#if ! defined(DONT_HAVE_O_DIRECT)
#if defined(linux) || defined(__AIX__) || defined(IRIX) || defined(IRIX64) || defined(Windows) || defined(__FreeBSD__) || defined(solaris)
            direct_flag++;
            sprintf(splash[splash_line++],"\tO_DIRECT feature enabled\n");
            break;
#endif
#if defined(TRU64)
            direct_flag++;
            sprintf(splash[splash_line++],"\tO_DIRECTIO feature enabled\n");
            break;
#endif
#else
            break;
#endif
#if defined(Windows)
            sprintf(splash[splash_line++],"\tO_DIRECTIO feature not available in Windows version.\n");
            break;
#endif
        case 'B':   /* Use mmap file for test file */
            sprintf(splash[splash_line++],"\tUsing mmap files\n");
            mmapflag++;
            mmapnsflag++;
            break;
        case 'D':   /* Use async msync mmap file */
            sprintf(splash[splash_line++],"\tUsing msync(MS_ASYNC) on mmap files\n");
            mmapflag++;
            mmapasflag++;
            mmapnsflag=0;
            break;
        case 'G':   /* Use msync sync for mmap file */
            sprintf(splash[splash_line++],"\tUsing msync(MS_SYNC) on mmap files\n");
            mmapssflag++;
            mmapnsflag=0;
            break;
        case 'C':   /* show children xfer counts */
            Cflag++;
            break;
        case 'Q':   /* Enable output offset/latency files */
                sprintf(splash[splash_line++],"\tOffset/latency files enabled.\n");
            Q_flag++;
            break;
        case 'x':   /* Disable stone_wall */
                sprintf(splash[splash_line++],"\tStonewall disabled\n");
            xflag++;
            break;

        case 'a':   /* auto mode */
            fetchon=1;
            purge=0;
            multi_buffer=0;
                auto_mode = 1;
            aflag++;
                sprintf(splash[splash_line++],"\tAuto Mode\n");
            break;
        case 'c':   /* Include close in timing */
            include_close++;
                sprintf(splash[splash_line++],"\tInclude close in write timing\n");
            break;
        case 'e':   /* Include fsync in timing */
            include_flush++;
                sprintf(splash[splash_line++],"\tInclude fsync in write timing\n");
            break;
        case 'A':   /* auto2 mode. Soon to go away. Please use -az */
            fetchon=1;
            purge=0;
            multi_buffer=0;
                auto_mode = 1;
            aflag++;
                sprintf(splash[splash_line++],"\tAuto Mode 2. This option is obsolete. Use -az -i0 -i1 \n");
            RWONLYflag++;
            NOCROSSflag++;
            include_tflag++;    /* automatically set WRITER_TEST and READER_TEST */
            include_test[WRITER_TEST]++;
            include_test[READER_TEST]++;
            break;
        case 's':   /* Set file size */
#ifdef NO_PRINT_LLD
            sscanf(optarg,"%ld",&kilobytes64);
#else
            sscanf(optarg,"%lld",&kilobytes64);
#endif
            if(optarg[strlen(optarg)-1]=='k' ||
                optarg[strlen(optarg)-1]=='K'){
                ;
            }
            if(optarg[strlen(optarg)-1]=='m' ||
                optarg[strlen(optarg)-1]=='M'){
                kilobytes64 = kilobytes64 * 1024;
            }
            if(optarg[strlen(optarg)-1]=='g' ||
                optarg[strlen(optarg)-1]=='G'){
                kilobytes64 = kilobytes64 *1024 * 1024;
            }
            if(kilobytes64 <= 0)
                kilobytes64=512;

            s_range[s_count++]=kilobytes64;
            max_file_size = (off64_t)s_range[s_count-1];   /* Make visable globally */
            min_file_size = (off64_t)s_range[0];   /* Make visable globally */

#ifdef NO_PRINT_LLD
                sprintf(splash[splash_line++],"\tFile size set to %ld KB\n",kilobytes64);
#else
                sprintf(splash[splash_line++],"\tFile size set to %lld KB\n",kilobytes64);
#endif
            sflag++;
            break;
        case 'l':   /* Set lower thread/proc limit  */
            mint = (long long)(atoi(optarg));
            if(mint <= 0)
            {
                mint=1;
                num_child=1;
            }else
                num_child=mint;
            if(mint > (unsigned long long)MAXSTREAMS){
              printf("Invalid options: maximum streams for ");
              printf("throughput is MAXSTREAMS\n");
              exit(4);
            }
            lflag++;
            trflag++;
            if(Uflag)
            {
                printf("Can not run throughput tests with unmount & remounts.\n");
                exit(5);
            }
            break;
        case 'u':   /* Set upper thread/proc limit  */
            maxt = (long long)(atoi(optarg));
            if(maxt <= 0)
                maxt=1;
            if(maxt > MAXSTREAMS){
              printf("Invalid options: maximum streams for ");
              printf("throughput is MAXSTREAMS\n");
              exit(6);
            }
            uflag++;
            trflag++;
            if(Uflag)
            {
                printf("Can not run throughput tests with unmount & remounts.\n");
                exit(7);
            }
            break;
        case 'm':   /* Use multiple buffers */
            fetchon=0;
            multi_buffer=1;
            mflag++;
                mbuffer = (char *) alloc_mem((long long)MAXBUFFERSIZE,(int)0);
            if(mbuffer == 0) {
                            perror("Memory allocation failed:");
                            exit(8);
            }
                sprintf(splash[splash_line++],"\tMulti_buffer. Work area %d bytes\n",
                MAXBUFFERSIZE);
            break;
                case 'M':       /* Report machine name and OS */
            bzero(reply,sizeof(reply));
                        pi=popen("uname -a", "r");
            if(pi == (FILE *)0)
            {
                sprintf(splash[splash_line++],"\n\tError using popen() on uname\n");
                sprintf(splash[splash_line++],"\t-M option suppressed.\n");
            }
            else
            {
                            junk=fread(reply,IBUFSIZE-1,1,pi);
                            pclose(pi);
                m=reply;
                            while(*m) /* Strip new line */
                {
                    if(*m=='\n')
                                        *m=0;
                    else    
                                        m++;
                }
                            sprintf(splash[splash_line++],"\n\tMachine = %s\n",reply);
            }
                        break;

        case 'P':   /* Set beginning processor for binding. */
#ifndef NO_THREADS
#if defined(_HPUX_SOURCE) || defined(linux)
#if defined(_HPUX_SOURCE)
            num_processors= pthread_num_processors_np();
#else
      num_processors = sysconf(_SC_NPROCESSORS_ONLN);
#endif      
            begin_proc = atoi(optarg);
            if(begin_proc < 0)
                begin_proc=0;
            if(begin_proc > num_processors)
                begin_proc=0;
                        sprintf(splash[splash_line++],"\tBinding of processors beginning with %d \n",begin_proc);
            ioz_processor_bind++;
#else
            sprintf(splash[splash_line++],"\tProcessor binding not available in this version\n");
#endif
#endif
                        break;
        case 'p':   /* purge the processor cache */
                sprintf(splash[splash_line++],"\tPurge Mode On\n");
            fetchon=0;
            pflag++;
            purge=1;
            break;
        case 'h':   /* show help */
            hflag++;
                show_help();
            exit(0);
            break;
        case 'E':   /* Extended testing for pread/pwrite... */
            Eflag++;
            break;
        case 'R':   /* Generate Excel compatible Report */
            Rflag++;
                sprintf(splash[splash_line++],"\tExcel chart generation enabled\n");
            break;
        case 'o':   /* Open OSYNC */
                sprintf(splash[splash_line++],"\tSYNC Mode. \n");
            oflag++;
            break;
        case 'O':   /* Report in Ops/sec instead of KB/sec */
                sprintf(splash[splash_line++],"\tOPS Mode. Output is in operations per second.\n");
            OPS_flag++;
            break;
        case 'N':   /* Report in usec/op  */
                sprintf(splash[splash_line++],"\tMicroseconds/op Mode. Output is in microseconds per operation.\n");
            MS_flag++;
            break;
        case 'V':   /* Turn on Verify every byte */
            sverify=0;
            inp_pat = (char)(atoi(optarg));
            if(inp_pat == 0)
                inp_pat = PATTERN;
            pattern = ((inp_pat << 24) | (inp_pat << 16) | (inp_pat << 8) 
                | inp_pat);
            verify=1;
                sprintf(splash[splash_line++],"\tVerify Mode. Pattern %x\n",pattern);
                sprintf(splash[splash_line++],"\tPerformance measurements are invalid in this mode.\n");
            break;
        case 'S':   /* Set the processor cache size */
            cache_size = (long)(atoi(optarg)*1024);
            if(cache_size == 0)
                cache_size = CACHE_SIZE;
            break;
        case 'L':   /* Set processor cache line size */
            cache_line_size = (long)(atoi(optarg));
            if(cache_line_size == 0)
                cache_line_size = CACHE_LINE_SIZE;
            break;
        case 'f':   /* Specify the file name */
            if(mfflag) {
              printf("invalid options: -f and -F are mutually exclusive\n");
              exit(10);
            }
            fflag++;
            strcpy(filename,optarg);
            sprintf(dummyfile[0],"%s.DUMMY",optarg);
            break;
        case 'b':   /* Specify the biff file name */
            Rflag++;
            bif_flag++;
            strcpy(bif_filename,optarg);
            break;
        case 'F':   /* Specify multiple file names for -t */
            mfflag++;
            if(fflag) {
              printf("invalid options: -f and -F are mutually exclusive\n");
              exit(11);
            }
            if(!trflag) {
              printf("invalid options: must specify -t N before -F\n");
              exit(12);
            }
            optind--;
            for(fileindx=0;fileindx<maxt;fileindx++) {
              filearray[fileindx]=argv[optind++];
              if(optind > argc) {
#ifdef NO_PRINT_LLD
                printf("invalid options: not enough filenames for %ld streams\n",num_child);
#else
                printf("invalid options: not enough filenames for %lld streams\n",num_child);
#endif
                exit(13);
              }
            }
            break;
        case 'r':   /* Specify the record size to use */
            rflag++;
            reclen = ((long long)(atoi(optarg))*1024);
            if(optarg[strlen(optarg)-1]=='k' ||
                optarg[strlen(optarg)-1]=='K'){
                reclen = (long long)(1024 * atoi(optarg));
            }
            if(optarg[strlen(optarg)-1]=='m' ||
                optarg[strlen(optarg)-1]=='M'){
                reclen = (long long)(1024 * 1024 * atoi(optarg));
            }
            if(optarg[strlen(optarg)-1]=='g' ||
                optarg[strlen(optarg)-1]=='G'){
                reclen = (long long)(1024 * 1024 * 1024 *(long long)atoi(optarg));
            }
            if(reclen <= 0)
                reclen=(long long)4096;

            r_range[r_count++]=reclen;
            max_rec_size = (off64_t)r_range[r_count-1];   /* Make visable globally */
            min_rec_size = (off64_t)r_range[0];   /* Make visable globally */
#ifdef NO_PRINT_LLD
                sprintf(splash[splash_line++],"\tRecord Size %ld KB\n",reclen/1024);
#else
                sprintf(splash[splash_line++],"\tRecord Size %lld KB\n",reclen/1024);
#endif
            if(max_rec_size > MAXBUFFERSIZE) {
#ifdef NO_PRINT_LLD
                printf("Error: maximum record size %ld KB is greater than maximum buffer size %ld KB\n ",
                    max_rec_size/1024, MAXBUFFERSIZE/1024);
#else
                printf("Error: maximum record size %lld KB is greater than maximum buffer size %lld KB\n ",
                    (long long)(max_rec_size/1024LL), (long long)MAXBUFFERSIZE/1024LL);
#endif
                exit(23);
            }
            break;
        case 'J':   /* Specify the compute time in millisecs */
            compute_time = (float)(atoi(optarg));
            compute_time=compute_time/1000; 
            if(compute_time < (float)0)
                compute_time=(float)0;
            else
                compute_flag=1;
            jflag++;
            break;
        case 'j':   /* Specify the stride in records */
            stride = (long long)(atoi(optarg));
            if(stride < 0)
                stride=0;
            stride_flag=1;
            break;
        case 't':   /* Specify the number of children to run */
            num_child1=(atoi(optarg));
            num_child = (long long)num_child1;
            if(num_child > (long long)MAXSTREAMS) {
              printf("invalid options: maximum streams for throughput is MAXSTREAMS\n");
#ifdef NO_PRINT_LLD
              printf("Numchild %ld %s\n",num_child,optarg);
#else
              printf("Numchild %lld %s\n",num_child,optarg);
#endif
              exit(14);
            }
            if(num_child <= 0)
                num_child = 8;
            if(num_child == 0)
                num_child=1;
                        t_range[t_count++]=num_child;
                        maxt = (maxt>num_child?maxt:num_child);
            trflag++;
            if(Uflag)
            {
                printf("Can not run throughput tests with unmount & remounts.\n");
                exit(15);
            }
            break;
        case 'd':   /* Specify the delay of children to run */
            delay_start = (long long)(atoi(optarg));
            if(delay_start < 0)
                delay_start=0;
            break;
        case 'i':   /* Specify specific tests */
            tval=(long long)(atoi(optarg));
            if(tval < 0) tval=0;
#ifndef HAVE_PREAD
            if(tval > RANDOM_MIX_TEST)
            {
                printf("\tPread tests not available on this operating system.\n");
                exit(183);
            }
#endif
            if(tval > sizeof(func)/sizeof(char *)) 
            {
                tval=0;
                sprintf(splash[splash_line++],"\tSelected test not available on the version.\n");
            }
            include_test[tval]++;
            include_tflag++;
            break;
        case 'v':   /* Show version information */
                for(ind=0; strlen(head1[ind]); ind++)
                {
                printf("%s\n", head1[ind]);
                }
            exit(0);
            break;
        case 'U':   /* Specify the dev name for umount/mount*/
            Uflag++;
            strcpy(mountname,optarg);
            if(trflag)
            {
                printf("Can not run throughput tests with unmount & remounts.\n");
                exit(16);
            }
            break;
        case 'w':   /* Do not unlink files */
            sprintf(splash[splash_line++],"\tSetting no_unlink\n");
            no_unlink = 1;
            break;
        case 'Z':   /* Turn on the mmap and file I/O mixing */
            sprintf(splash[splash_line++],"\tEnable mmap & file I/O mixing.\n");
            mmap_mix = 1;
            break;
        case 'W':   /* Read/Write with file locked */
            file_lock=1;
            sprintf(splash[splash_line++],"\tLock file when reading/writing.\n");
            break;
        case 'K':   /* Cause disrupted read pattern */
            disrupt_flag=1;
            sprintf(splash[splash_line++],"\tDisrupted read patterns selected.\n");
            break;
        case 'X':   /* Open write telemetry file */
            compute_flag=1;
            sverify=2;  /* touch lightly */
            w_traj_flag=1;
            strcpy(write_traj_filename,optarg);
            traj_vers();
            w_traj_size();
            sprintf(splash[splash_line++],"\tUsing write telemetry file \"%s\"\n",
                write_traj_filename);
            w_traj_fd=open_w_traj();
            if(w_traj_fd == (FILE *)0)
                exit(200);
            break;
        case 'Y':   /* Open Read telemetry file */
            compute_flag=1;
            sverify=2;  /* touch lightly */
            r_traj_flag=1;
            strcpy(read_traj_filename,optarg);
            sprintf(splash[splash_line++],"\tUsing read telemetry file \"%s\"\n",
                read_traj_filename);
            traj_vers();
            r_traj_size();
            r_traj_fd=open_r_traj();
            if(r_traj_fd == (FILE*) 0)
                exit(200);
            break;
        case 'n':   /* Set min file size for auto mode */
            nflag=1;
            minimum_file_size = (off64_t)atoi(optarg);
            if(optarg[strlen(optarg)-1]=='k' ||
                optarg[strlen(optarg)-1]=='K'){
                ;
            }
            if(optarg[strlen(optarg)-1]=='m' ||
                optarg[strlen(optarg)-1]=='M'){
                minimum_file_size = (long long)(1024 * atoi(optarg));
            }
            if(optarg[strlen(optarg)-1]=='g' ||
                optarg[strlen(optarg)-1]=='G'){
                minimum_file_size = (long long)(1024 * 1024 * (long long)atoi(optarg));
            }
            if(minimum_file_size < RECLEN_START/1024)
                minimum_file_size=(off64_t)(RECLEN_START/1024);
            if(minimum_file_size < page_size/1024)
                minimum_file_size=(off64_t)(page_size/1024);
#ifdef NO_PRINT_LLD
            sprintf(splash[splash_line++],"\tUsing minimum file size of %ld kilobytes.\n",minimum_file_size);
#else
            sprintf(splash[splash_line++],"\tUsing minimum file size of %lld kilobytes.\n",minimum_file_size);
#endif
            break;
        case 'g':   /* Set maximum file size for auto mode */
            gflag=1;
            maximum_file_size = (off64_t)atoi(optarg);
            if(optarg[strlen(optarg)-1]=='k' ||
                optarg[strlen(optarg)-1]=='K'){
                ;
            }
            if(optarg[strlen(optarg)-1]=='m' ||
                optarg[strlen(optarg)-1]=='M'){
                maximum_file_size = (long long)(1024 * atoi(optarg));
            }
            if(optarg[strlen(optarg)-1]=='g' ||
                optarg[strlen(optarg)-1]=='G'){
                maximum_file_size = (long long)(1024 * 1024 * (long long)atoi(optarg));
            }
            if(maximum_file_size < RECLEN_START/1024)
                maximum_file_size=(off64_t)(RECLEN_START/1024);
#ifdef NO_PRINT_LLD
            sprintf(splash[splash_line++],"\tUsing maximum file size of %ld kilobytes.\n",maximum_file_size);
#else
            sprintf(splash[splash_line++],"\tUsing maximum file size of %lld kilobytes.\n",maximum_file_size);
#endif
            break;
        case 'z':   /* Set no cross over */
            sprintf(splash[splash_line++],"\tCross over of record size disabled.\n");
            NOCROSSflag=1;
            break;
        case 'y':       /* Set min record size for auto mode */
            yflag=1;
            min_rec_size = ((long long)(atoi(optarg))*1024);
            if(optarg[strlen(optarg)-1]=='k' ||
                optarg[strlen(optarg)-1]=='K'){
                min_rec_size = (long long)(1024 * atoi(optarg));
            }
            if(optarg[strlen(optarg)-1]=='m' ||
                optarg[strlen(optarg)-1]=='M'){
                min_rec_size = (long long)(1024 * 1024 * atoi(optarg));
            }
            if(optarg[strlen(optarg)-1]=='g' ||
                optarg[strlen(optarg)-1]=='G'){
                min_rec_size = (long long)(1024 * 1024 * 1024 *(long long)atoi(optarg));
            }
            if(min_rec_size <= 0)
                min_rec_size=(long long)RECLEN_START;
#ifdef NO_PRINT_LLD
                sprintf(splash[splash_line++],"\tUsing Minimum Record Size %ld KB\n", min_rec_size/1024);
#else
                sprintf(splash[splash_line++],"\tUsing Minimum Record Size %lld KB\n", min_rec_size/1024);
#endif
            break;
        case 'q':       /* Set max record size for auto mode */
            qflag=1;
            max_rec_size = ((long long)(atoi(optarg))*1024);
            if(optarg[strlen(optarg)-1]=='k' ||
                optarg[strlen(optarg)-1]=='K'){
                max_rec_size = (long long)(1024 * atoi(optarg));
            }
            if(optarg[strlen(optarg)-1]=='m' ||
                optarg[strlen(optarg)-1]=='M'){
                max_rec_size = (long long)(1024 * 1024 * atoi(optarg));
            }
            if(optarg[strlen(optarg)-1]=='g' ||
                optarg[strlen(optarg)-1]=='G'){
                max_rec_size = (long long)(1024 * 1024 * 1024 *(long long)atoi(optarg));
            }
            if(max_rec_size <= 0)
                min_rec_size=(long long)RECLEN_END;
            if(max_rec_size > MAXBUFFERSIZE) {
#ifdef NO_PRINT_LLD
                printf("Error: maximum record size %ld KB is greater than maximum buffer size %ld KB\n ",
                    max_rec_size/1024, MAXBUFFERSIZE/1024);
#else
                printf("Error: maximum record size %lld KB is greater than maximum buffer size %lld KB\n ",
                    (long long)(max_rec_size/1024LL), (long long)MAXBUFFERSIZE/1024LL);
#endif
                exit(23);
            }
#ifdef NO_PRINT_LLD
            sprintf(splash[splash_line++],"\tUsing Maximum Record Size %ld KB\n", max_rec_size/1024);
#else
            sprintf(splash[splash_line++],"\tUsing Maximum Record Size %lld KB\n", max_rec_size/1024);
#endif
            break;

        /* 
         * The + operator is for the new extended options mechanism 
         * Syntax is -+ followed by option leter, and if the optino
         * takes an operand  then it is implemented below. An example
         * -+a arg    is shown below. This is a sub option with an argument.
         * -+b  is shown below. This is a sub option with no argument.
         */
        case '+':
            /* printf("Plus option = >%s<\n",optarg);*/
            switch (*((char *)optarg))
            {
                case 'a':  /* Example: Has argument */
                    subarg=argv[optind++];
                    /* if(subarg!=(char *)0)   Error checking. */
                    /* printf("Plus option argument = >%s<\n",subarg);*/
                    break;
                case 'b':  /* Example: Does not have an argument */
                    break;
                case 'c':  /* Argument is the controlling host name */
                    /* I am a client for distributed Iozone */
                    subarg=argv[optind++];
                    if(subarg==(char *)0)
                    {
                         printf("-+c takes an operand !!\n");
                         exit(200);
                    }
                    strcpy(controlling_host_name,subarg);
                    distributed=1;
                    client_iozone=1;
                    master_iozone=0;
                    break;
                                case 'h':  /* Argument is the controlling host name */
                                        subarg=argv[optind++];
                                        if(subarg==(char *)0)
                                        {
                                             printf("-+h takes an operand !!\n");
                                             exit(200);
                                        }
                                        strcpy(controlling_host_name,subarg);
                                        sprintf(splash[splash_line++],"\tHostname = %s\n",controlling_host_name);
                                        break;
                case 'm':  /* I am the controlling process for distributed Iozone */
                       /* Does not have an argument */
                    subarg=argv[optind++];
                    if(subarg==(char *)0)
                    {
                         printf("-+m takes an operand. ( filename )\n");
                         exit(201);
                    }
                    strcpy(client_filename,subarg);
                    ret=get_client_info();
                    if(ret <= 0)
                    {
                        printf("Error reading client file\n");
                        exit(178);
                    }
                    clients_found=ret;
                    distributed=1;
                    master_iozone=1;
                    client_iozone=0;
                    sprintf(splash[splash_line++],"\tNetwork distribution mode enabled.\n");
                    break;
                case 'N':  /* turn off truncating the file before write test */
                    notruncate = 1;
                    break;
                case 'u':   /* Set CPU utilization output flag */
                    cpuutilflag = 1;    /* only used if R(eport) flag is also set */
                    get_rusage_resolution();
                        sprintf(splash[splash_line++],"\tCPU utilization Resolution = %5.3f seconds.\n",cputime_res);
                        sprintf(splash[splash_line++],"\tCPU utilization Excel chart enabled\n");
                    break;
                case 's':  /* Clients operate in silent mode. */
                       /* Does not have an argument */
                    silent=1;
                    break;
                case 'd':  /* Diagnostics mode */
                    sprintf(splash[splash_line++],"\t>>> I/O Diagnostic mode enabled. <<<\n");
                        sprintf(splash[splash_line++],"\tPerformance measurements are invalid in this mode.\n");
                    diag_v=1;
                    sverify=0;
                    break;
                case 'x':  /* Argument is the multiplier for rec size and file size */
                    subarg=argv[optind++];
                    if(subarg==(char *)0)
                    {
                         printf("-+c takes an operand !!\n");
                         exit(200);
                    }
                    multiplier = atoi(subarg);
                    if(multiplier <=1)
                        multiplier = 2;
                    break;
                case 'i':  /* Argument is the host port */
                    subarg=argv[optind++];
                    if(subarg==(char *)0)
                    {
                         printf("-+i takes an operand !!\n");
                         exit(200);
                    }
                    controlling_host_port = atoi(subarg);
                    break;
                case 'p':  /* Argument is the percentage read */
                    subarg=argv[optind++];
                    if(subarg==(char *)0)
                    {
                         printf("-+p takes an operand !!\n");
                         exit(200);
                    }
                    pct_read = atoi(subarg);
                    if(pct_read < 1)
                        pct_read = 1;
                    if(pct_read >=100)
                        pct_read = 100;
                        sprintf(splash[splash_line++],"\tPercent read in mix test is %d\n",pct_read);
                    break;
                case 't':  /* Speed code activated */
                    speed_code=1;
                    break;
#if defined(_HPUX_SOURCE) || defined(linux) || defined(solaris)
                case 'r':  /* Read sync too */
                    read_sync=1;
                        sprintf(splash[splash_line++],"\tRead & Write sync mode active.\n");
                    break;
#endif
#ifndef NO_MADVISE
                case 'A':  /* Argument is madvise selector */
                    subarg=argv[optind++];
                    if(subarg==(char *)0)
                    {
                       printf("-+A take an operand !!\n");
                       exit(200);
                    }
                    advise_flag=1;
                    advise_op=atoi(subarg);
                    sprintf(splash[splash_line++],"\tMadvise enabled: %d\n",advise_op);
                    break;
#endif
                case 'n':   /* Set no-retest */
                    noretest = 1;   
                        sprintf(splash[splash_line++],"\tNo retest option selected\n");
                    break;
                case 'k':   /* Constant aggregate data set size */
                    aggflag=1;
                    break;
                case 'q':  /* Argument is the rest time between tests in seconds */
                    subarg=argv[optind++];
                    if(subarg==(char *)0)
                    {
                         printf("-+q takes an operand !!\n");
                         exit(200);
                    }
                    rest_val = (long long)atoi(subarg);
                    if(rest_val <=0)
                        rest_val = 0;
                    restf=1;
                    sprintf(splash[splash_line++],"\tDelay %d seconds between tests enabled.\n",atoi(subarg));
                    break;
#if defined(O_DSYNC)
                case 'D':  /* O_DSYNC mode */
                    sprintf(splash[splash_line++],"\t>>> O_DSYNC mode enabled. <<<\n");
                    odsync=1;
                    break;
#endif
                case 'l':  /* Record locking mode */
                    sprintf(splash[splash_line++],"\t>>> Record locking mode enabled. <<<\n");
                    rlocking=1;
                    break;
                case 'L':  /* Record locking mode shared files*/
                    sprintf(splash[splash_line++],"\t>>> Record locking, shared file mode enabled. <<<\n");
                    share_file=1;
                    rlocking=1;
                    break;
                case 'V':  /* No Record locking shared files*/
                    sprintf(splash[splash_line++],"\t>>> Shared file mode enabled. <<<\n");
                    share_file=1;
                    break;
                case 'B':  /* Sequential mix */
                    sprintf(splash[splash_line++],"\t>>> Sequential Mixed workload. <<<\n");
                    seq_mix=1;
                    break;
                        /* Use an existing user file, that does
                     not contain Iozone's pattern. Use file
                                     for testing, but read only, and no 
                                         delete at the end of the test. Also, 
                                         no pattern verification, but do touch
                                         the pages. */
                case 'E':  
                    sprintf(splash[splash_line++],"\t>>> No Verify mode. <<<\n");
                    sverify=2;
                    no_unlink=1;
                    no_write=1;
                    break;
                case 'T':  /* Time stamps on */
                    L_flag=1;
                    break;
                case 'X': /* Short circuit test mode */
                    X_flag = 1;
                    sverify=1;
                    verify=1;
                    inp_pat = 0xBB;
                    pattern = ((inp_pat << 24) | 
                      (inp_pat << 16) | (inp_pat << 8) |
                       inp_pat);
                        sprintf(splash[splash_line++],"\tShort circuit mode. For\n");
                        sprintf(splash[splash_line++],"\t filesystem development testing ONLY !\n");
                    break;
                case 'Z': /* Compatibility mode for 0xA5 */ 
                    Z_flag = 1;
                    sverify=1;
                    verify=1;
                    inp_pat = 0xA5;
                    pattern = ((inp_pat << 24) | 
                      (inp_pat << 16) | (inp_pat << 8) |
                       inp_pat);
                        sprintf(splash[splash_line++],"\tUsing old data sets.\n");
                        sprintf(splash[splash_line++],"\t Performance measurements may be invalid in this\n");
                        sprintf(splash[splash_line++],"\t mode due to published hack.\n");
                    break;
#if defined(Windows)
                case 'U':  /* Windows only Unbufferd I/O */
                    unbuffered=1;
                    sprintf(splash[splash_line++],"\tUnbuffered Windows API usage. >>> Very Experimental <<<\n");
                    break;
#endif
                case 'K':  /* Sony special for manual control of test 8 */
                    subarg=argv[optind++];
                    if(subarg==(char *)0)
                    {
                         printf("-+K takes an operand !!\n");
                         exit(204);
                    }
                    Kplus_readers = (int)atoi(subarg);
                    if(Kplus_readers <=0)
                        Kplus_readers = 1;
                    Kplus_flag=1;
                    sprintf(splash[splash_line++],"\tManual control of test 8. >>> Very Experimental. Sony special <<<\n");
                    break;
                case 'w':  /* Argument is the percent of dedup */
                       /* Sets size of dedup region across files */
                    subarg=argv[optind++];
                    if(subarg==(char *)0)
                    {
                         printf("-+w takes an operand !!\n");
                         exit(200);
                    }
                    dedup = atoi(subarg);
                    if(dedup <=0)
                        dedup = 0;
                    if(dedup >100)
                        dedup = 100;
                    sprintf(splash[splash_line++],"\tDedup activated %d percent.\n",dedup);
                    break;
                case 'y':  /* Argument is the percent of interior dedup */
                       /* Sets size of dedup region within and across files */
                    subarg=argv[optind++];
                    if(subarg==(char *)0)
                    {
                         printf("-+y takes an operand !!\n");
                         exit(200);
                    }
                    dedup_interior = atoi(subarg);
                    if(dedup_interior <0)
                        dedup_interior = 0;
                    if(dedup_interior >100)
                        dedup_interior = 100;
                    sprintf(splash[splash_line++],"\tDedupe within & across %d percent.\n",dedup_interior);
                    break;
                case 'C':  /* Argument is the percent of dedupe within & !across */
                       /* Sets size of dedup region within and !across files */
                    subarg=argv[optind++];
                    if(subarg==(char *)0)
                    {
                         printf("-+C takes an operand !!\n");
                         exit(200);
                    }
                    dedup_compress = atoi(subarg);
                    if(dedup_compress <0)
                        dedup_compress = 0;
                    if(dedup_compress >100)
                        dedup_compress = 100;
                    sprintf(splash[splash_line++],"\tDedupe within %d percent.\n",dedup_compress);
                    break;
                case 'S':  /* Argument is the seed for dedup */
                    subarg=argv[optind++];
                    if(subarg==(char *)0)
                    {
                         printf("-+S takes an operand !!\n");
                         exit(200);
                    }
                    dedup_mseed = atoi(subarg);
                    if(dedup_mseed ==0)
                        dedup_mseed = 1;
                    sprintf(splash[splash_line++],"\tDedup manual seed %d .\n",dedup_mseed);
                    break;
                case 'H':  /* Argument is hostname of the PIT */
                    subarg=argv[optind++];
                    if(subarg==(char *)0)
                    {
                         printf("-+H takes operand !!\n");
                         exit(200);
                    }
                    strcpy(pit_hostname,subarg);
                    sprintf(splash[splash_line++],"\tPIT_host %s\n",pit_hostname);
                    
                    break;
                case 'P':  /* Argument is port of the PIT */
                    subarg=argv[optind++];
                    if(subarg==(char *)0)
                    {
                         printf("-+P takes operand !!\n");
                         exit(200);
                    }
                    strcpy(pit_service,subarg);
                    sprintf(splash[splash_line++],"\tPIT_port %s\n",pit_service);
                    break;
                case 'z':  /* Enable hist summary*/
                    hist_summary=1;
                    sprintf(splash[splash_line++],"\tHistogram summary enabled.\n");
                    break;
                case 'O':  /* Argument is the Op rate */
                    subarg=argv[optind++];
                    if(subarg==(char *)0)
                    {
                         printf("-+O takes an operand !!\n");
                         exit(200);
                    }
                    op_rate = atoi(subarg);
                    if(op_rate <= 0)
                        op_rate = 1;
                    op_rate_flag = 1;
                    sprintf(splash[splash_line++],"\tRate control active %d Ops/sec .\n",op_rate);
                    break;
                default:
                    printf("Unsupported Plus option -> %s <-\n",optarg);
                    exit(255);
                    break;
            }   
            break;
        default:
            printf("Unsupported option -> %s <-\n",optarg);
            exit(255);
        }
    }
    base_time=(long)time_so_far();
    get_resolution();       /* Get clock resolution */
    if(speed_code)
    {
        do_speed_check(client_iozone);
        exit(0);
    }
    if(r_count > 1)
    {
        aflag=1;
        rflag=0;
        NOCROSSflag=1;
    }
    if(s_count > 1)
    {
        aflag=1;
        sflag=0;
        NOCROSSflag=1;
    }
    /*
         * If not in silent mode then display the splash screen.
     */
    for(i=0;i<splash_line;i++)
        if(!silent) printf("%s",splash[i]);
    /*
     * Save the command line for later 
     */
    record_command_line(argcsave, argvsave);

    if(pflag) /* Allocate after cache_size is set */
    {
        alloc_pbuf();
    }
    if(distributed && master_iozone)
    {
        if(maxt > clients_found)
        {
            printf("You can not specify more threads/processes than you have in the client file list\n");
            exit(202);
        }
    }
    
    if(!OPS_flag && !MS_flag)
    {
        if(!silent) printf("\tOutput is in Kbytes/sec\n");
    }
    if (min_rec_size > max_rec_size) {
#ifdef NO_PRINT_LLD
        printf("Error: minimum record size %ld KB is greater than maximum record size %ld KB\n ",
            min_rec_size/1024, max_rec_size/1024);
#else
        printf("Error: minimum record size %lld KB is greater than maximum record size %lld KB\n ",
            min_rec_size/1024, max_rec_size/1024);
#endif
        exit(23);
        }
    orig_min_rec_size=min_rec_size;
    orig_max_rec_size=max_rec_size;
    /*
     * No telemetry files... just option selected 
     */
    if(compute_flag && jflag  && !(r_traj_flag || w_traj_flag))
        if(!silent) printf("\tCompute time %f seconds for reads and writes.\n",compute_time);
    /*
     * Read telemetry file and option selected
     */
    if(compute_flag && r_traj_flag && !w_traj_flag)
    {
        if(r_traj_items==3)
        {
            if(!silent) printf("\tCompute time from telemetry files for reads.\n");
        }
        else
        {
            if(jflag)
                if(!silent) printf("\tCompute time %f seconds for reads.\n",compute_time);
        }
        if(jflag)
            if(!silent) printf("\tCompute time %f seconds for writes.\n",compute_time);
    }
    /*
     * Write telemetry file and option selected
     */
    if(compute_flag && !r_traj_flag && w_traj_flag)
    {
        if(w_traj_items==3)
        {
            if(!silent) printf("\tCompute time from telemetry files for writes.\n");
        }
        else
        {
            if(jflag)
                if(!silent) printf("\tCompute time %f seconds for writes.\n",compute_time);
        }
        if(jflag)
            if(!silent) printf("\tCompute time %f seconds for reads.\n",compute_time);
    }
    if(compute_flag && r_traj_flag && w_traj_flag && jflag)
    {
        if(r_traj_items==3)
        {
            if(!silent) printf("\tCompute time from telemetry files for reads.\n");
        }
        else
        {
            if(!silent) printf("\tCompute time %f seconds for reads.\n",compute_time);
        }
        if(w_traj_items==3) 
        {
            if(!silent) printf("\tCompute time from telemetry files for writes.\n");
        }
        else
        {
            if(!silent) printf("\tCompute time %f seconds for writes.\n",compute_time);
        }
    }
    if(compute_flag && r_traj_flag && w_traj_flag && !jflag)
    {
        if(r_traj_items==3)
        {
            if(!silent) printf("\tCompute time from telemetry files for reads.\n");
        }
        else
        {
            if(!silent) printf("\tNo compute time for reads.\n");
        }

        if(w_traj_items==3) 
        {
            if(!silent) printf("\tCompute time from telemetry files for writes.\n");
        }
        else
        {
            if(!silent) printf("\tNo compute time for writes.\n");
        }
    }

        /* Enforce only write,rewrite,read,reread */
        if(w_traj_flag || r_traj_flag)
        {
                for(i=2;i<sizeof(func)/sizeof(char *);i++)
                {
                        if(seq_mix && (i==8))
                                ;
                        else
                                include_test[i] = 0;
                }
        }

    if(r_traj_flag)
    {
        if(include_test[READER_TEST] == 0) 
        {
            include_test[WRITER_TEST]=1;
            include_test[READER_TEST]=1;
            include_tflag=1;
        }
    }
    if(w_traj_flag)
    {
        if(include_test[WRITER_TEST] == 0) 
        {
            include_test[WRITER_TEST]=1;
            include_tflag=1;
        }
    }
    if(w_traj_flag && w_traj_fsize != 0)
        kilobytes64=w_traj_fsize/1024;
    if(r_traj_flag && r_traj_fsize != 0)
        kilobytes64=r_traj_fsize/1024;

    if( sverify==0 && (w_traj_flag || r_traj_flag))
    {
        printf("\n\tFull verification not supported in telemetry mode.\n\n");
        exit(17);
    }
    ;
    if(disrupt_flag &&(w_traj_flag || r_traj_flag) )
    {
        printf("\n\tDisrupt not supported in telemetry mode.\n\n");
        exit(17);
    }
    if(aflag &&(w_traj_flag || r_traj_flag) )
    {
        printf("\n\tAuto mode not supported in telemetry mode.\n");
        printf("\tTry:   -i 0 -i 1 \n\n");
        exit(17);
    }
    if(sflag && w_traj_flag )
    {
        printf("\n\tSize of file is determined by telemetry file.\n\n");
        exit(17);
    }
    if(rflag && w_traj_flag )
    {
        printf("\n\tRecord size of file is determined by telemetry file.\n\n");
        exit(17);
    }
    if(stride_flag && (w_traj_flag || r_traj_flag))
    {
        printf("\n\tStride size is determined by telemetry file.\n\n");
        exit(17);
    }
    if(trflag && MS_flag)
    {
        printf("\n\tMicrosecond mode not supported in throughput mode.\n\n");
        exit(17);
    }
    if (trflag  /* throughput mode, don't allow auto-mode options: */
        && (auto_mode || aflag || yflag || qflag || nflag || gflag))
    {
        printf("\n\tCan not mix throughput mode and auto-mode flags.\n\n");
        exit(17);
    }
    if(fflag && trflag)
    {
        printf("\n\tYou must use -F when using multiple threads or processes.\n\n");
        exit(17);
    }
    if(aflag && mfflag)
    {
        printf("\n\tYou must use -f when using auto mode.\n\n");
        exit(17);
    }
    if(async_flag && mmapflag)
    {
        printf("\n\tSorry ... Only mmap or async but not both\n\n");
        exit(18);
    }
#ifndef ASYNC_IO
    if(async_flag)
    {
        printf("\n\tSorry ... This version does not support async I/O\n\n");
        exit(19);
    }
#endif
    if(no_write)
    {
       if(!include_tflag)
       {
         printf("You must specify which tests ( -i # ) when using -+E\n");
         exit(19);
       }
    }
    if(include_tflag)
    {
        for(i=0;i<sizeof(func)/sizeof(char *);i++)
            if(include_test[i])
                include_mask|=(long long)(1<<i);
        /* printf(">> %llx",include_mask);  HERE */
    }
    if(no_write) /* Disable if any writer would disturbe existing file */
    {
       if(include_test[0] || include_test[4] ||
          include_test[6] || include_test[8] || include_test[9] ||
              include_test[11])
       {
          printf("You must disable any test that writes when using -+E\n");
          exit(20);
       }
    }
    if(no_write) /* User must specify the existing file name */
    {
       if(!(fflag | mfflag))
       {
          printf("You must use -f or -F when using -+E\n");
          exit(20);
       }
    }
    if(h_flag && k_flag)
    {
        printf("\n\tCan not do both -H and -k\n");
        exit(20);
    }
    if((dedup | dedup_interior) && diag_v)
    {
        printf("\n\tCan not do both -+d and -+w\n");
        exit(20);
    }
        
    if(!aflag && !rflag)
        max_rec_size=min_rec_size;

    init_record_sizes(min_rec_size,max_rec_size);
        if(!silent) printf("\tTime Resolution = %1.6f seconds.\n",time_res);
#ifdef NO_PRINT_LLD
        if(!silent) printf("\tProcessor cache size set to %ld Kbytes.\n",cache_size/1024);
        if(!silent) printf("\tProcessor cache line size set to %ld bytes.\n",cache_line_size);
    if(!silent) printf("\tFile stride size set to %ld * record size.\n",stride);
#else
        if(!silent) printf("\tProcessor cache size set to %ld Kbytes.\n",cache_size/1024);
        if(!silent) printf("\tProcessor cache line size set to %ld bytes.\n",cache_line_size);
    if(!silent) printf("\tFile stride size set to %lld * record size.\n",stride);
#endif
    if(!rflag)
        reclen=(long long)4096;

    if(uflag && !lflag)
        num_child=mint = 1;
    if(lflag && !uflag)
        maxt = mint;
    if(use_thread)
        port="thread";
    else
        port="process";
    if(lflag || uflag){
#ifdef NO_PRINT_LLD
        if(!silent) printf("\tMin %s = %ld \n",port,mint);
        if(!silent) printf("\tMax %s = %ld \n",port,maxt);
#else
        if(!silent) printf("\tMin %s = %lld \n",port,mint);
        if(!silent) printf("\tMax %s = %lld \n",port,maxt);
#endif
    }
    if(trflag)
    {
        if(num_child > 1)
        {
            if(use_thread)
            {
                port="threads";
            }
            else
            {
                port="processes";
            }
        }

#ifdef NO_PRINT_LLD
        if(!silent) printf("\tThroughput test with %ld %s\n", num_child,port);
#else
        if(!silent) printf("\tThroughput test with %lld %s\n", num_child,port);
#endif
    }
        numrecs64 = (long long)(kilobytes64*1024)/reclen;
        if (reclen >  (long long)MAXBUFFERSIZE) {
#ifdef NO_PRINT_LLD
                printf("Error: Maximum record length is %ld bytes\n",
                                MAXBUFFERSIZE);
#else
                printf("Error: Maximum record length is %lld bytes\n",
                                (long long)MAXBUFFERSIZE);
#endif
                exit(21);
        }
        if (reclen < (long long)MINBUFFERSIZE) {
#ifdef NO_PRINT_LLD
                printf("Error: Minimum record length is %ld bytes\n",
                                MINBUFFERSIZE);
#else
                printf("Error: Minimum record length is %lld bytes\n",
                                (long long)MINBUFFERSIZE);
#endif
                exit(22);
        }
    /* Only bzero or fill that which you will use. The buffer is very large */
    if(verify ) 
    {
        fill_buffer((char *)buffer,l_min(reclen,(long long)cache_size),(long long)pattern,(char)sverify,(long long)0);
        if(pflag)
            fill_buffer((char *)pbuffer,l_min(reclen,(long long)cache_size),(long long)pattern,(char)sverify,(long long)0);
        if(mflag)
            fill_buffer((char *)mbuffer,l_min(reclen,(long long)cache_size),(long long)pattern,(char)sverify,(long long)0);
    }
    else
    {
        bzero(buffer,(size_t)l_min(reclen,(long long)cache_size));
    }

#ifndef NO_THREADS
#if defined( _HPUX_SOURCE ) || defined ( linux )
    if(ioz_processor_bind)
    {
         bind_cpu=begin_proc;
#if defined( _HPUX_SOURCE )
         pthread_processor_bind_np(PTHREAD_BIND_FORCED_NP,
                         (pthread_spu_t *)&anwser, (pthread_spu_t)bind_cpu, pthread_self());
#else
     cpu_set_t cpuset;
     CPU_ZERO(&cpuset);
     CPU_SET(bind_cpu, &cpuset);

     pthread_setaffinity_np(pthread_self(), sizeof(cpuset),&cpuset);
#endif
        my_nap(40); /* Switch to new cpu */
    }
#endif
#endif
    orig_size=kilobytes64;
    if(trflag){
        (void)multi_throughput_test(mint,maxt);
        goto out;
    }
    if(trflag && (mint == maxt)){
        auto_mode=0;
        throughput_test();
            goto out;
    }
        if (aflag) {
        print_header();
        auto_test();
        goto out;
    } 
    print_header();
    (void) begin(kilobytes64,reclen);
out:
    if(r_traj_flag)
        fclose(r_traj_fd);
    if(w_traj_flag)
        fclose(w_traj_fd);
    if (!no_unlink)
    {
        if(check_filename(dummyfile[0]))
               unlink(dummyfile[0]);    /* delete the file */
    }
    if(!silent) printf("\niozone test complete.\n");
    if(res_prob)
    {
        printf("Timer resolution is poor. Some small transfers may have \n");
        printf("reported inaccurate results. Sizes %ld Kbytes and below.\n",
            (long)(rec_prob/(long long)1024));
    }

    if(Rflag && !trflag){
        dump_excel();
    }
    return(0);  
}

#ifdef HAVE_ANSIC_C
void
record_command_line(int argc, char **argv)
#else
void
record_command_line(argc, argv)
int argc;
char **argv;
#endif
{
    int ix, len = 0;

    /* print and save the entire command line */
    if(!silent) printf("\tCommand line used:");
    for (ix=0; ix < argc; ix++) {
        if(!silent) printf(" %s", argv[ix]);
        if ((len + strlen(argv[ix])) < sizeof(command_line)) {
            strcat (command_line, argv[ix]);
            strcat (command_line, " ");
            len += strlen(argv[ix]) + 1;
        }
        else {
            printf ("Command line too long to save completely.\n");
            break;
        }
    }
    if(!silent) printf("\n");
}

/*************************************************************************/
/* BEGIN()                               */
/* This is the main work horse. It is called from main and from      */
/* auto_test. The caller provides the size of file and the record length.*/
/*************************************************************************/
#ifdef HAVE_ANSIC_C
void 
begin(off64_t kilos64,long long reclength)
#else
void
begin(kilos64,reclength)
off64_t kilos64;
long long reclength;
#endif
{
    long long num_tests,test_num,i,j;
    long long data1[MAXTESTS], data2[MAXTESTS];
    num_tests = sizeof(func)/sizeof(char *);
#if defined(HAVE_PREAD) 
    if(!Eflag)
    {
#if defined(HAVE_PREAD) && defined(HAVE_PREADV)
        num_tests -= 4;
#else
        num_tests -= 2;
#endif
        if(mmapflag || async_flag)
        {
            num_tests -= 2;
        }
    }
    else
    {
        if(mmapflag || async_flag)
#if defined(HAVE_PREAD) && defined(HAVE_PREADV)
            num_tests -= 6;
#else
            num_tests -= 4;
#endif
    }
#else
    if(mmapflag || async_flag)
    {
        num_tests -= 2;
    }
#endif

    if(RWONLYflag) num_tests = 2;       /* kcollins 8-21-96*/
    sync();         /* just in case there were some dirty */
    sync();
    kilobytes64=kilos64;
    reclen=reclength;
    numrecs64 = (kilobytes64*1024)/reclen;
    store_value(kilobytes64);
    if(r_traj_flag || w_traj_flag)
        store_value((off64_t)0);
    else    
        store_value((off64_t)(reclen/1024));

#ifdef NO_PRINT_LLD
    if(!silent) printf("%16ld",kilobytes64);
    if(r_traj_flag || w_traj_flag)
    {
        if(!silent) printf("%8ld",0);
    }
    else
    {
        if(!silent) printf("%8ld",reclen/1024);
    }
#else
    if(!silent) printf("%16lld",kilobytes64);
    if(r_traj_flag || w_traj_flag)
    {
        if(!silent) printf("%8lld",(long long )0);
    }
    else
    {
        if(!silent) printf("%8lld",reclen/1024);
    }
#endif
    if(include_tflag)
    {
        for(i=0;i<num_tests;i++)
        {
            if(include_mask & (long long)(1<<i))
               func[i](kilobytes64,reclen,&data1[i],&data2[i]);
            else
            {
                    if(!silent) printf("%s",test_output[i]); 
                fflush(stdout);
                for(j=0;j<test_soutput[i];j++)
                    store_value((off64_t)0);
            }
        }
    }
    else
    {
        for(test_num=0;test_num < num_tests;test_num++)
        {
            func[test_num](kilobytes64,reclen,&data1[test_num],&data2[test_num]);
        };
    }
    if(!silent) printf("\n");
    if(!OPS_flag && !include_tflag){            /* Report in ops/sec ? */
       if(data1[1]!=0 && data2[1] != 0)
       {   
        totaltime = data1[1] + data1[0];
        if (totaltime < TOOFAST) 
        {
            goodkilos = (TOOFAST/totaltime)*2*kilobytes64;
                printf("\nThe test completed too quickly to give a good result\n");
                printf("You will get a more precise measure of this machine's\n");
                printf("performance by re-running iozone using the command:\n");
#ifdef NO_PRINT_LLD
                printf("\n\tiozone %ld ", goodkilos);
                printf("\t(i.e., file size = %ld kilobytes64)\n", goodkilos);
#else
                printf("\n\tiozone %lld ", goodkilos);
                printf("\t(i.e., file size = %lld kilobytes64)\n", goodkilos);
#endif
        }
       } else {
            goodrecl = reclen/2;
            printf("\nI/O error during read.  Try again with the command:\n");
#ifdef NO_PRINT_LLD
            printf("\n\tiozone %ld %ld ", kilobytes64,  goodrecl);
            printf("\t(i.e. record size = %ld bytes)\n",  goodrecl);
#else
            printf("\n\tiozone %lld %lld ", kilobytes64,  goodrecl);
            printf("\t(i.e. record size = %lld bytes)\n",  goodrecl);
#endif
       }
    }
    if (!no_unlink)
    {
        if(check_filename(filename))
              unlink(filename); /* delete the file */
    }
                    /*stop timer*/
    return ;
}
/******************************************************************

    SHOW_HELP -- show development help of this program

******************************************************************/
#ifdef HAVE_ANSIC_C
void show_help(void)
#else
void show_help()
#endif
{
        long long i;
        if(!silent) printf("iozone: help mode\n\n");
        for(i=0; strlen(help[i]); i++)
        {
        if(!silent) printf("%s\n", help[i]);
        }
}
/******************************************************************

    SIGNAL_HANDLER -- clean up if user interrupts the program

******************************************************************/
#ifdef HAVE_ANSIC_C
void signal_handler(void)
#else
void signal_handler()
#endif
{
    long long i;
    if(distributed)
    {
        if(master_iozone)
            cleanup_children();
    }
    if((long long)getpid()==myid)
    {
            if(!silent) printf("\niozone: interrupted\n\n");
#ifndef VMS
        if (!no_unlink)
        {
           if(check_filename(filename))
                unlink(filename);   /* delete the file */
        }
        for(i=1;i<num_child;i++)
        {
           if(check_filename(dummyfile[i]))
                unlink(dummyfile[i]);   /* delete the file */
        }
        if (!no_unlink)
        {
           if(check_filename(dummyfile[0]))
                unlink(dummyfile[0]);   /* delete the file */
        }
        
#endif
        if(Rflag && !trflag){
            dump_excel();
        }
        if(Rflag && trflag){
            dump_throughput();
        }

            if(!silent) printf("exiting iozone\n\n");
        if(res_prob)
        {
            printf("Timer resolution is poor. Some small transfers may have \n");
            printf("reported inaccurate results. Sizes %ld Kbytes and below.\n",
                (long)rec_prob/1024);
        }
        if(trflag && !use_thread)
           for(i=0;i<num_child;i++)
            kill((pid_t)childids[i],SIGTERM);
        if(r_traj_flag)
            fclose(r_traj_fd);
        if(w_traj_flag)
            fclose(w_traj_fd);
    }
    if(sp_msfd)
        close(sp_msfd);
    if(sp_mrfd)
        close(sp_mrfd);
        exit(0);
}

/****************************************************************/
/*                              */
/*   AUTO_TEST -- perform series of tests and tabulate results  */
/*                              */
/****************************************************************/
#ifdef HAVE_ANSIC_C
void 
auto_test(void)
#else
void auto_test()
#endif
{
        off64_t kilosi;
    long long recszi,count1;
    long long mult;
    long long xx;

    /****************************************************************/
    /* Start with file size of 1 megabyte and repeat the test   */
    /* KILOBYTES_ITER_LIMIT                     */
    /* times.  Each time we run, the file size is doubled       */
    /****************************************************************/
/*
        if(sflag) {
          min_file_size = kilobytes64;
          max_file_size = kilobytes64;
        }
        if(rflag) {
          min_rec_size = reclen;
          max_rec_size = reclen;
        }
*/
    if(gflag)
        max_file_size = maximum_file_size;
    if(nflag)
        min_file_size = minimum_file_size;

        if (min_rec_size > (long long)(min_file_size*1024)) {
#ifdef NO_PRINT_LLD
            printf("Error: record length %ld is greater than filesize %ld KB\n ",
                                min_rec_size,min_file_size);
#else
            printf("Error: record length %lld is greater than filesize %lld KB\n ",
                                min_rec_size,min_file_size);
#endif
                exit(23);
        }

    if(NOCROSSflag) xover = max_file_size;

    init_file_sizes(min_file_size, max_file_size);
    del_record_sizes();
    orig_min_rec_size=min_rec_size;
    orig_max_rec_size=max_rec_size;
    init_record_sizes(min_rec_size, max_rec_size);

        for(kilosi=get_next_file_size((off64_t)0); kilosi>0; kilosi=get_next_file_size(kilosi))
        {
    /****************************************************************/
    /* Start with record size of min_rec_size bytes and repeat the  */
    /* test, multiplying the record size by MULTIPLIER each time,   */
    /* until we reach max_rec_size.  At the CROSSOVER we stop doing */
    /* small buffers as it takes forever and becomes very       */
    /* un-interesting.                      */
    /****************************************************************/
             if(!rflag && !sflag && !yflag && !qflag)
                if(kilosi > xover){
                    min_rec_size = LARGE_REC;
            mult = orig_min_rec_size/1024;
            del_record_sizes();
            init_record_sizes(min_rec_size, max_rec_size);
                /************************************/
            /* Generate dummy entries in the    */
            /* Excel buffer for skipped         */
            /* record sizes          */
            /************************************/
            for(count1=min_rec_size;
                    (count1 != orig_min_rec_size) && (
                        mult <= (kilosi*1024)) ;
                        count1=(count1>>1))
            {
                    current_x=0;
                    store_value((off64_t)kilosi);
                    store_value((off64_t)mult);
                    for(xx=0;xx<20;xx++)
                        store_value((off64_t)0);
                    mult=mult*2;
                    current_y++;
                    if(current_y>max_y)
                        max_y=current_y;
                    current_x=0;
            }
         }

             for (recszi=get_next_record_size((off64_t)0);recszi!=0;recszi=get_next_record_size(recszi))
             {
                     if(recszi > (kilosi*1024)) 
            break;
                     begin(kilosi, recszi );
             current_x=0;
             current_y++;
             }
    }
}


/****************************************************************/
/*                              */
/* THROUGHPUT_TEST () Multi process throughput test     */
/*                              */
/* Note: States for share memory barrier are:           */
/* 0 = Child not running or has finished.           */
/* 1 = Child is ready to begin.                 */
/* 2 = Child is told to begin.                  */
/****************************************************************/
/* Data in shared memory format is:             */
/*                              */
/* struct child_stats {                     */
/*  long long flag;     Used to barrier         */
/*  double walltime;    Child's elapsed time        */
/*  double cputime;     Child's CPU time        */
/*  double throughput;  Child's throughput      */
/*  double actual;      Child's actual read/written */
/* }                                */
/*                              */
/* There is an array of child_stat structures layed out in  */
/* shared memory.                       */
/*                              */
/****************************************************************/

#ifdef HAVE_ANSIC_C
void 
throughput_test(void)
#else
void 
throughput_test()
#endif
{
    char *unit;
    double starttime1 = 0;
    double jstarttime = 0;
    double jtime = 0;
    double walltime = 0;
    double cputime = 0;
    char *port;
    char getout;
    long long throughsize = KILOBYTES;
    long long xx,xy,i;
    long long xyz;
    double ptotal;
    off64_t written_so_far, read_so_far, re_written_so_far,re_read_so_far;
    VOLATILE char *temp;
    double min_throughput = 0;
    double max_throughput = 0;
    double avg_throughput = 0;
    double min_xfer = 0; 


    toutputindex=0;
    strcpy(&toutput[0][0],throughput_tests[0]);
    ptotal=written_so_far=read_so_far=re_written_so_far=re_read_so_far=0 ;

    if(OPS_flag)
        unit="ops";
    else
        unit="KB";

    if(!haveshm)
    {
        shmaddr=(struct child_stats *)alloc_mem((long long)SHMSIZE,(int)1);
#ifdef _64BIT_ARCH_
        if((long long)shmaddr==(long long)-1)
#else
        if((long )shmaddr==(long)-1)
#endif
        {
            printf("\nShared memory not working\n");
            exit(24);
        }
        haveshm=(char*)shmaddr;
    }
    else
        shmaddr=(struct child_stats *)haveshm;

    if(use_thread)
        stop_flag = &stoptime;
    else
    {
        temp = (char *)&shmaddr[0];
        stop_flag = (char *)&temp[(long long)SHMSIZE]-4;
    }
    for(xyz=0;xyz<num_child;xyz++){ /* all children to state 0 (HOLD) */
        child_stat = (struct child_stats *)&shmaddr[xyz];
        child_stat->flag=CHILD_STATE_HOLD;
        child_stat->actual=0;
        child_stat->throughput=0;
        child_stat->cputime=0;
        child_stat->walltime=0;
    }
    *stop_flag = 0;
    if(!sflag)
        kilobytes64=throughsize;
    if(!rflag)
            reclen=(long long)4096;
    if(aggflag)
        kilobytes64=orig_size/num_child;
        numrecs64 = (long long)(kilobytes64*1024)/reclen;
    buffer=mainbuffer;
    if(use_thread)
        port="thread";
    else
        port="process";
    if(w_traj_flag)
    {
#ifdef NO_PRINT_LLD
    if(!silent) printf("\tEach %s writes a %ld Kbyte file in telemetry controlled records\n",
        port,kilobytes64);
#else
    if(!silent) printf("\tEach %s writes a %lld Kbyte file in telemetry controlled records\n",
        port,kilobytes64);
#endif
    }
    else
    {
#ifdef NO_PRINT_LLD
    if(!silent) printf("\tEach %s writes a %ld Kbyte file in %ld Kbyte records\n",
        port,kilobytes64,reclen/1024);
#else
    if(!silent) printf("\tEach %s writes a %lld Kbyte file in %lld Kbyte records\n",
        port,kilobytes64,reclen/1024);
#endif
    }

    if(fflag)       /* Each child has a file name to write */
      for(xx=0;xx<num_child;xx++)
        filearray[xx] = filename;
    myid = (long long)getpid();

    /* rags: skip writer test */
    if(include_tflag)
        if(!(include_mask & (long long)WRITER_MASK))
        {
            store_dvalue( (double)0);
            store_dvalue( (double)0);
            toutputindex++;
            goto next0;
        }

    if((!distributed) || (distributed && master_iozone))
        start_monitor("Write");
    /* Hooks to start the distributed Iozone client/server code */
    if(distributed)
    {
        use_thread=0;  /* Turn of any Posix threads */
        if(master_iozone)
            master_listen_socket = start_master_listen();
        else
            become_client();
    }
    if(!use_thread)
    {
       for(xx = 0; xx< num_child ; xx++){   /* Create the children */
        chid=xx;
        childids[xx] = start_child_proc(THREAD_WRITE_TEST,numrecs64,reclen);
        if(childids[xx]==-1){
            printf("\nFork failed\n");
            for(xy = 0; xy< xx ; xy++){
                if(!use_thread)
                    kill((pid_t)childids[xy],SIGTERM);
            }
            exit(25);
        }
        if(childids[xx]!=0 && debug1)
#ifdef NO_PRINT_LLD
            printf("Parent starting slot %ld\n",xx);    
#else
            printf("Parent starting slot %lld\n",xx);   
#endif
        if( childids[xx] == 0 ){
#ifdef _64BIT_ARCH_
          thread_write_test((void *)xx);
#else
          thread_write_test((void *)(long)xx);
#endif
        }else {
#ifdef NO_PRINT_LLD
          sprintf(dummyfile[xx],"%s.DUMMY.%ld",filearray[xx], xx);
#else
          sprintf(dummyfile[xx],"%s.DUMMY.%lld",filearray[xx], xx);
#endif
        }
       }
    }
#ifndef NO_THREADS
    else
    {
       for(xx = 0; xx< num_child ; xx++){   /* Create the children */

#ifdef NO_PRINT_LLD
        sprintf(dummyfile[xx],"%s.DUMMY.%ld",filearray[xx], xx);
#else
        sprintf(dummyfile[xx],"%s.DUMMY.%lld",filearray[xx], xx);
#endif
        if(!barray[xx])
        {
            barray[xx]=(char *) alloc_mem((long long)(MAXBUFFERSIZE+cache_size),(int)0);
            if(barray[xx] == 0) {
                   perror("Memory allocation failed:");
                   exit(26);
                }
                barray[xx] =(char *)(((long)barray[xx] + cache_size ) & 
            ~(cache_size-1));
        }

#ifdef _64BIT_ARCH_
        childids[xx] = mythread_create(thread_write_test,(void*)xx);
#else
        childids[xx] = mythread_create(thread_write_test,(void*)(long)xx);
#endif
        if(childids[xx]==-1){
            printf("Thread create failed\n");
            for(xy = 0; xy< xx ; xy++){
                kill((pid_t)myid,SIGTERM);
            }
            exit(27);
        }
       }
    }
#endif
    if((long long)getpid() == myid)
    {
        prepage(buffer,reclen);     /* Force copy on write */
                /* wait for children to start */
        if(distributed && master_iozone)
        {
            start_master_listen_loop((int) num_child);
        }
        for(i=0;i<num_child; i++){
            child_stat = (struct child_stats *)&shmaddr[i]; 
            while(child_stat->flag==CHILD_STATE_HOLD)
                Poll((long long)1);
        }
        for(i=0;i<num_child; i++)   /* Start all children going */
        {
            if(delay_start!=0)
                Poll((long long)delay_start);
                        /* State "go" */
            child_stat = (struct child_stats *)&shmaddr[i]; 
            child_stat->flag=CHILD_STATE_BEGIN;
            if(distributed && master_iozone)
                tell_children_begin(i);
        }
        starttime1 = time_so_far(); /* Start parents timer */
        goto waitout;
    }

waitout:
    getout=0;
    if((long long)getpid() == myid) {   /* Parent only */
        starttime1 = time_so_far(); /* Wait for all children */
        for( i = 0; i < num_child; i++){
            child_stat = (struct child_stats *) &shmaddr[i];
            if(distributed && master_iozone)
            {
                printf("\n\tTest running:");
                wait_dist_join();
                break;
            }
            else
            {
             if(use_thread)
             {
                thread_join(childids[i],(void *)&pstatus);
             }
             else
             {
                wait(0);
             }
            }
            if(!jstarttime)
                jstarttime = time_so_far(); 
        }
        jtime = (time_so_far()-jstarttime)-time_res;
        if(jtime < (double).000001) 
        {
            jtime=time_res; 
        }
    }
    total_time = (time_so_far() - starttime1)-time_res; /* get parents total time */
    if(total_time < (double).000001) 
    {
        total_time=time_res; 
        if(rec_prob < reclen)
            rec_prob = reclen;
        res_prob=1;
    }
#ifdef JTIME
    total_time=total_time-jtime;/* Remove the join time */
    if(!silent) printf("\nJoin time %10.2f\n",jtime);
#endif

    total_kilos=0;
    ptotal=0;
    walltime = 0.0;
    cputime = 0.0;
    if(!silent) printf("\n");
    for(xyz=0;xyz<num_child;xyz++){
        child_stat = (struct child_stats *) &shmaddr[xyz];
        total_kilos += child_stat->throughput; /* add up the children */
        ptotal += child_stat->actual;
        if(!min_xfer)
            min_xfer=child_stat->actual;
        if(child_stat->actual < min_xfer)
            min_xfer=child_stat->actual;
        if(!min_throughput)
            min_throughput=child_stat->throughput;
        if(child_stat->throughput < min_throughput)
            min_throughput=child_stat->throughput;
        if(child_stat->throughput > max_throughput)
            max_throughput=child_stat->throughput;
        /* Add up the cpu times of all children */
        cputime += child_stat->cputime;

        /* and find the child with the longest wall time */
        /* Get the earliest start time and latest fini time to calc. elapsed time. */
        if (child_stat->walltime < child_stat->cputime)
            child_stat->walltime = child_stat->cputime;
        if (child_stat->walltime > walltime)
            walltime = child_stat->walltime;
    }
    avg_throughput=total_kilos/num_child;
    if(cpuutilflag)
    {
        if (cputime < cputime_res)
            cputime = 0.0;
    }
    
    for(xyz=0;xyz<num_child;xyz++){
        child_stat = (struct child_stats *) &shmaddr[xyz];
        child_stat->flag = CHILD_STATE_HOLD; /* Start children at state 0 (HOLD) */
    }
    if(cpuutilflag)
        store_times (walltime, cputime);    /* Must be Before store_dvalue(). */
    store_dvalue(total_kilos);
#ifdef NO_PRINT_LLD
    if(!silent) printf("\tChildren see throughput for %2ld initial writers \t= %10.2f %s/sec\n", num_child, total_kilos,unit);
    if(!silent && !distributed) printf("\tParent sees throughput for %2ld initial writers \t= %10.2f %s/sec\n",num_child,((double)(ptotal)/total_time),unit);
#else
    if(!silent) printf("\tChildren see throughput for %2lld initial writers \t= %10.2f %s/sec\n", num_child, total_kilos,unit);
    if(!silent && !distributed) printf("\tParent sees throughput for %2lld initial writers \t= %10.2f %s/sec\n",num_child,((double)(ptotal)/total_time),unit);
#endif
    if(!silent) printf("\tMin throughput per %s \t\t\t= %10.2f %s/sec \n", port,min_throughput,unit);
    if(!silent) printf("\tMax throughput per %s \t\t\t= %10.2f %s/sec\n", port,max_throughput,unit);
    if(!silent) printf("\tAvg throughput per %s \t\t\t= %10.2f %s/sec\n", port,avg_throughput,unit);
    if(!silent) printf("\tMin xfer \t\t\t\t\t= %10.2f %s\n", min_xfer,unit);
    /* CPU% can be > 100.0 for multiple CPUs */
    if(cpuutilflag)
    {
        if(walltime == 0.0)
        {
            if(!silent) printf("\tCPU utilization: Wall time %8.3f    CPU time %8.3f    CPU utilization %6.2f %%\n\n",
                walltime, cputime, 0.0);
        }
        else
        {
            if(!silent) printf("\tCPU Utilization: Wall time %8.3f    CPU time %8.3f    CPU utilization %6.2f %%\n\n",
                walltime, cputime, 100.0 * cputime / walltime);
        }
    }
    if(Cflag)
    {
        for(xyz=0;xyz<num_child;xyz++)
        {
            child_stat = (struct child_stats *) &shmaddr[xyz];
            if(cpuutilflag)
            {
                if(!silent) 
                    printf("\tChild[%ld] xfer count = %10.2f %s, Throughput = %10.2f %s/sec, wall=%6.3f, cpu=%6.3f, %%=%6.2f\n",
                    (long)xyz, child_stat->actual, unit, child_stat->throughput, unit, child_stat->walltime, 
                    child_stat->cputime, cpu_util(child_stat->cputime, child_stat->walltime));
            }
            else
            {
                if(!silent) printf("\tChild[%ld] xfer count = %10.2f %s, Throughput = %10.2f %s/sec\n",
                    (long)xyz, child_stat->actual, unit, child_stat->throughput, unit);
            }
        }
    }
    if((!distributed) || (distributed && master_iozone))
        stop_monitor("Write");
    /**********************************************************/
    /*************** End of intitial writer *******************/
    /**********************************************************/
    sync();
    sleep(2);
    if(restf)
        sleep((int)rest_val);
    *stop_flag=0;
    if(distributed && master_iozone)
    {
        stop_master_listen(master_listen_socket);
        cleanup_comm();
    }

    /**********************************************************/
    /* Re-write throughput performance test. ******************/
    /**********************************************************/
    walltime = 0.0;
    cputime = 0.0;
    jstarttime=0;
    total_kilos=0;
    toutputindex=1;
    strcpy(&toutput[1][0],throughput_tests[1]);
    if(noretest)
    {
        store_dvalue( (double)0);
        goto next0;
    }
    if((!distributed) || (distributed && master_iozone))
        start_monitor("Rewrite");
    /* Hooks to start the distributed Iozone client/server code */
    if(distributed)
    {
        use_thread=0;  /* Turn of any Posix threads */
        if(master_iozone)
            master_listen_socket = start_master_listen();
        else
            become_client();
    }
    if(!use_thread)
    {
       for(xx = 0; xx< num_child ; xx++){
        chid=xx;
        childids[xx] = start_child_proc(THREAD_REWRITE_TEST,numrecs64,reclen);
        if(childids[xx]==-1){
            printf("\nFork failed\n");
            for(xy = 0; xy< xx ; xy++){
                Kill((long long)childids[xy],(long long)SIGTERM);
            }
            exit(28);
        }
        if(childids[xx] == 0){
#ifdef _64BIT_ARCH_
            thread_rwrite_test((void *)xx);
#else
            thread_rwrite_test((void *)((long)xx));
#endif
        }   
       }
    }
#ifndef NO_THREADS
    else
    {
       for(xx = 0; xx< num_child ; xx++){   /* Create the children */
        if(!barray[xx])
        {
            barray[xx]=(char *) alloc_mem((long long)(MAXBUFFERSIZE+cache_size),(int)0);
            if(barray[xx] == 0) {
                   perror("Memory allocation failed:");
                   exit(26);
                }
                barray[xx] =(char *)(((long)barray[xx] + cache_size ) & 
            ~(cache_size-1));
        }
#ifdef _64BIT_ARCH_
        childids[xx] = mythread_create( thread_rwrite_test,xx);
#else
        childids[xx] = mythread_create( thread_rwrite_test,(void *)(long)xx);
#endif
        if(childids[xx]==-1){
            printf("\nThread create failed\n");
            for(xy = 0; xy< xx ; xy++){
                Kill((long long)myid,(long long)SIGTERM);
            }
            exit(29);
        }
       }
    }
#endif
    if((long long)myid == getpid())
    {
        if(distributed && master_iozone)
        {
            start_master_listen_loop((int) num_child);
        }
        for(i=0;i<num_child; i++){
            child_stat = (struct child_stats *)&shmaddr[i];
                    /* wait for children to start */
            while(child_stat->flag==CHILD_STATE_HOLD) 
                Poll((long long)1);
        }
        for(i=0;i<num_child; i++)
        {
            child_stat = (struct child_stats *)&shmaddr[i];
            child_stat->flag = CHILD_STATE_BEGIN;   /* tell children to go */
            if(delay_start!=0)
                Poll((long long)delay_start);
            if(distributed && master_iozone)
                tell_children_begin(i);
        }
        starttime1 = time_so_far();
        goto jump3;
    }
    
jump3:
    getout=0;
    if((long long)myid == getpid()){    /* Parent only here */
        for( i = 0; i < num_child; i++){
            child_stat=(struct child_stats *)&shmaddr[i];
            if(distributed && master_iozone)
            {
                printf("\n\tTest running:");
                wait_dist_join();
                break;
            }
            else
            {
               if(use_thread)
               {
                thread_join(childids[i],(void *)&pstatus);
               }
               else
               {
                wait(0);
               }
            }
            if(!jstarttime)
                jstarttime = time_so_far(); 
        }
        jtime = (time_so_far()-jstarttime)-time_res;
        if(jtime < (double).000001) 
        {
            jtime=time_res;
        }
    }
    total_time = (time_so_far() - starttime1)-time_res; /* Parents total time */
    if(total_time  < (double).000001) 
    {
        total_time=time_res;
        if(rec_prob < reclen)
            rec_prob = reclen;
        res_prob=1;
    }
#ifdef JTIME
    total_time=total_time-jtime;/* Remove the join time */
    if(!silent) printf("\nJoin time %10.2f\n",jtime);
#endif
    

    total_kilos=0;
    ptotal=0;

    min_throughput=max_throughput=min_xfer=0;
    if(!silent) printf("\n");
    for(xyz=0;xyz<num_child;xyz++){
        child_stat=(struct child_stats *)&shmaddr[xyz];
        total_kilos+=child_stat->throughput;
        ptotal+=child_stat->actual;
        if(!min_xfer)
            min_xfer=child_stat->actual;
        if(child_stat->actual < min_xfer)
            min_xfer=child_stat->actual;
        if(!min_throughput)
            min_throughput=child_stat->throughput;
        if(child_stat->throughput < min_throughput)
            min_throughput=child_stat->throughput;
        if(child_stat->throughput > max_throughput)
            max_throughput=child_stat->throughput;
        cputime += child_stat->cputime;
        /* Get the earliest start time and latest fini time to calc. elapsed time. */
        if (child_stat->walltime < child_stat->cputime)
            child_stat->walltime = child_stat->cputime;
        if (child_stat->walltime > walltime)
            walltime = child_stat->walltime;
    }
    avg_throughput=total_kilos/num_child;
    if(cpuutilflag)
    {
/*
        if (walltime < cputime_res)
            walltime = 0.0;
*/
        if (cputime < cputime_res)
            cputime = 0.0;
    }

    for(xyz=0;xyz<num_child;xyz++){ /* Reset state to 0 (HOLD) */
        child_stat=(struct child_stats *)&shmaddr[xyz];
        child_stat->flag = CHILD_STATE_HOLD;
    }
    if(cpuutilflag)
        store_times (walltime, cputime);    /* Must be Before store_dvalue(). */
    store_dvalue(total_kilos);
#ifdef NO_PRINT_LLD
    if(!silent) printf("\tChildren see throughput for %2ld rewriters \t= %10.2f %s/sec\n", num_child, total_kilos,unit);
    if(!silent && !distributed) printf("\tParent sees throughput for %2ld rewriters \t= %10.2f %s/sec\n", num_child, (double)(ptotal)/total_time,unit);
#else
    if(!silent) printf("\tChildren see throughput for %2lld rewriters \t= %10.2f %s/sec\n", num_child, total_kilos,unit);
    if(!silent && !distributed) printf("\tParent sees throughput for %2lld rewriters \t= %10.2f %s/sec\n", num_child, (double)(ptotal)/total_time,unit);
#endif
    if(!silent) printf("\tMin throughput per %s \t\t\t= %10.2f %s/sec \n", port,min_throughput,unit);
    if(!silent) printf("\tMax throughput per %s \t\t\t= %10.2f %s/sec\n", port,max_throughput,unit);
    if(!silent) printf("\tAvg throughput per %s \t\t\t= %10.2f %s/sec\n", port,avg_throughput,unit);
    if(!silent) printf("\tMin xfer \t\t\t\t\t= %10.2f %s\n", min_xfer,unit);
    /* CPU% can be > 100.0 for multiple CPUs */
    if(cpuutilflag)
    {
        if(walltime == 0.0)
        {
            if(!silent) printf("\tCPU utilization: Wall time %8.3f    CPU time %8.3f    CPU utilization %6.2f %%\n\n",
                walltime, cputime, 0.0);
        }
        else
        {
            if(!silent) printf("\tCPU utilization: Wall time %8.3f    CPU time %8.3f    CPU utilization %6.2f %%\n\n",
                walltime, cputime, 100.0 * cputime / walltime);
        }
    }
    if(Cflag)
    {
        for(xyz=0;xyz<num_child;xyz++)
        {
            child_stat = (struct child_stats *) &shmaddr[xyz];
            if(cpuutilflag)
            {
                if(!silent) printf("\tChild[%ld] xfer count = %10.2f %s, Throughput = %10.2f %s/sec, wall=%6.3f, cpu=%6.3f, %%=%6.2f\n",
                    (long)xyz, child_stat->actual, unit, child_stat->throughput, unit, child_stat->walltime, 
                    child_stat->cputime, cpu_util(child_stat->cputime, child_stat->walltime));
            }
            else
            {
                if(!silent) printf("\tChild[%ld] xfer count = %10.2f %s, Throughput = %10.2f %s/sec\n",
                    (long)xyz, child_stat->actual, unit, child_stat->throughput, unit);
            }
        }
    }
    *stop_flag=0;
    if((!distributed) || (distributed && master_iozone))
        stop_monitor("Rewrite");
    /**********************************************************/
    /*************** End of rewrite throughput ****************/
    /**********************************************************/
    sync();
    sleep(2);
    if(restf)
        sleep((int)rest_val);
    if(distributed && master_iozone)
    {
        stop_master_listen(master_listen_socket);
        cleanup_comm();
    }
next0:
    if(include_tflag)
        if(!(include_mask & (long long)READER_MASK))
            goto next1;
    /**************************************************************/
    /*** Reader throughput tests **********************************/
    /**************************************************************/
    if((!distributed) || (distributed && master_iozone))
        start_monitor("Read");
    toutputindex++;
    strcpy(&toutput[toutputindex][0],throughput_tests[2]);
    walltime = 0.0;
    cputime = 0.0;
    jstarttime=0;
    total_kilos=0;
    if(distributed)
    {
        use_thread=0;
        if(master_iozone)
            master_listen_socket=start_master_listen();
        else
            become_client();
    }
    if(!use_thread)
    {
       for(xx = 0; xx< num_child ; xx++){
        chid=xx;
        childids[xx] = start_child_proc(THREAD_READ_TEST,numrecs64,reclen);
        if(childids[xx]==-1){
            printf("\nFork failed\n");
            for(xy = 0; xy< xx ; xy++){
                Kill((long long)childids[xy],(long long)SIGTERM);
            }
            exit(30);
        }
        if(childids[xx]==0){
#ifdef _64BIT_ARCH_
            thread_read_test((void *)xx);
#else
            thread_read_test((void *)((long)xx));
#endif
        }   
       }
    }
#ifndef NO_THREADS
    else
    {
       for(xx = 0; xx< num_child ; xx++){   /* Create the children */
        if(!barray[xx])
        {
            barray[xx]=(char *) alloc_mem((long long)(MAXBUFFERSIZE+cache_size),(int)0);
            if(barray[xx] == 0) {
                   perror("Memory allocation failed:");
                   exit(26);
                }
                barray[xx] =(char *)(((long)barray[xx] + cache_size ) & 
            ~(cache_size-1));
        }
#ifdef _64BIT_ARCH_
        childids[xx] = mythread_create( thread_read_test,xx);
#else
        childids[xx] = mythread_create( thread_read_test,(void *)(long)xx);
#endif
        if(childids[xx]==-1){
            printf("\nThread create failed\n");
            for(xy = 0; xy< xx ; xy++){
                kill((pid_t)myid,(int)SIGTERM);
            }
            exit(31);
        }
       }
    }
#endif
    if(myid == (long long)getpid()){
        if(distributed && master_iozone)
        {
            start_master_listen_loop((int) num_child);
        }
        for(i=0;i<num_child; i++){ /* wait for children to start */
            child_stat=(struct child_stats *)&shmaddr[i];
            while(child_stat->flag==CHILD_STATE_HOLD)
                Poll((long long)1);
        }
        for(i=0;i<num_child; i++)
        {
            child_stat=(struct child_stats *)&shmaddr[i];
            child_stat->flag = CHILD_STATE_BEGIN; /* tell children to go */
            if(delay_start!=0)
                Poll((long long)delay_start);
            if(distributed && master_iozone)
                tell_children_begin(i);
        }
        starttime1 = time_so_far();
        goto jumpend4;
    }
jumpend4:
    getout=0;
    if(myid == (long long)getpid()){    /* Parent here */
        for( i = 0; i < num_child; i++){
            child_stat = (struct child_stats *)&shmaddr[i];
            if(distributed && master_iozone)
            {
                printf("\n\tTest running:");
                wait_dist_join();
                break;
            }
            else
            {
               if(use_thread)
               {
                thread_join(childids[i],(void *)&pstatus);
               }
               else
               {
                wait(0);
               }
            }
            if(!jstarttime)
                jstarttime = time_so_far(); 
        }
        jtime = (time_so_far()-jstarttime)-time_res; 
        if(jtime < (double).000001)
        {
            jtime=time_res; 
        }
    }
    total_time = (time_so_far() - starttime1)-time_res; /* Parents time */
    if(total_time < (double).000001) 
    {
        total_time=time_res; 
        if(rec_prob < reclen)
            rec_prob = reclen;
        res_prob=1;
    }
#ifdef JTIME
    total_time=total_time-jtime;/* Remove the join time */
    if(!silent) printf("\nJoin time %10.2f\n",jtime);
#endif
    
    total_kilos=0;
    ptotal=0;
    min_throughput=max_throughput=min_xfer=0;
    if(!silent) printf("\n");
    for(xyz=0;xyz<num_child;xyz++){
        child_stat=(struct child_stats *)&shmaddr[xyz];
        total_kilos+=child_stat->throughput;
        ptotal+=child_stat->actual;
        if(!min_xfer)
            min_xfer=child_stat->actual;
        if(child_stat->actual < min_xfer)
            min_xfer=child_stat->actual;
        if(!min_throughput)
            min_throughput=child_stat->throughput;
        if(child_stat->throughput < min_throughput)
            min_throughput=child_stat->throughput;
        if(child_stat->throughput > max_throughput)
            max_throughput=child_stat->throughput;
        cputime += child_stat->cputime;
        /* Get the earliest start time and latest fini time to calc. elapsed time. */
        if (child_stat->walltime < child_stat->cputime)
            child_stat->walltime = child_stat->cputime;
        if (child_stat->walltime > walltime)
            walltime = child_stat->walltime;
    }
    avg_throughput=total_kilos/num_child;
    if(cpuutilflag)
    {
        if (cputime < cputime_res)
            cputime = 0.0;
    }
    if(cpuutilflag)
        store_times (walltime, cputime);    /* Must be Before store_dvalue(). */
    store_dvalue(total_kilos);
#ifdef NO_PRINT_LLD
    if(!silent) printf("\tChildren see throughput for %2ld readers \t\t= %10.2f %s/sec\n", num_child, total_kilos,unit);
    if(!silent && !distributed) printf("\tParent sees throughput for %2ld readers \t\t= %10.2f %s/sec\n", num_child, (double)(ptotal)/total_time,unit);
#else
    if(!silent) printf("\tChildren see throughput for %2lld readers \t\t= %10.2f %s/sec\n", num_child, total_kilos,unit);
    if(!silent && !distributed) printf("\tParent sees throughput for %2lld readers \t\t= %10.2f %s/sec\n", num_child, (double)(ptotal)/total_time,unit);
#endif
    if(!silent) printf("\tMin throughput per %s \t\t\t= %10.2f %s/sec \n", port,min_throughput,unit);
    if(!silent) printf("\tMax throughput per %s \t\t\t= %10.2f %s/sec\n", port,max_throughput,unit);
    if(!silent) printf("\tAvg throughput per %s \t\t\t= %10.2f %s/sec\n", port,avg_throughput,unit);
    if(!silent) printf("\tMin xfer \t\t\t\t\t= %10.2f %s\n", min_xfer,unit);
    /* CPU% can be > 100.0 for multiple CPUs */
    if(cpuutilflag)
    {
        if(walltime == 0.0)
        {
            if(!silent) printf("\tCPU utilization: Wall time %8.3f    CPU time %8.3f    CPU utilization %6.2f %%\n\n",
                walltime, cputime, 0.0);
        }
        else
        {
            if(!silent) printf("\tCPU utilization: Wall time %8.3f    CPU time %8.3f    CPU utilization %6.2f %%\n\n",
                walltime, cputime, 100.0 * cputime / walltime);
        }
    }
    if(Cflag)
    {
        for(xyz=0;xyz<num_child;xyz++)
        {
            child_stat = (struct child_stats *) &shmaddr[xyz];
            if(cpuutilflag)
            {
                if(!silent) printf("\tChild[%ld] xfer count = %10.2f %s, Throughput = %10.2f %s/sec, wall=%6.3f, cpu=%6.3f, %%=%6.2f\n",
                    (long)xyz, child_stat->actual, unit, child_stat->throughput, unit, child_stat->walltime, 
                    child_stat->cputime, cpu_util(child_stat->cputime, child_stat->walltime));
            }
            else
            {
                if(!silent) printf("\tChild[%ld] xfer count = %10.2f %s, Throughput = %10.2f %s/sec\n",
                    (long)xyz, child_stat->actual, unit, child_stat->throughput, unit);
            }
        }
    }
    if((!distributed) || (distributed && master_iozone))
        stop_monitor("Read");
    /**********************************************************/
    /*************** End of readers throughput ****************/
    /**********************************************************/
    sync();
    sleep(2);
    if(restf)
        sleep((int)rest_val);
    if(distributed && master_iozone)
    {
        stop_master_listen(master_listen_socket);
        cleanup_comm();
    }

    /**************************************************************/
    /*** ReReader throughput tests **********************************/
    /**************************************************************/
    toutputindex++;
    strcpy(&toutput[toutputindex][0],throughput_tests[3]);
    if(noretest)
    {
        store_dvalue( (double)0);
        goto next1;
    }
    if((!distributed) || (distributed && master_iozone))
        start_monitor("Reread");
    walltime = 0.0;
    cputime = 0.0;
    jstarttime=0;
    *stop_flag=0;
    total_kilos=0;
        /* Hooks to start the distributed Iozone client/server code */
        if(distributed)
        {
                use_thread=0;  /* Turn of any Posix threads */
                if(master_iozone)
                        master_listen_socket = start_master_listen();
                else
                        become_client();
        }
    if(!use_thread)
    {
       for(xx = 0; xx< num_child ; xx++){
        chid=xx;
        childids[xx] = start_child_proc(THREAD_REREAD_TEST, numrecs64,reclen);
        if(childids[xx]==-1){
            printf("\nFork failed\n");
            for(xy = 0; xy< xx ; xy++){
                Kill((long long)childids[xy],(long long)SIGTERM);
            }
            exit(32);
        }
        if(childids[xx]==0){
#ifdef _64BIT_ARCH_
            thread_rread_test((void *)xx);
#else
            thread_rread_test((void *)((long)xx));
#endif
        }   
       }
    }
#ifndef NO_THREADS
    else
    {
       for(xx = 0; xx< num_child ; xx++){   /* Create the children */
        chid=xx;
        if(!barray[xx])
        {
            barray[xx]=(char *) alloc_mem((long long)(MAXBUFFERSIZE+cache_size),(int)0);
            if(barray[xx] == 0) {
                   perror("Memory allocation failed:");
                   exit(26);
                }
                barray[xx] =(char *)(((long)barray[xx] + cache_size ) & 
            ~(cache_size-1));
        }
#ifdef _64BIT_ARCH_
        childids[xx] = mythread_create( thread_rread_test,xx);
#else
        childids[xx] = mythread_create( thread_rread_test,(void *)(long)xx);
#endif
        if(childids[xx]==-1){
            printf("\nThread create failed\n");
            for(xy = 0; xy< xx ; xy++){
                kill((pid_t)myid,(int)SIGTERM);
            }
            exit(33);
        }
       }
    }
#endif
    if(myid == (long long)getpid()){
                if(distributed && master_iozone)
                {
                        start_master_listen_loop((int) num_child);
                }
        for(i=0;i<num_child; i++){ /* wait for children to start */
            child_stat = (struct child_stats *)&shmaddr[i];
            while(child_stat->flag==CHILD_STATE_HOLD)
                Poll((long long)1);
        }
        for(i=0;i<num_child; i++){
            child_stat = (struct child_stats *)&shmaddr[i];
            child_stat->flag = CHILD_STATE_BEGIN;   /* tell children to go */
            if(delay_start!=0)
                Poll((long long)delay_start);
                       if(distributed && master_iozone)
                                tell_children_begin(i);
        }
        starttime1 = time_so_far();
        goto jumpend2;
    }
    
jumpend2:
    getout=0;
    if(myid == (long long)getpid()){     /* Parent here */
        for( i = 0; i < num_child; i++){ /* wait for children to stop */
            child_stat = (struct child_stats *)&shmaddr[i];
                        if(distributed && master_iozone)
                        {
                printf("\n\tTest running:");
                                wait_dist_join();
                                break;
                        }
                        else
                        {
               if(use_thread)
               {
                thread_join(childids[i],(void *)&pstatus);
               }
               else
               {
                wait(0);
               }
            }
            if(!jstarttime)
                jstarttime = time_so_far(); 
        }
        jtime = (time_so_far()-jstarttime)-time_res; 
        if(jtime < (double).000001)
        {
            jtime=time_res;
        }
    }
    total_time = (time_so_far() - starttime1)-time_res; /* Parents time */
    if(total_time < (double).000001) 
    {
        total_time=time_res;
        if(rec_prob < reclen)
            rec_prob = reclen;
        res_prob=1;
    }
#ifdef JTIME
    total_time=total_time-jtime;/* Remove the join time */
    if(!silent) printf("\nJoin time %10.2f\n",jtime);
#endif
    min_throughput=max_throughput=min_xfer=0;
    total_kilos=0;
    ptotal=0;
    if(!silent) printf("\n");
    for(xyz=0;xyz<num_child;xyz++){
        child_stat = (struct child_stats *)&shmaddr[xyz];
        total_kilos+=child_stat->throughput;
        ptotal+=child_stat->actual;
        if(!min_xfer)
            min_xfer=child_stat->actual;
        if(child_stat->actual < min_xfer)
            min_xfer=child_stat->actual;
        if(!min_throughput)
            min_throughput=child_stat->throughput;
        if(child_stat->throughput < min_throughput)
            min_throughput=child_stat->throughput;
        if(child_stat->throughput > max_throughput)
            max_throughput=child_stat->throughput;
        cputime += child_stat->cputime;
        /* Get the earliest start time and latest fini time to calc. elapsed time. */
        if (child_stat->walltime < child_stat->cputime)
            child_stat->walltime = child_stat->cputime;
        if (child_stat->walltime > walltime)
            walltime = child_stat->walltime;
    }
    avg_throughput=total_kilos/num_child;
    if(cpuutilflag)
    {
/*
        if (walltime < cputime_res)
            walltime = 0.0;
*/
        if (cputime < cputime_res)
            cputime = 0.0;
    }
    if(cpuutilflag)
        store_times (walltime, cputime);    /* Must be Before store_dvalue(). */
    store_dvalue(total_kilos);
#ifdef NO_PRINT_LLD
    if(!silent) printf("\tChildren see throughput for %ld re-readers \t= %10.2f %s/sec\n", num_child, total_kilos,unit);
    if(!silent && !distributed) printf("\tParent sees throughput for %ld re-readers \t= %10.2f %s/sec\n", num_child, (double)(ptotal)/total_time,unit);
#else
    if(!silent) printf("\tChildren see throughput for %lld re-readers \t= %10.2f %s/sec\n", num_child, total_kilos,unit);
    if(!silent && !distributed) printf("\tParent sees throughput for %lld re-readers \t= %10.2f %s/sec\n", num_child, (double)(ptotal)/total_time,unit);
#endif
    if(!silent) printf("\tMin throughput per %s \t\t\t= %10.2f %s/sec \n", port,min_throughput,unit);
    if(!silent) printf("\tMax throughput per %s \t\t\t= %10.2f %s/sec\n", port,max_throughput,unit);
    if(!silent) printf("\tAvg throughput per %s \t\t\t= %10.2f %s/sec\n", port,avg_throughput,unit);
    if(!silent) printf("\tMin xfer \t\t\t\t\t= %10.2f %s\n", min_xfer,unit);
    /* CPU% can be > 100.0 for multiple CPUs */
    if(cpuutilflag)
    {
        if(walltime == 0.0)
        {
            if(!silent) printf("\tCPU utilization: Wall time %8.3f    CPU time %8.3f    CPU utilization %6.2f %%\n\n",
                walltime, cputime, 0.0);
        }
        else
        {
            if(!silent) printf("\tCPU utilization: Wall time %8.3f    CPU time %8.3f    CPU utilization %6.2f %%\n\n",
                walltime, cputime, 100.0 * cputime / walltime);
        }
    }
    if(Cflag)
    {
        for(xyz=0;xyz<num_child;xyz++)
        {
            child_stat = (struct child_stats *) &shmaddr[xyz];
            if(cpuutilflag)
            {
                if(!silent) printf("\tChild[%ld] xfer count = %10.2f %s, Throughput = %10.2f %s/sec, wall=%6.3f, cpu=%6.3f, %%=%6.2f\n",
                    (long)xyz, child_stat->actual, unit, child_stat->throughput, unit, child_stat->walltime, 
                    child_stat->cputime, cpu_util(child_stat->cputime, child_stat->walltime));
            }
            else
            {
                if(!silent) printf("\tChild[%ld] xfer count = %10.2f %s, Throughput = %10.2f %s/sec\n",
                    (long)xyz, child_stat->actual, unit, child_stat->throughput, unit);
            }
        }
    }
    if((!distributed) || (distributed && master_iozone))
        stop_monitor("Reread");
    /**********************************************************/
    /*************** End of re-readers throughput ****************/
    /**********************************************************/
    sync();
    sleep(2);
    if(restf)
        sleep((int)rest_val);
        if(distributed && master_iozone)
    {
                stop_master_listen(master_listen_socket);
        cleanup_comm();
    }

next1:
    if(include_tflag)
        if(!(include_mask & (long long)REVERSE_MASK))
            goto next2;
    sync();
    sleep(2);

    /**************************************************************/
    /*** Reverse reader throughput tests **************************/
    /**************************************************************/
    toutputindex++;
    strcpy(&toutput[toutputindex][0],throughput_tests[4]);
    if((!distributed) || (distributed && master_iozone))
        start_monitor("Revread");
    walltime = 0.0;
    cputime = 0.0;
    jstarttime=0;
    *stop_flag=0;
    total_kilos=0;
        /* Hooks to start the distributed Iozone client/server code */
        if(distributed)
        {
                use_thread=0;  /* Turn of any Posix threads */
                if(master_iozone)
                        master_listen_socket = start_master_listen();
                else
                        become_client();
        }
    if(!use_thread)
    {
       for(xx = 0; xx< num_child ; xx++){
        chid=xx;
        childids[xx] = start_child_proc(THREAD_REVERSE_READ_TEST,numrecs64,reclen);
        if(childids[xx]==-1){
            printf("\nFork failed\n");
            for(xy = 0; xy< xx ; xy++){
                Kill((long long)childids[xy],(long long)SIGTERM);
            }
            exit(34);
        }
        if(childids[xx]==0){
#ifdef _64BIT_ARCH_
            thread_reverse_read_test((void *)xx);
#else
            thread_reverse_read_test((void *)((long)xx));
#endif
        }   
       }
    }
#ifndef NO_THREADS
    else
    {
       for(xx = 0; xx< num_child ; xx++){   /* Create the children */
        chid=xx;
        if(!barray[xx])
        {
            barray[xx]=(char *) alloc_mem((long long)(MAXBUFFERSIZE+cache_size),(int)0);
            if(barray[xx] == 0) {
                   perror("Memory allocation failed:");
                   exit(26);
                }
                barray[xx] =(char *)(((long)barray[xx] + cache_size ) & 
            ~(cache_size-1));
        }
#ifdef _64BIT_ARCH_
        childids[xx] = mythread_create( thread_reverse_read_test,xx);
#else
        childids[xx] = mythread_create( thread_reverse_read_test,(void *)(long)xx);
#endif
        if(childids[xx]==-1){
            printf("\nThread create failed\n");
            for(xy = 0; xy< xx ; xy++){
                kill((pid_t)myid,(int)SIGTERM);
            }
            exit(35);
        }
       }
    }
#endif
    if(myid == (long long)getpid()){
                if(distributed && master_iozone)
                {
                        start_master_listen_loop((int) num_child);
                }
        for(i=0;i<num_child; i++){ /* wait for children to start */
            child_stat = (struct child_stats *)&shmaddr[i];
            while(child_stat->flag==CHILD_STATE_HOLD)
                Poll((long long)1);
        }
        for(i=0;i<num_child; i++){
            child_stat = (struct child_stats *)&shmaddr[i];
            child_stat->flag = CHILD_STATE_BEGIN;   /* tell children to go */
            if(delay_start!=0)
                Poll((long long)delay_start);
                       if(distributed && master_iozone)
                                tell_children_begin(i);
        }
        starttime1 = time_so_far();
    }
    
    getout=0;
    if(myid == (long long)getpid()){     /* Parent here */
        for( i = 0; i < num_child; i++){ /* wait for children to stop */
            child_stat = (struct child_stats *)&shmaddr[i];
                        if(distributed && master_iozone)
                        {
                printf("\n\tTest running:");
                                wait_dist_join();
                                break;
                        }
                        else
                        {
                           if(use_thread)
                           {
                                thread_join(childids[i],(void *)&pstatus);
                           }
                           else
                           {
                                wait(0);
                           }
                        }
            if(!jstarttime)
                jstarttime = time_so_far(); 
        }
        jtime = (time_so_far()-jstarttime)-time_res;
        if(jtime < (double).000001)
        {
            jtime=time_res;
        }
    }
    total_time = (time_so_far() - starttime1)-time_res; /* Parents time */
    if(total_time < (double).000001) 
    {
        total_time=time_res;
        if(rec_prob < reclen)
            rec_prob = reclen;
        res_prob=1;
    }
#ifdef JTIME
    total_time=total_time-jtime;/* Remove the join time */
    if(!silent) printf("\nJoin time %10.2f\n",jtime);
#endif
    total_kilos=0;
    ptotal=0;
    min_throughput=max_throughput=min_xfer=0;
    if(!silent) printf("\n");
    for(xyz=0;xyz<num_child;xyz++){
        child_stat = (struct child_stats *)&shmaddr[xyz];
        total_kilos+=child_stat->throughput;
        ptotal+=child_stat->actual;
        if(!min_xfer)
            min_xfer=child_stat->actual;
        if(child_stat->actual < min_xfer)
            min_xfer=child_stat->actual;
        if(!min_throughput)
            min_throughput=child_stat->throughput;
        if(child_stat->throughput < min_throughput)
            min_throughput=child_stat->throughput;
        if(child_stat->throughput > max_throughput)
            max_throughput=child_stat->throughput;
        /* walltime += child_stat->walltime; */
        cputime += child_stat->cputime;
        /* Get the earliest start time and latest fini time to calc. elapsed time. */
        if (child_stat->walltime < child_stat->cputime)
            child_stat->walltime = child_stat->cputime;
        if (child_stat->walltime > walltime)
            walltime = child_stat->walltime;
    }
    avg_throughput=total_kilos/num_child;
    if(cpuutilflag)
    {
/*
        if (walltime < cputime_res)
            walltime = 0.0;
*/
        if (cputime < cputime_res)
            cputime = 0.0;
    }
    if(cpuutilflag)
        store_times (walltime, cputime);    /* Must be Before store_dvalue(). */
    store_dvalue(total_kilos);
#ifdef NO_PRINT_LLD
    if(!silent) printf("\tChildren see throughput for %ld reverse readers \t= %10.2f %s/sec\n", num_child, total_kilos,unit);
    if(!silent && !distributed) printf("\tParent sees throughput for %ld reverse readers \t= %10.2f %s/sec\n", num_child, (double)(ptotal)/total_time,unit);
#else
    if(!silent) printf("\tChildren see throughput for %lld reverse readers \t= %10.2f %s/sec\n", num_child, total_kilos,unit);
    if(!silent && !distributed) printf("\tParent sees throughput for %lld reverse readers \t= %10.2f %s/sec\n", num_child, (double)(ptotal)/total_time,unit);
#endif
    if(!silent) printf("\tMin throughput per %s \t\t\t= %10.2f %s/sec \n", port,min_throughput,unit);
    if(!silent) printf("\tMax throughput per %s \t\t\t= %10.2f %s/sec\n", port,max_throughput,unit);
    if(!silent) printf("\tAvg throughput per %s \t\t\t= %10.2f %s/sec\n", port,avg_throughput,unit);
    if(!silent) printf("\tMin xfer \t\t\t\t\t= %10.2f %s\n", min_xfer,unit);
    /* CPU% can be > 100.0 for multiple CPUs */
    if(cpuutilflag)
    {
        if(walltime == 0.0)
        {
            if(!silent) printf("\tCPU utilization: Wall time %8.3f    CPU time %8.3f    CPU utilization %6.2f %%\n\n",
                walltime, cputime, 0.0);
        }
        else
        {
            if(!silent) printf("\tCPU utilization: Wall time %8.3f    CPU time %8.3f    CPU utilization %6.2f %%\n\n",
                walltime, cputime, 100.0 * cputime / walltime);
        }
    }
    if(Cflag)
    {
        for(xyz=0;xyz<num_child;xyz++)
        {
            child_stat = (struct child_stats *) &shmaddr[xyz];
            if(cpuutilflag)
            {
                if(!silent) printf("\tChild[%ld] xfer count = %10.2f %s, Throughput = %10.2f %s/sec, wall=%6.3f, cpu=%6.3f, %%=%6.2f\n",
                    (long)xyz, child_stat->actual, unit, child_stat->throughput, unit, child_stat->walltime, 
                    child_stat->cputime, cpu_util(child_stat->cputime, child_stat->walltime));
            }
            else
            {
                if(!silent) printf("\tChild[%ld] xfer count = %10.2f %s, Throughput = %10.2f %s/sec\n",
                    (long)xyz, child_stat->actual, unit, child_stat->throughput, unit);
            }
        }
    }
    if((!distributed) || (distributed && master_iozone))
        stop_monitor("Revread");
    sync();
    sleep(2);
    if(restf)
        sleep((int)rest_val);
        if(distributed && master_iozone)
    {
                stop_master_listen(master_listen_socket);
        cleanup_comm();
    }
next2:
    if(include_tflag)
        if(!(include_mask & (long long)STRIDE_READ_MASK))
            goto next3;
    /**************************************************************/
    /*** stride reader throughput tests **************************/
    /**************************************************************/
    toutputindex++;
    strcpy(&toutput[toutputindex][0],throughput_tests[5]);
    if((!distributed) || (distributed && master_iozone))
        start_monitor("Strideread");
    walltime = 0.0;
    cputime = 0.0;
    jstarttime=0;
    sync();
    sleep(2);
    *stop_flag=0;
    total_kilos=0;
        /* Hooks to start the distributed Iozone client/server code */
        if(distributed)
        {
                use_thread=0;  /* Turn of any Posix threads */
                if(master_iozone)
                        master_listen_socket = start_master_listen();
                else
                        become_client();
        }
    if(!use_thread)
    {
       for(xx = 0; xx< num_child ; xx++){
        chid=xx;
        childids[xx] = start_child_proc(THREAD_STRIDE_TEST,numrecs64,reclen);
        if(childids[xx]==-1){
            printf("\nFork failed\n");
            for(xy = 0; xy< xx ; xy++){
                Kill((long long)childids[xy],(long long)SIGTERM);
            }
            exit(36);
        }
        if(childids[xx]==0){
#ifdef _64BIT_ARCH_
            thread_stride_read_test((void *)xx);
#else
            thread_stride_read_test((void *)((long)xx));
#endif
        }   
       }
    }
#ifndef NO_THREADS
    else
    {
       for(xx = 0; xx< num_child ; xx++){   /* Create the children */
        chid=xx;
        if(!barray[xx])
        {
            barray[xx]=(char *) alloc_mem((long long)(MAXBUFFERSIZE+cache_size),(int)0);
            if(barray[xx] == 0) {
                   perror("Memory allocation failed:");
                   exit(26);
                }
                barray[xx] =(char *)(((long)barray[xx] + cache_size ) & 
            ~(cache_size-1));
        }
#ifdef _64BIT_ARCH_
        childids[xx] = mythread_create( thread_stride_read_test,xx);
#else
        childids[xx] = mythread_create( thread_stride_read_test,(void *)(long)xx);
#endif
        if(childids[xx]==-1){
            printf("\nThread create failed\n");
            for(xy = 0; xy< xx ; xy++){
                kill((pid_t)myid,(int)SIGTERM);
            }
            exit(37);
        }
       }
    }
#endif
    if(myid == (long long)getpid()){
                if(distributed && master_iozone)
                {
                        start_master_listen_loop((int) num_child);
                }
        for(i=0;i<num_child; i++){ /* wait for children to start */
            child_stat = (struct child_stats *)&shmaddr[i];
            while(child_stat->flag==CHILD_STATE_HOLD)
                Poll((long long)1);
        }
        for(i=0;i<num_child; i++){
            child_stat = (struct child_stats *)&shmaddr[i];
            child_stat->flag = CHILD_STATE_BEGIN;   /* tell children to go */
            if(delay_start!=0)
                Poll((long long)delay_start);
                       if(distributed && master_iozone)
                                tell_children_begin(i);
        }
        starttime1 = time_so_far();
    }
    
    getout=0;
    if(myid == (long long)getpid()){     /* Parent here */
        for( i = 0; i < num_child; i++){ /* wait for children to stop */
            child_stat = (struct child_stats *)&shmaddr[i];
                        if(distributed && master_iozone)
                        {
                printf("\n\tTest running:");
                                wait_dist_join();
                                break;
                        }
                        else
                        {
                           if(use_thread)
                           {
                                thread_join(childids[i],(void *)&pstatus);
                           }
                           else
                           {
                                wait(0);
                           }
                        }
            if(!jstarttime)
                jstarttime = time_so_far(); 
        }
        jtime = (time_so_far()-jstarttime)-time_res;
        if(jtime < (double).000001) 
        {
            jtime=time_res; 
        }
    }
    total_time = (time_so_far() - starttime1)-time_res; /* Parents time */
    if(total_time < (double).000001) 
    {
        total_time=time_res;
        if(rec_prob < reclen)
            rec_prob = reclen;
        res_prob=1;
    }
#ifdef JTIME
    total_time=total_time-jtime;/* Remove the join time */
    if(!silent) printf("\nJoin time %10.2f\n",jtime);
#endif
    total_kilos=0;
    ptotal=0;
    min_throughput=max_throughput=min_xfer=0;
    if(!silent) printf("\n");
    for(xyz=0;xyz<num_child;xyz++){
        child_stat = (struct child_stats *)&shmaddr[xyz];
        total_kilos+=child_stat->throughput;
        ptotal+=child_stat->actual;
        if(!min_xfer)
            min_xfer=child_stat->actual;
        if(child_stat->actual < min_xfer)
            min_xfer=child_stat->actual;
        if(!min_throughput)
            min_throughput=child_stat->throughput;
        if(child_stat->throughput < min_throughput)
            min_throughput=child_stat->throughput;
        if(child_stat->throughput > max_throughput)
            max_throughput=child_stat->throughput;
        /* walltime += child_stat->walltime; */
        cputime += child_stat->cputime;
        /* Get the biggest walltime */
        if (child_stat->walltime < child_stat->cputime)
            child_stat->walltime = child_stat->cputime;
        if (child_stat->walltime > walltime)
            walltime = child_stat->walltime;
    }
    avg_throughput=total_kilos/num_child;
    if(cpuutilflag)
    {
/*
        if (walltime < cputime_res)
            walltime = 0.0;
*/
        if (cputime < cputime_res)
            cputime = 0.0;
    }
    if(cpuutilflag)
        store_times (walltime, cputime);    /* Must be Before store_dvalue(). */
    store_dvalue(total_kilos);
#ifdef NO_PRINT_LLD
    if(!silent) printf("\tChildren see throughput for %ld stride readers \t= %10.2f %s/sec\n", num_child, total_kilos,unit);
    if(!silent && !distributed) printf("\tParent sees throughput for %ld stride readers \t= %10.2f %s/sec\n", num_child, (double)(ptotal)/total_time,unit);
#else
    if(!silent) printf("\tChildren see throughput for %lld stride readers \t= %10.2f %s/sec\n", num_child, total_kilos,unit);
    if(!silent && !distributed) printf("\tParent sees throughput for %lld stride readers \t= %10.2f %s/sec\n", num_child, (double)(ptotal)/total_time,unit);
#endif
    if(!silent) printf("\tMin throughput per %s \t\t\t= %10.2f %s/sec \n", port,min_throughput,unit);
    if(!silent) printf("\tMax throughput per %s \t\t\t= %10.2f %s/sec\n", port,max_throughput,unit);
    if(!silent) printf("\tAvg throughput per %s \t\t\t= %10.2f %s/sec\n", port,avg_throughput,unit);
    if(!silent) printf("\tMin xfer \t\t\t\t\t= %10.2f %s\n", min_xfer,unit);
    /* CPU% can be > 100.0 for multiple CPUs */
    if(cpuutilflag)
    {
        if(walltime == 0.0)
        {
            if(!silent) printf("\tCPU utilization: Wall time %8.3f    CPU time %8.3f    CPU utilization %6.2f %%\n\n",
                walltime, cputime, 0.0);
        }
        else
        {
            if(!silent) printf("\tCPU utilization: Wall time %8.3f    CPU time %8.3f    CPU utilization %6.2f %%\n\n",
                walltime, cputime, 100.0 * cputime / walltime);
        }
    }
    if(Cflag)
    {
        for(xyz=0;xyz<num_child;xyz++)
        {
            child_stat = (struct child_stats *) &shmaddr[xyz];
            if(cpuutilflag)
            {
                if(!silent) printf("\tChild[%ld] xfer count = %10.2f %s, Throughput = %10.2f %s/sec, wall=%6.3f, cpu=%6.3f, %%=%6.2f\n",
                    (long)xyz, child_stat->actual, unit, child_stat->throughput, unit, child_stat->walltime, 
                    child_stat->cputime, cpu_util(child_stat->cputime, child_stat->walltime));
            }
            else
            {
                if(!silent) printf("\tChild[%ld] xfer count = %10.2f %s, Throughput = %10.2f %s/sec\n",
                    (long)xyz, child_stat->actual, unit, child_stat->throughput, unit);
            }
        }
    }
    if((!distributed) || (distributed && master_iozone))
        stop_monitor("Strideread");
    sync();
    sleep(2);
    if(restf)
        sleep((int)rest_val);
        if(distributed && master_iozone)
    {
                stop_master_listen(master_listen_socket);
        cleanup_comm();
    }
    /**************************************************************/
    /*** random reader throughput tests ***************************/
    /**************************************************************/
next3:
    if(include_tflag)
        if(!(include_mask & (long long)RANDOM_RW_MASK))
            goto next4;
    
    toutputindex++;
    strcpy(&toutput[toutputindex][0],throughput_tests[6]);
    if((!distributed) || (distributed && master_iozone))
        start_monitor("Randread");
    walltime = 0.0;
    cputime = 0.0;
    jstarttime=0;
    sync();
    sleep(2);
    *stop_flag=0;
    total_kilos=0;
        /* Hooks to start the distributed Iozone client/server code */
        if(distributed)
        {
                use_thread=0;  /* Turn of any Posix threads */
                if(master_iozone)
                        master_listen_socket = start_master_listen();
                else
                        become_client();
        }
    if(!use_thread)
    {
       for(xx = 0; xx< num_child ; xx++){
        chid=xx;
        childids[xx] = start_child_proc(THREAD_RANDOM_READ_TEST,numrecs64,reclen);
        if(childids[xx]==-1){
            printf("\nFork failed\n");
            for(xy = 0; xy< xx ; xy++){
                Kill((long long)childids[xy],(long long)SIGTERM);
            }
            exit(38);
        }
        if(childids[xx]==0){
#ifdef _64BIT_ARCH_
            thread_ranread_test((void *)xx);
#else
            thread_ranread_test((void *)((long)xx));
#endif
        }   
       }
    }
#ifndef NO_THREADS
    else
    {
       for(xx = 0; xx< num_child ; xx++){   /* Create the children */
        chid=xx;
        if(!barray[xx])
        {
            barray[xx]=(char *) alloc_mem((long long)(MAXBUFFERSIZE+cache_size),(int)0);
            if(barray[xx] == 0) {
                   perror("Memory allocation failed:");
                   exit(26);
                }
                barray[xx] =(char *)(((long)barray[xx] + cache_size ) & 
            ~(cache_size-1));
        }
#ifdef _64BIT_ARCH_
        childids[xx] = mythread_create( thread_ranread_test,xx);
#else
        childids[xx] = mythread_create( thread_ranread_test,(void *)(long)xx);
#endif
        if(childids[xx]==-1){
            printf("\nThread create failed\n");
            for(xy = 0; xy< xx ; xy++){
                kill((pid_t)myid,(int)SIGTERM);
            }
            exit(39);
        }
       }
    }
#endif
    if(myid == (long long)getpid()){
                if(distributed && master_iozone)
                {
                        start_master_listen_loop((int) num_child);
                }
        for(i=0;i<num_child; i++){ /* wait for children to start */
            child_stat = (struct child_stats *)&shmaddr[i];
            while(child_stat->flag==CHILD_STATE_HOLD)
                Poll((long long)1);
        }
        for(i=0;i<num_child; i++){
            child_stat = (struct child_stats *)&shmaddr[i];
            child_stat->flag = CHILD_STATE_BEGIN;   /* tell children to go */
            if(delay_start!=0)
                Poll((long long)delay_start);
                       if(distributed && master_iozone)
                                tell_children_begin(i);
        }
        starttime1 = time_so_far();
    }
    
    getout=0;
    if(myid == (long long)getpid()){     /* Parent here */
        for( i = 0; i < num_child; i++){ /* wait for children to stop */
            child_stat = (struct child_stats *)&shmaddr[i];
                        if(distributed && master_iozone)
                        {
                printf("\n\tTest running:");
                                wait_dist_join();
                                break;
                        }
                        else
                        {
                           if(use_thread)
                           {
                                thread_join(childids[i],(void *)&pstatus);
                           }
                           else
                           {
                                wait(0);
                           }
                        }
            if(!jstarttime)
                jstarttime = time_so_far(); 
        }
        jtime = (time_so_far()-jstarttime)-time_res;
        if(jtime < (double).000001) 
        {
            jtime=time_res; 
        }
    }
    total_time = (time_so_far() - starttime1)-time_res; /* Parents time */
    if(total_time < (double).000001) 
    {
        total_time=time_res;
        if(rec_prob < reclen)
            rec_prob = reclen;
        res_prob=1;
    }
#ifdef JTIME
    total_time=total_time-jtime;/* Remove the join time */
    if(!silent) printf("\nJoin time %10.2f\n",jtime);
#endif
    total_kilos=0;
    ptotal=0;
    min_throughput=max_throughput=min_xfer=0;
    if(!silent) printf("\n");
    for(xyz=0;xyz<num_child;xyz++){
        child_stat = (struct child_stats *)&shmaddr[xyz];
        total_kilos+=child_stat->throughput;
        ptotal+=child_stat->actual;
        if(!min_xfer)
            min_xfer=child_stat->actual;
        if(child_stat->actual < min_xfer)
            min_xfer=child_stat->actual;
        if(!min_throughput)
            min_throughput=child_stat->throughput;
        if(child_stat->throughput < min_throughput)
            min_throughput=child_stat->throughput;
        if(child_stat->throughput > max_throughput)
            max_throughput=child_stat->throughput;
        cputime += child_stat->cputime;
        /* Get the biggest walltime */
        if (child_stat->walltime < child_stat->cputime)
            child_stat->walltime = child_stat->cputime;
        if (child_stat->walltime > walltime)
            walltime = child_stat->walltime;
    }
    avg_throughput=total_kilos/num_child;
    if(cpuutilflag)
    {
        if (cputime < cputime_res)
            cputime = 0.0;
    }
    if(cpuutilflag)
        store_times (walltime, cputime);    /* Must be Before store_dvalue(). */
    store_dvalue(total_kilos);
#ifdef NO_PRINT_LLD
    if(!silent) printf("\tChildren see throughput for %ld random readers \t= %10.2f %s/sec\n", num_child, total_kilos,unit);
    if(!silent && !distributed) printf("\tParent sees throughput for %ld random readers \t= %10.2f %s/sec\n", num_child, (double)(ptotal)/total_time,unit);
#else
    if(!silent) printf("\tChildren see throughput for %lld random readers \t= %10.2f %s/sec\n", num_child, total_kilos,unit);
    if(!silent && !distributed) printf("\tParent sees throughput for %lld random readers \t= %10.2f %s/sec\n", num_child, (double)(ptotal)/total_time,unit);
#endif
    if(!silent) printf("\tMin throughput per %s \t\t\t= %10.2f %s/sec \n", port,min_throughput,unit);
    if(!silent) printf("\tMax throughput per %s \t\t\t= %10.2f %s/sec\n", port,max_throughput,unit);
    if(!silent) printf("\tAvg throughput per %s \t\t\t= %10.2f %s/sec\n", port,avg_throughput,unit);
    if(!silent) printf("\tMin xfer \t\t\t\t\t= %10.2f %s\n", min_xfer,unit);
    /* CPU% can be > 100.0 for multiple CPUs */
    if(cpuutilflag)
    {
        if(walltime == 0.0)
        {
            if(!silent) printf("\tCPU utilization: Wall time %8.3f    CPU time %8.3f    CPU utilization %6.2f %%\n\n",
                walltime, cputime, 0.0);
        }
        else
        {
            if(!silent) printf("\tCPU utilization: Wall time %8.3f    CPU time %8.3f    CPU utilization %6.2f %%\n\n",
                walltime, cputime, 100.0 * cputime / walltime);
        }
    }
    if(Cflag)
    {
        for(xyz=0;xyz<num_child;xyz++)
        {
            child_stat = (struct child_stats *) &shmaddr[xyz];
            if(cpuutilflag)
            {
                if(!silent) printf("\tChild[%ld] xfer count = %10.2f %s, Throughput = %10.2f %s/sec, wall=%6.3f, cpu=%6.3f, %%=%6.2f\n",
                    (long)xyz, child_stat->actual, unit, child_stat->throughput, unit, child_stat->walltime, 
                    child_stat->cputime, cpu_util(child_stat->cputime, child_stat->walltime));
            }
            else
            {
                if(!silent) printf("\tChild[%ld] xfer count = %10.2f %s, Throughput = %10.2f %s/sec\n",
                    (long)xyz, child_stat->actual, unit, child_stat->throughput, unit);
            }
        }
    }
    if((!distributed) || (distributed && master_iozone))
        stop_monitor("Randread");
    sync();
    sleep(2);
    if(restf)
        sleep((int)rest_val);
        if(distributed && master_iozone)
    {
                stop_master_listen(master_listen_socket);
        cleanup_comm();
    }
    /**************************************************************/
    /***  mixed workload throughput tests ***************************/
    /**************************************************************/
next4:
    if(include_tflag)
        if(!(include_mask & (long long)RANDOM_MIX_MASK))
            goto next5;
    
    toutputindex++;
    strcpy(&toutput[toutputindex][0],throughput_tests[7]);
    if((!distributed) || (distributed && master_iozone))
        start_monitor("Mixed");
    walltime = 0.0;
    cputime = 0.0;
    jstarttime=0;
    sync();
    sleep(2);
    *stop_flag=0;
    total_kilos=0;
        /* Hooks to start the distributed Iozone client/server code */
        if(distributed)
        {
                use_thread=0;  /* Turn of any Posix threads */
                if(master_iozone)
                        master_listen_socket = start_master_listen();
                else
                        become_client();
        }
    if(!use_thread)
    {
       for(xx = 0; xx< num_child ; xx++){
        chid=xx;
        childids[xx] = start_child_proc(THREAD_RANDOM_MIX_TEST,numrecs64,reclen);
        if(childids[xx]==-1){
            printf("\nFork failed\n");
            for(xy = 0; xy< xx ; xy++){
                Kill((long long)childids[xy],(long long)SIGTERM);
            }
            exit(38);
        }
        if(childids[xx]==0){
#ifdef _64BIT_ARCH_
            thread_mix_test((void *)xx);
#else
            thread_mix_test((void *)((long)xx));
#endif
        }   
       }
    }
#ifndef NO_THREADS
    else
    {
       for(xx = 0; xx< num_child ; xx++){   /* Create the children */
        chid=xx;
        if(!barray[xx])
        {
            barray[xx]=(char *) alloc_mem((long long)(MAXBUFFERSIZE+cache_size),(int)0);
            if(barray[xx] == 0) {
                   perror("Memory allocation failed:");
                   exit(26);
                }
                barray[xx] =(char *)(((long)barray[xx] + cache_size ) & 
            ~(cache_size-1));
        }
#ifdef _64BIT_ARCH_
        childids[xx] = mythread_create( thread_mix_test,xx);
#else
        childids[xx] = mythread_create( thread_mix_test,(void *)(long)xx);
#endif
        if(childids[xx]==-1){
            printf("\nThread create failed\n");
            for(xy = 0; xy< xx ; xy++){
                kill((pid_t)myid,(int)SIGTERM);
            }
            exit(39);
        }
       }
    }
#endif
    if(myid == (long long)getpid()){
                if(distributed && master_iozone)
                {
                        start_master_listen_loop((int) num_child);
                }
        for(i=0;i<num_child; i++){ /* wait for children to start */
            child_stat = (struct child_stats *)&shmaddr[i];
            while(child_stat->flag==CHILD_STATE_HOLD)
                Poll((long long)1);
        }
        for(i=0;i<num_child; i++){
            child_stat = (struct child_stats *)&shmaddr[i];
            child_stat->flag = CHILD_STATE_BEGIN;   /* tell children to go */
            if(delay_start!=0)
                Poll((long long)delay_start);
                       if(distributed && master_iozone)
                                tell_children_begin(i);
        }
        starttime1 = time_so_far();
    }
    
    getout=0;
    if(myid == (long long)getpid()){     /* Parent here */
        for( i = 0; i < num_child; i++){ /* wait for children to stop */
            child_stat = (struct child_stats *)&shmaddr[i];
                        if(distributed && master_iozone)
                        {
                printf("\n\tTest running:");
                                wait_dist_join();
                                break;
                        }
                        else
                        {
                           if(use_thread)
                           {
                                thread_join(childids[i],(void *)&pstatus);
                           }
                           else
                           {
                                wait(0);
                           }
                        }
            if(!jstarttime)
                jstarttime = time_so_far(); 
        }
        jtime = (time_so_far()-jstarttime)-time_res;
        if(jtime < (double).000001) 
        {
            jtime=time_res; 
        }
    }
    total_time = (time_so_far() - starttime1)-time_res; /* Parents time */
    if(total_time < (double).000001) 
    {
        total_time=time_res;
        if(rec_prob < reclen)
            rec_prob = reclen;
        res_prob=1;
    }
#ifdef JTIME
    total_time=total_time-jtime;/* Remove the join time */
    if(!silent) printf("\nJoin time %10.2f\n",jtime);
#endif
    total_kilos=0;
    ptotal=0;
    min_throughput=max_throughput=min_xfer=0;
    if(!silent) printf("\n");
    for(xyz=0;xyz<num_child;xyz++){
        child_stat = (struct child_stats *)&shmaddr[xyz];
        total_kilos+=child_stat->throughput;
        ptotal+=child_stat->actual;
        if(!min_xfer)
            min_xfer=child_stat->actual;
        if(child_stat->actual < min_xfer)
            min_xfer=child_stat->actual;
        if(!min_throughput)
            min_throughput=child_stat->throughput;
        if(child_stat->throughput < min_throughput)
            min_throughput=child_stat->throughput;
        if(child_stat->throughput > max_throughput)
            max_throughput=child_stat->throughput;
        cputime += child_stat->cputime;
        /* Get the biggest walltime */
        if (child_stat->walltime < child_stat->cputime)
            child_stat->walltime = child_stat->cputime;
        if (child_stat->walltime > walltime)
            walltime = child_stat->walltime;
    }
    avg_throughput=total_kilos/num_child;
    if(cpuutilflag)
    {
        if (cputime < cputime_res)
            cputime = 0.0;
    }
    if(cpuutilflag)
        store_times (walltime, cputime);    /* Must be Before store_dvalue(). */
    store_dvalue(total_kilos);
#ifdef NO_PRINT_LLD
    if(!silent) printf("\tChildren see throughput for %ld mixed workload \t= %10.2f %s/sec\n", num_child, total_kilos,unit);
    if(!silent && !distributed) printf("\tParent sees throughput for %ld mixed workload \t= %10.2f %s/sec\n", num_child, (double)(ptotal)/total_time,unit);
#else
    if(!silent) printf("\tChildren see throughput for %lld mixed workload \t= %10.2f %s/sec\n", num_child, total_kilos,unit);
    if(!silent && !distributed) printf("\tParent sees throughput for %lld mixed workload \t= %10.2f %s/sec\n", num_child, (double)(ptotal)/total_time,unit);
#endif
    if(!silent) printf("\tMin throughput per %s \t\t\t= %10.2f %s/sec \n", port,min_throughput,unit);
    if(!silent) printf("\tMax throughput per %s \t\t\t= %10.2f %s/sec\n", port,max_throughput,unit);
    if(!silent) printf("\tAvg throughput per %s \t\t\t= %10.2f %s/sec\n", port,avg_throughput,unit);
    if(!silent) printf("\tMin xfer \t\t\t\t\t= %10.2f %s\n", min_xfer,unit);
    /* CPU% can be > 100.0 for multiple CPUs */
    if(cpuutilflag)
    {
        if(walltime == 0.0)
        {
            if(!silent) printf("\tCPU utilization: Wall time %8.3f    CPU time %8.3f    CPU utilization %6.2f %%\n\n",
                walltime, cputime, 0.0);
        }
        else
        {
            if(!silent) printf("\tCPU utilization: Wall time %8.3f    CPU time %8.3f    CPU utilization %6.2f %%\n\n",
                walltime, cputime, 100.0 * cputime / walltime);
        }
    }
    if(Cflag)
    {
        for(xyz=0;xyz<num_child;xyz++)
        {
            child_stat = (struct child_stats *) &shmaddr[xyz];
            if(cpuutilflag)
            {
                if(!silent) printf("\tChild[%ld] xfer count = %10.2f %s, Throughput = %10.2f %s/sec, wall=%6.3f, cpu=%6.3f, %%=%6.2f\n",
                    (long)xyz, child_stat->actual, unit, child_stat->throughput, unit, child_stat->walltime, 
                    child_stat->cputime, cpu_util(child_stat->cputime, child_stat->walltime));
            }
            else
            {
                if(!silent) printf("\tChild[%ld] xfer count = %10.2f %s, Throughput = %10.2f %s/sec\n",
                    (long)xyz, child_stat->actual, unit, child_stat->throughput, unit);
            }
        }
    }
    if((!distributed) || (distributed && master_iozone))
        stop_monitor("Mixed");
    sync();
    sleep(2);
    if(restf)
        sleep((int)rest_val);
        if(distributed && master_iozone)
    {
                stop_master_listen(master_listen_socket);
        cleanup_comm();
    }
next5:
    /**************************************************************/
    /*** random writer throughput tests  **************************/
    /**************************************************************/
    if(include_tflag)
        if(!(include_mask & (long long)RANDOM_RW_MASK) || no_write)
            goto next6;
    
    toutputindex++;
    strcpy(&toutput[toutputindex][0],throughput_tests[8]);
    if((!distributed) || (distributed && master_iozone))
        start_monitor("Randwrite");
    walltime = 0.0;
    cputime = 0.0;
    jstarttime=0;
    sync();
    sleep(2);
    *stop_flag=0;
    total_kilos=0;
        /* Hooks to start the distributed Iozone client/server code */
        if(distributed)
        {
                use_thread=0;  /* Turn of any Posix threads */
                if(master_iozone)
                        master_listen_socket = start_master_listen();
                else
                        become_client();
        }
    if(!use_thread)
    {
       for(xx = 0; xx< num_child ; xx++){
        chid=xx;
        childids[xx] = start_child_proc(THREAD_RANDOM_WRITE_TEST,numrecs64,reclen);
        if(childids[xx]==-1){
            printf("\nFork failed\n");
            for(xy = 0; xy< xx ; xy++){
                Kill((long long)childids[xy],(long long)SIGTERM);
            }
            exit(38);
        }
        if(childids[xx]==0){
#ifdef _64BIT_ARCH_
            thread_ranwrite_test((void *)xx);
#else
            thread_ranwrite_test((void *)((long)xx));
#endif
        }   
       }
    }
#ifndef NO_THREADS
    else
    {
       for(xx = 0; xx< num_child ; xx++){   /* Create the children */
        chid=xx;
        if(!barray[xx])
        {
            barray[xx]=(char *) alloc_mem((long long)(MAXBUFFERSIZE+cache_size),(int)0);
            if(barray[xx] == 0) {
                   perror("Memory allocation failed:");
                   exit(26);
                }
                barray[xx] =(char *)(((long)barray[xx] + cache_size ) & 
            ~(cache_size-1));
        }
#ifdef _64BIT_ARCH_
        childids[xx] = mythread_create( thread_ranwrite_test,xx);
#else
        childids[xx] = mythread_create( thread_ranwrite_test,(void *)(long)xx);
#endif
        if(childids[xx]==-1){
            printf("\nThread create failed\n");
            for(xy = 0; xy< xx ; xy++){
                kill((pid_t)myid,(int)SIGTERM);
            }
            exit(39);
        }
       }
    }
#endif
    if(myid == (long long)getpid()){
                if(distributed && master_iozone)
                {
                        start_master_listen_loop((int) num_child);
                }
        for(i=0;i<num_child; i++){ /* wait for children to start */
            child_stat = (struct child_stats *)&shmaddr[i];
            while(child_stat->flag==CHILD_STATE_HOLD)
                Poll((long long)1);
        }
        for(i=0;i<num_child; i++){
            child_stat = (struct child_stats *)&shmaddr[i];
            child_stat->flag = CHILD_STATE_BEGIN;   /* tell children to go */
            if(delay_start!=0)
                Poll((long long)delay_start);
                       if(distributed && master_iozone)
                                tell_children_begin(i);
        }
        starttime1 = time_so_far();
    }
    
    getout=0;
    if(myid == (long long)getpid()){     /* Parent here */
        for( i = 0; i < num_child; i++){ /* wait for children to stop */
            child_stat = (struct child_stats *)&shmaddr[i];
                        if(distributed && master_iozone)
                        {
                printf("\n\tTest running:");
                                wait_dist_join();
                                break;
                        }
                        else
                        {
                           if(use_thread)
                           {
                                thread_join(childids[i],(void *)&pstatus);
                           }
                           else
                           {
                                wait(0);
                           }
                        }
            if(!jstarttime)
                jstarttime = time_so_far(); 
        }
        jtime = (time_so_far()-jstarttime)-time_res;
        if(jtime < (double).000001) 
        {
            jtime=time_res; 
        }
    }
    total_time = (time_so_far() - starttime1)-time_res; /* Parents time */
    if(total_time < (double).000001) 
    {
        total_time=time_res;
        if(rec_prob < reclen)
            rec_prob = reclen;
        res_prob=1;
    }
#ifdef JTIME
    total_time=total_time-jtime;/* Remove the join time */
    if(!silent) printf("\nJoin time %10.2f\n",jtime);
#endif
    total_kilos=0;
    ptotal=0;
    min_throughput=max_throughput=min_xfer=0;
    if(!silent) printf("\n");
    for(xyz=0;xyz<num_child;xyz++){
        child_stat = (struct child_stats *)&shmaddr[xyz];
        total_kilos+=child_stat->throughput;
        ptotal+=child_stat->actual;
        if(!min_xfer)
            min_xfer=child_stat->actual;
        if(child_stat->actual < min_xfer)
            min_xfer=child_stat->actual;
        if(!min_throughput)
            min_throughput=child_stat->throughput;
        if(child_stat->throughput < min_throughput)
            min_throughput=child_stat->throughput;
        if(child_stat->throughput > max_throughput)
            max_throughput=child_stat->throughput;
        cputime += child_stat->cputime;
        /* Get the biggest walltime */
        if (child_stat->walltime < child_stat->cputime)
            child_stat->walltime = child_stat->cputime;
        if (child_stat->walltime > walltime)
            walltime = child_stat->walltime;
    }
    avg_throughput=total_kilos/num_child;
    if(cpuutilflag)
    {
        if (cputime < cputime_res)
            cputime = 0.0;
    }
    if(cpuutilflag)
        store_times (walltime, cputime);    /* Must be Before store_dvalue(). */
    store_dvalue(total_kilos);
#ifdef NO_PRINT_LLD
    if(!silent) printf("\tChildren see throughput for %ld random writers \t= %10.2f %s/sec\n", num_child, total_kilos,unit);
    if(!silent && !distributed) printf("\tParent sees throughput for %ld random writers \t= %10.2f %s/sec\n", num_child, (double)(ptotal)/total_time,unit);
#else
    if(!silent) printf("\tChildren see throughput for %lld random writers \t= %10.2f %s/sec\n", num_child, total_kilos,unit);
    if(!silent && !distributed) printf("\tParent sees throughput for %lld random writers \t= %10.2f %s/sec\n", num_child, (double)(ptotal)/total_time,unit);
#endif
    if(!silent) printf("\tMin throughput per %s \t\t\t= %10.2f %s/sec \n", port,min_throughput,unit);
    if(!silent) printf("\tMax throughput per %s \t\t\t= %10.2f %s/sec\n", port,max_throughput,unit);
    if(!silent) printf("\tAvg throughput per %s \t\t\t= %10.2f %s/sec\n", port,avg_throughput,unit);
    if(!silent) printf("\tMin xfer \t\t\t\t\t= %10.2f %s\n", min_xfer,unit);
    /* CPU% can be > 100.0 for multiple CPUs */
    if(cpuutilflag)
    {
        if(walltime == 0.0)
        {
            if(!silent) printf("\tCPU utilization: Wall time %8.3f    CPU time %8.3f    CPU utilization %6.2f %%\n\n",
                walltime, cputime, 0.0);
        }
        else
        {
            if(!silent) printf("\tCPU utilization: Wall time %8.3f    CPU time %8.3f    CPU utilization %6.2f %%\n\n",
                walltime, cputime, 100.0 * cputime / walltime);
        }
    }
    if(Cflag)
    {
        for(xyz=0;xyz<num_child;xyz++)
        {
            child_stat = (struct child_stats *) &shmaddr[xyz];
            if(cpuutilflag)
            {
                if(!silent) printf("\tChild[%ld] xfer count = %10.2f %s, Throughput = %10.2f %s/sec, wall=%6.3f, cpu=%6.3f, %%=%6.2f\n",
                    (long)xyz, child_stat->actual, unit, child_stat->throughput, unit, child_stat->walltime, 
                    child_stat->cputime, cpu_util(child_stat->cputime, child_stat->walltime));
            }
            else
            {
                if(!silent) printf("\tChild[%ld] xfer count = %10.2f %s, Throughput = %10.2f %s/sec\n",
                    (long)xyz, child_stat->actual, unit, child_stat->throughput, unit);
            }
        }
    }
    if((!distributed) || (distributed && master_iozone))
        stop_monitor("Randwrite");
    sync();
    sleep(2);
    if(restf)
        sleep((int)rest_val);
        if(distributed && master_iozone)
    {
                stop_master_listen(master_listen_socket);
        cleanup_comm();
    }
next6:
    /**************************************************************/
    /*** Pwrite writer throughput tests  **************************/
    /**************************************************************/
#ifndef HAVE_PREAD
        goto next7;
#else
    if(include_tflag)
        if(!(include_mask & (long long)PWRITER_MASK))
            goto next7;
    
    toutputindex++;
    strcpy(&toutput[toutputindex][0],throughput_tests[9]);
    if((!distributed) || (distributed && master_iozone))
        start_monitor("Pwrite");
    walltime = 0.0;
    cputime = 0.0;
    jstarttime=0;
    sync();
    sleep(2);
    *stop_flag=0;
    total_kilos=0;
        /* Hooks to start the distributed Iozone client/server code */
        if(distributed)
        {
                use_thread=0;  /* Turn of any Posix threads */
                if(master_iozone)
                        master_listen_socket = start_master_listen();
                else
                        become_client();
        }
    if(!use_thread)
    {
       for(xx = 0; xx< num_child ; xx++){
        chid=xx;
        childids[xx] = start_child_proc(THREAD_PWRITE_TEST,numrecs64,reclen);
        if(childids[xx]==-1){
            printf("\nFork failed\n");
            for(xy = 0; xy< xx ; xy++){
                Kill((long long)childids[xy],(long long)SIGTERM);
            }
            exit(38);
        }
        if(childids[xx]==0){
#ifdef _64BIT_ARCH_
            thread_pwrite_test((void *)xx);
#else
            thread_pwrite_test((void *)((long)xx));
#endif
        }   
       }
    }
#ifndef NO_THREADS
    else
    {
       for(xx = 0; xx< num_child ; xx++){   /* Create the children */
        chid=xx;
        if(!barray[xx])
        {
            barray[xx]=(char *) alloc_mem((long long)(MAXBUFFERSIZE+cache_size),(int)0);
            if(barray[xx] == 0) {
                   perror("Memory allocation failed:");
                   exit(26);
                }
                barray[xx] =(char *)(((long)barray[xx] + cache_size ) & 
            ~(cache_size-1));
        }
#ifdef _64BIT_ARCH_
        childids[xx] = mythread_create( thread_pwrite_test,xx);
#else
        childids[xx] = mythread_create( thread_pwrite_test,(void *)(long)xx);
#endif
        if(childids[xx]==-1){
            printf("\nThread create failed\n");
            for(xy = 0; xy< xx ; xy++){
                kill((pid_t)myid,(int)SIGTERM);
            }
            exit(39);
        }
       }
    }
#endif
    if(myid == (long long)getpid()){
                if(distributed && master_iozone)
                {
                        start_master_listen_loop((int) num_child);
                }
        for(i=0;i<num_child; i++){ /* wait for children to start */
            child_stat = (struct child_stats *)&shmaddr[i];
            while(child_stat->flag==CHILD_STATE_HOLD)
                Poll((long long)1);
        }
        for(i=0;i<num_child; i++){
            child_stat = (struct child_stats *)&shmaddr[i];
            child_stat->flag = CHILD_STATE_BEGIN;   /* tell children to go */
            if(delay_start!=0)
                Poll((long long)delay_start);
                       if(distributed && master_iozone)
                                tell_children_begin(i);
        }
        starttime1 = time_so_far();
    }
    
    getout=0;
    if(myid == (long long)getpid()){     /* Parent here */
        for( i = 0; i < num_child; i++){ /* wait for children to stop */
            child_stat = (struct child_stats *)&shmaddr[i];
                        if(distributed && master_iozone)
                        {
                printf("\n\tTest running:");
                                wait_dist_join();
                                break;
                        }
                        else
                        {
                           if(use_thread)
                           {
                                thread_join(childids[i],(void *)&pstatus);
                           }
                           else
                           {
                                wait(0);
                           }
                        }
            if(!jstarttime)
                jstarttime = time_so_far(); 
        }
        jtime = (time_so_far()-jstarttime)-time_res;
        if(jtime < (double).000001) 
        {
            jtime=time_res; 
        }
    }
    total_time = (time_so_far() - starttime1)-time_res; /* Parents time */
    if(total_time < (double).000001) 
    {
        total_time=time_res;
        if(rec_prob < reclen)
            rec_prob = reclen;
        res_prob=1;
    }
#ifdef JTIME
    total_time=total_time-jtime;/* Remove the join time */
    if(!silent) printf("\nJoin time %10.2f\n",jtime);
#endif
    total_kilos=0;
    ptotal=0;
    min_throughput=max_throughput=min_xfer=0;
    if(!silent) printf("\n");
    for(xyz=0;xyz<num_child;xyz++){
        child_stat = (struct child_stats *)&shmaddr[xyz];
        total_kilos+=child_stat->throughput;
        ptotal+=child_stat->actual;
        if(!min_xfer)
            min_xfer=child_stat->actual;
        if(child_stat->actual < min_xfer)
            min_xfer=child_stat->actual;
        if(!min_throughput)
            min_throughput=child_stat->throughput;
        if(child_stat->throughput < min_throughput)
            min_throughput=child_stat->throughput;
        if(child_stat->throughput > max_throughput)
            max_throughput=child_stat->throughput;
        cputime += child_stat->cputime;
        /* Get the biggest walltime*/
        if (child_stat->walltime < child_stat->cputime)
            child_stat->walltime = child_stat->cputime;
        if (child_stat->walltime > walltime)
            walltime = child_stat->walltime;
    }
    avg_throughput=total_kilos/num_child;
    if(cpuutilflag)
    {
        if (cputime < cputime_res)
            cputime = 0.0;
    }
    if(cpuutilflag)
        store_times (walltime, cputime);    /* Must be Before store_dvalue(). */
    store_dvalue(total_kilos);
#ifdef NO_PRINT_LLD
    if(!silent) printf("\tChildren see throughput for %ld pwrite writers \t= %10.2f %s/sec\n", num_child, total_kilos,unit);
    if(!silent && !distributed) printf("\tParent sees throughput for %ld pwrite writers \t= %10.2f %s/sec\n", num_child, (double)(ptotal)/total_time,unit);
#else
    if(!silent) printf("\tChildren see throughput for %lld pwrite writers \t= %10.2f %s/sec\n", num_child, total_kilos,unit);
    if(!silent && !distributed) printf("\tParent sees throughput for %lld pwrite writers \t= %10.2f %s/sec\n", num_child, (double)(ptotal)/total_time,unit);
#endif
    if(!silent) printf("\tMin throughput per %s \t\t\t= %10.2f %s/sec \n", port,min_throughput,unit);
    if(!silent) printf("\tMax throughput per %s \t\t\t= %10.2f %s/sec\n", port,max_throughput,unit);
    if(!silent) printf("\tAvg throughput per %s \t\t\t= %10.2f %s/sec\n", port,avg_throughput,unit);
    if(!silent) printf("\tMin xfer \t\t\t\t\t= %10.2f %s\n", min_xfer,unit);
    /* CPU% can be > 100.0 for multiple CPUs */
    if(cpuutilflag)
    {