#!/usr/bin/env python # You (a) want to use larger buffers, and (b) a program which uses # O_DIRECT for i/o. I had a news server which was running 28 aps until I # started using dd, then it dropped to 3 aps. Usinf O_DIRECT there is no # measurable slowdown (and no buffer contention). # Sun Jan 9 11:21:38 PST 2005 # Code released under the terms of GPLv2: http://www.gnu.org/copyleft/gpl.html import select import sys import string import os import stat import time import signal import fcntl signal.signal(signal.SIGPIPE, signal.SIG_DFL) def one_decimal(n): percent = (int(n*10.0)) / 10.0 return percent def compute_percent(n): percent = (int(n*1000.0)) / 10.0 return percent def usage(): sys.stderr.write('usage: '+sys.argv[0]+' [-nt] [-t] [-e n] [-g /fs] [-S size] blocksize timeout\n') sys.stderr.write('-s n\tsleep n (fractions of) seconds between blocks\n') sys.stderr.write('-S size\ttransfer no more than "size" bytes\n') sys.stderr.write('-b\tbeep three times when transfer completes on stderr\n') sys.stderr.write('-t\treports throughput (default now)\n') sys.stderr.write('-nt\tdoes not report throughput\n') sys.stderr.write('-e n\tsays we\'re estimating that the filesize is n kilobytes\n') sys.stderr.write('-g /fs\tguess the number of Kilobytes in /fs. Can be repeated.\n') sys.stderr.write('-m metacompress mode :)\n') sys.stderr.write('-M metacompress mode with disk-based compression(s) :)\n') sys.stderr.write('\t(not implemented yet!)\n') sys.stderr.write('-f give a final report only, not a running report\n') sys.stderr.write('-np do not pad the final block to a full length block\n') sys.stderr.write('blocksize\tin bytes\n') sys.stderr.write('timeout\tin seconds (floating point)\n') sys.exit(0) def tm(seconds): list = [(60,'s'), (60,'m'), (24,'h'), (30, 'd'), (12, 'M'), (10, 'y'), (10, 'd')] temp = int(seconds) s = '' #s = s + 'initial: ->%d<- ' % temp for tuple in list: #sys.stderr.write(str(tuple)) if temp <= 0: break #t = temp % tuple[0] quotient = int(temp / tuple[0]) remainder = temp - quotient * tuple[0] temp = quotient s += ' '+str(remainder)+tuple[1] return s def report(size,dt,do_percent,estimate,prevmin,total_file_len): bitsps = float(size * 8)/float(dt) units=['bits/s','Kbits/s','Mbits/s','Gbits/s','Tbits/s','Pbits/s'] i = 0 scaled = bitsps while 1: #sys.stderr.write(units[i]+'\n') if scaled > 1024: scaled = scaled / 1024.0 i = i + 1 else: break if i == len(units) - 1: break curmin = one_decimal(dt/60) if do_percent and total_file_len != -1: percent = compute_percent(float(size) / total_file_len) sys.stderr.write(str(percent)+'% ') bytes_left = total_file_len - size #sys.stderr.write('->bytes left %d <-' % bytes_left) seconds_left = bytes_left / (bitsps / 8) sys.stderr.write(tm(seconds_left)+') ') if estimate != 0L: percent = compute_percent(float(size) / estimate) sys.stderr.write('(estimate: '+str(percent)+'% ') bytes_left = estimate - size #sys.stderr.write('->bytes left %d <-' % bytes_left) seconds_left = int((estimate - size) / (bitsps / 8)) sys.stderr.write(tm(seconds_left)) if estimate: sys.stderr.write(')') sys.stderr.write(' ') sys.stderr.write('Kbytes: '+str(one_decimal(size/1024L))+\ ' '+units[i]+': '+\ str(one_decimal(scaled))+\ ' Gbytes/hr: '+\ str(one_decimal(((size * 60*60) / (dt * 1024*1024*1024))))+\ ' min: '+str(curmin)) sys.stderr.write(' \r') if prevmin != int(curmin): sys.stderr.write('\n') prevmin = int(curmin) return prevmin def main(): verbose = 0 estimate=0L tp = 1 beep = 0 duration = 0.0 #sys.stderr.write(sys.argv[1]+'\n') # this really should use getopt! t0 = time.time() guess = 0 guess_fses=[] final=0 total_file_len = -1 padding=1 while sys.argv[1:]: if sys.argv[1] == '-s': # thruput option duration = string.atof(sys.argv[2]) del sys.argv[1] del sys.argv[1] if sys.argv[1] == '-b': # thruput option del sys.argv[1] beep = 1 elif sys.argv[1] == '-t': # thruput option del sys.argv[1] size = 0L elif sys.argv[1] == '-f': # thruput option del sys.argv[1] final = 1 size = 0L elif sys.argv[1] == '-g': if estimate: sys.stderr.write('-e conflicts with -g\n') usage() # thruput option guess = 1 guess_fses.append(sys.argv[2]) del sys.argv[1] del sys.argv[1] elif sys.argv[1] == '-e': if guess: sys.stderr.write('-e conflicts with -g\n') usage() # estimate of total size option estimate = string.atol(sys.argv[2])*1024 del sys.argv[1] del sys.argv[1] #sys.stderr.write('estimate of size in bytes is %d\n' % estimate) size = 0L elif sys.argv[1] == '-nt': tp = 0 elif sys.argv[1] == '-S' and sys.argv[2:]: maximum_transfer_size=string.atol(sys.argv[2]) del sys.argv[1] del sys.argv[1] elif sys.argv[1] == '-np': padding=0 #sys.stderr.write('Got -np\n') del sys.argv[1] elif sys.argv[1][0] == '-': usage() else: break if sys.argv[1:] and not sys.argv[3:]: blocksize=string.atoi(sys.argv[1]) timeout=string.atof(sys.argv[2]) else: usage() #sys.stderr.write('padding is %d\n' % padding) do_percent=0 try: os.lseek(0,0,0) except: sys.stderr.write('stdin not seekable - will not give percentage\n') else: sb=os.fstat(0) total_file_len=sb[stat.ST_SIZE] if total_file_len != 0: sys.stderr.write('stdin seekable, and file len nonzero - will give percentage. File length is %d Kilobytes\n' % (total_file_len/1024)) do_percent=1 if estimate != 0L: sys.stderr.write('Size estimate ignored\n') estimate = 0L else: sys.stderr.write('stdin seems seekable, but file length is 0 - no exact percentages\n') if estimate != 0L: sys.stderr.write('Estimated filetransfer size is %ld bytes\n' % estimate) sys.stderr.write('Estimated percentages will only be as accurate as your size estimate\n') #sys.stderr.write('tp is'+str(tp)+'\n') # if hasattr(os, 'O_DIRECT'): # try: # flags = fcntl.fcntl(sys.stdin.fileno(), fcntl.F_GETFL) # flags |= os.O_DIRECT # fcntl.fcntl(sys.stdin.fileno(), fcntl.F_SETFL, flags) # except: # sys.stderr.write('Setting O_DIRECT on stdin attempted but failed\n') # else: # sys.stderr.write('Setting O_DIRECT on stdin succeeded :)\n') bigblock='' #sys.stderr.write(str(tp)+'\n') prevmin = 0 size = 0L oserror=0 while 1: retval=select.select([0],[],[],timeout) if len(retval[0]) == 0: # timeout if verbose: sys.stderr.write('got timeout\n') finish(blocksize,bigblock,tp,padding) break elif len(retval[0]) == 1 and retval[0][0] == 0: # got input if verbose: sys.stderr.write('select says ready to read\n') block = os.read(0,blocksize) if not block: # eof if verbose: sys.stderr.write('got eof\n') if final: prevmin = report(size,dt,do_percent,estimate,prevmin,total_file_len) finish(blocksize,bigblock,tp,padding) break if len(block) < blocksize and verbose: sys.stderr.write('got short block\n') bigblock=bigblock+block if len(bigblock) >= blocksize: time.sleep(duration) try: os.write(1,bigblock[0:blocksize]) except OSError,value: oserror=1 if oserror: break if tp or estimate or final: # size is in bytes size = size + blocksize dt = time.time() - t0 if not final: prevmin = report(size,dt,do_percent,estimate,prevmin,total_file_len) bigblock=bigblock[blocksize:] else: sys.stderr.write('Something bizarre happened\n') sys.stderr.write(str(retval)+'\n') if oserror: sys.stderr.write('\n'+sys.argv[0]+' '+str(value)+'\n') if beep: for i in range(3): time.sleep(1) sys.stderr.write('\007') def finish(blocksize,block,tp,padding): # write what's left, and pad to an integral block size if len(block) > blocksize: sys.stderr.write('internal error: len(block) > blocksize\n') sys.exit(1) remainder = blocksize - len(block) oserror = 0 if padding: try: os.write(1,block + (chr(0) * remainder)) except OSError,value: oserror=1 else: try: os.write(1,block) except OSError,value: oserror=1 if tp: sys.stderr.write('\n') if oserror: sys.stderr.write(sys.argv[0]+' '+str(value)+'\n') #os.write(1,block+ (chr(0) * remainder)) main() sys.exit(0)