bos@67: #!/usr/bin/env python
bos@4: #
bos@4: # This program takes something that resembles a shell script and runs
bos@4: # it, spitting input (commands from the script) and output into text
bos@4: # files, for use in examples.
bos@3: 
bos@3: import cStringIO
bos@73: import errno
bos@78: import getopt
bos@3: import os
bos@3: import pty
bos@3: import re
bos@79: import select
bos@4: import shutil
bos@6: import signal
bos@36: import stat
bos@3: import sys
bos@4: import tempfile
bos@4: import time
bos@3: 
bos@564: xml_subs = {
bos@564:     '<': '&lt;',
bos@564:     '>': '&gt;',
bos@564:     '&': '&amp;',
bos@77:     }
bos@77: 
bos@77: def gensubs(s):
bos@77:     start = 0
bos@77:     for i, c in enumerate(s):
bos@564:         sub = xml_subs.get(c)
bos@77:         if sub:
bos@77:             yield s[start:i]
bos@77:             start = i + 1
bos@77:             yield sub
bos@77:     yield s[start:]
bos@77: 
bos@564: def xml_escape(s):
bos@77:     return ''.join(gensubs(s))
bos@4:         
bos@137: def maybe_unlink(name):
bos@137:     try:
bos@137:         os.unlink(name)
bos@137:         return True
bos@137:     except OSError, err:
bos@137:         if err.errno != errno.ENOENT:
bos@137:             raise
bos@137:     return False
bos@137: 
bos@172: def find_path_to(program):
bos@172:     for p in os.environ.get('PATH', os.defpath).split(os.pathsep):
bos@172:         name = os.path.join(p, program)
bos@172:         if os.access(name, os.X_OK):
bos@172:             return p
bos@172:     return None
bos@172:         
bos@564: def result_name(name):
bos@579:     return os.path.join('results', name.replace(os.sep, '-'))
bos@564: 
bos@3: class example:
bos@579:     entities = dict.fromkeys(l.rstrip() for l in open('auto-snippets.xml'))
bos@579: 
bos@579:     def __init__(self, name, verbose, keep_change):
bos@579:         self.name = name
bos@579:         self.verbose = verbose
bos@579:         self.keep_change = keep_change
bos@579: 
bos@579:     def rename_output(self, base, ignore=[]):
bos@579:         mangle_re = re.compile('(?:' + '|'.join(ignore) + ')')
bos@579:         def mangle(s):
bos@579:             return mangle_re.sub('', s)
bos@579:         def matchfp(fp1, fp2):
bos@579:             while True:
bos@579:                 s1 = mangle(fp1.readline())
bos@579:                 s2 = mangle(fp2.readline())
bos@579:                 if cmp(s1, s2):
bos@579:                     break
bos@579:                 if not s1:
bos@579:                     return True
bos@579:             return False
bos@579: 
bos@579:         oldname = result_name(base + '.out')
bos@579:         tmpname = result_name(base + '.tmp')
bos@579:         errname = result_name(base + '.err')
bos@579:         errfp = open(errname, 'w+')
bos@579:         for line in open(tmpname):
bos@579:             errfp.write(mangle_re.sub('', line))
bos@579:         os.rename(tmpname, result_name(base + '.lxo'))
bos@579:         errfp.seek(0)
bos@579:         try:
bos@579:             oldfp = open(oldname)
bos@579:         except IOError, err:
bos@579:             if err.errno != errno.ENOENT:
bos@579:                 raise
bos@579:             os.rename(errname, oldname)
bos@579:             return False
bos@579:         if matchfp(oldfp, errfp):
bos@579:             os.unlink(errname)
bos@579:             return False
bos@579:         else:
bos@579:             print >> sys.stderr, '\nOutput of %s has changed!' % baseq
bos@579:             if self.keep_change:
bos@579:                 os.rename(errname, oldname)
bos@579:                 return False
bos@579:             else:
bos@579:                 os.system('diff -u %s %s 1>&2' % (oldname, errname))
bos@579:             return True
bos@579: 
bos@579: def wopen(name):
bos@579:     path = os.path.dirname(name)
bos@579:     if path:
bos@579:         try:
bos@579:             os.makedirs(path)
bos@579:         except OSError, err:
bos@579:             if err.errno != errno.EEXIST:
bos@579:                 raise
bos@579:     return open(name, 'w')
bos@579: 
bos@579: class static_example(example):
bos@579:     def run(self):
bos@579:         s = open(self.name).read().rstrip()
bos@579:         s = s.replace('&', '&amp;').replace('<', '&lt;').replace('>', '&gt;')
bos@579:         ofp = wopen(result_name(self.name + '.tmp'))
bos@579:         ofp.write('<programlisting>')
bos@579:         ofp.write(s)
bos@579:         ofp.write('</programlisting>\n')
bos@579:         ofp.close()
bos@579:         self.rename_output(self.name)
bos@579:         norm = self.name.replace(os.sep, '-')
bos@579:         example.entities[
bos@579:             '<!ENTITY %s SYSTEM "results/%s.out">' % (norm, norm)] = 1
bos@579: 
bos@579: 
bos@579: class shell_example(example):
bos@70:     shell = '/usr/bin/env bash'
bos@103:     ps1 = '__run_example_ps1__ '
bos@103:     ps2 = '__run_example_ps2__ '
bos@168:     pi_re = re.compile(r'#\$\s*(name|ignore):\s*(.*)$')
bos@4:     
bos@192:     timeout = 10
bos@79: 
bos@545:     def __init__(self, name, verbose, keep_change):
bos@579:         example.__init__(self, name, verbose, keep_change)
bos@79:         self.poll = select.poll()
bos@3: 
bos@3:     def parse(self):
bos@4:         '''yield each hunk of input from the file.'''
bos@3:         fp = open(self.name)
bos@3:         cfp = cStringIO.StringIO()
bos@3:         for line in fp:
bos@3:             cfp.write(line)
bos@3:             if not line.rstrip().endswith('\\'):
bos@3:                 yield cfp.getvalue()
bos@3:                 cfp.seek(0)
bos@3:                 cfp.truncate()
bos@3:         
bos@3:     def status(self, s):
bos@3:         sys.stdout.write(s)
bos@3:         if not s.endswith('\n'):
bos@3:             sys.stdout.flush()
bos@3: 
bos@6:     def send(self, s):
bos@78:         if self.verbose:
bos@78:             print >> sys.stderr, '>', self.debugrepr(s)
bos@73:         while s:
bos@73:             count = os.write(self.cfd, s)
bos@73:             s = s[count:]
bos@6: 
bos@78:     def debugrepr(self, s):
bos@78:         rs = repr(s)
bos@78:         limit = 60
bos@78:         if len(rs) > limit:
bos@78:             return ('%s%s ... [%d bytes]' % (rs[:limit], rs[0], len(s)))
bos@78:         else:
bos@78:             return rs
bos@78:             
bos@79:     timeout = 5
bos@79: 
bos@173:     def read(self, hint):
bos@79:         events = self.poll.poll(self.timeout * 1000)
bos@79:         if not events:
bos@173:             print >> sys.stderr, ('[%stimed out after %d seconds]' %
bos@173:                                   (hint, self.timeout))
bos@79:             os.kill(self.pid, signal.SIGHUP)
bos@79:             return ''
bos@79:         return os.read(self.cfd, 1024)
bos@79:         
bos@173:     def receive(self, hint):
bos@6:         out = cStringIO.StringIO()
bos@4:         while True:
bos@73:             try:
bos@78:                 if self.verbose:
bos@78:                     sys.stderr.write('< ')
bos@173:                 s = self.read(hint)
bos@73:             except OSError, err:
bos@73:                 if err.errno == errno.EIO:
bos@103:                     return '', ''
bos@73:                 raise
bos@78:             if self.verbose:
bos@78:                 print >> sys.stderr, self.debugrepr(s)
bos@6:             out.write(s)
bos@73:             s = out.getvalue()
bos@103:             if s.endswith(self.ps1):
bos@103:                 return self.ps1, s.replace('\r\n', '\n')[:-len(self.ps1)]
bos@103:             if s.endswith(self.ps2):
bos@103:                 return self.ps2, s.replace('\r\n', '\n')[:-len(self.ps2)]
bos@4:         
bos@173:     def sendreceive(self, s, hint):
bos@6:         self.send(s)
bos@173:         ps, r = self.receive(hint)
bos@6:         if r.startswith(s):
bos@6:             r = r[len(s):]
bos@103:         return ps, r
bos@6:     
bos@3:     def run(self):
bos@3:         ofp = None
bos@4:         basename = os.path.basename(self.name)
bos@4:         self.status('running %s ' % basename)
bos@4:         tmpdir = tempfile.mkdtemp(prefix=basename)
bos@78: 
bos@136:         # remove the marker file that we tell make to use to see if
bos@136:         # this run succeeded
bos@137:         maybe_unlink(self.name + '.run')
bos@136: 
bos@78:         rcfile = os.path.join(tmpdir, '.hgrc')
bos@579:         rcfp = wopen(rcfile)
bos@78:         print >> rcfp, '[ui]'
bos@78:         print >> rcfp, "username = Bryan O'Sullivan <bos@serpentine.com>"
bos@78:         
bos@6:         rcfile = os.path.join(tmpdir, '.bashrc')
bos@579:         rcfp = wopen(rcfile)
bos@103:         print >> rcfp, 'PS1="%s"' % self.ps1
bos@103:         print >> rcfp, 'PS2="%s"' % self.ps2
bos@6:         print >> rcfp, 'unset HISTFILE'
bos@172:         path = ['/usr/bin', '/bin']
bos@172:         hg = find_path_to('hg')
bos@172:         if hg and hg not in path:
bos@172:             path.append(hg)
bos@172:         def re_export(envar):
bos@172:             v = os.getenv(envar)
bos@172:             if v is not None:
bos@172:                 print >> rcfp, 'export ' + envar + '=' + v
bos@172:         print >> rcfp, 'export PATH=' + ':'.join(path)
bos@172:         re_export('PYTHONPATH')
bos@19:         print >> rcfp, 'export EXAMPLE_DIR="%s"' % os.getcwd()
bos@124:         print >> rcfp, 'export HGMERGE=merge'
bos@6:         print >> rcfp, 'export LANG=C'
bos@6:         print >> rcfp, 'export LC_ALL=C'
bos@6:         print >> rcfp, 'export TZ=GMT'
bos@6:         print >> rcfp, 'export HGRC="%s/.hgrc"' % tmpdir
bos@6:         print >> rcfp, 'export HGRCPATH=$HGRC'
bos@6:         print >> rcfp, 'cd %s' % tmpdir
bos@6:         rcfp.close()
bos@68:         sys.stdout.flush()
bos@68:         sys.stderr.flush()
bos@79:         self.pid, self.cfd = pty.fork()
bos@79:         if self.pid == 0:
bos@172:             cmdline = ['/usr/bin/env', '-i', 'bash', '--noediting',
bos@172:                        '--noprofile', '--norc']
bos@68:             try:
bos@68:                 os.execv(cmdline[0], cmdline)
bos@68:             except OSError, err:
bos@68:                 print >> sys.stderr, '%s: %s' % (cmdline[0], err.strerror)
bos@68:                 sys.stderr.flush()
bos@68:                 os._exit(0)
bos@79:         self.poll.register(self.cfd, select.POLLIN | select.POLLERR |
bos@79:                            select.POLLHUP)
bos@103: 
bos@103:         prompts = {
bos@103:             '': '',
bos@103:             self.ps1: '$',
bos@103:             self.ps2: '>',
bos@103:             }
bos@103: 
bos@137:         ignore = [
bos@137:             r'\d+:[0-9a-f]{12}', # changeset number:hash
bos@141:             r'[0-9a-f]{40}', # long changeset hash
bos@141:             r'[0-9a-f]{12}', # short changeset hash
bos@137:             r'^(?:---|\+\+\+) .*', # diff header with dates
bos@137:             r'^date:.*', # date
bos@141:             #r'^diff -r.*', # "diff -r" is followed by hash
bos@138:             r'^# Date \d+ \d+', # hg patch header
bos@137:             ]
bos@137: 
bos@138:         err = False
bos@173:         read_hint = ''
bos@138: 
bos@4:         try:
bos@71:             try:
bos@73:                 # eat first prompt string from shell
bos@173:                 self.read(read_hint)
bos@71:                 # setup env and prompt
bos@173:                 ps, output = self.sendreceive('source %s\n' % rcfile,
bos@173:                                               read_hint)
bos@71:                 for hunk in self.parse():
bos@71:                     # is this line a processing instruction?
bos@71:                     m = self.pi_re.match(hunk)
bos@71:                     if m:
bos@71:                         pi, rest = m.groups()
bos@71:                         if pi == 'name':
bos@71:                             self.status('.')
bos@71:                             out = rest
bos@155:                             if out in ('err', 'lxo', 'out', 'run', 'tmp'):
bos@155:                                 print >> sys.stderr, ('%s: illegal section '
bos@155:                                                       'name %r' %
bos@155:                                                       (self.name, out))
bos@155:                                 return 1
bos@71:                             assert os.sep not in out
bos@137:                             if ofp is not None:
bos@564:                                 ofp.write('</screen>\n')
bos@137:                                 ofp.close()
bos@160:                                 err |= self.rename_output(ofp_basename, ignore)
bos@71:                             if out:
bos@137:                                 ofp_basename = '%s.%s' % (self.name, out)
bos@566:                                 norm = os.path.normpath(ofp_basename)
bos@566:                                 example.entities[
bos@569:                                     '<!ENTITY interaction.%s '
bos@566:                                     'SYSTEM "results/%s.out">'
bos@566:                                     % (norm, norm)] = 1
bos@173:                                 read_hint = ofp_basename + ' '
bos@579:                                 ofp = wopen(result_name(ofp_basename + '.tmp'))
bos@564:                                 ofp.write('<screen>')
bos@71:                             else:
bos@71:                                 ofp = None
bos@137:                         elif pi == 'ignore':
bos@137:                             ignore.append(rest)
bos@71:                     elif hunk.strip():
bos@71:                         # it's something we should execute
bos@173:                         newps, output = self.sendreceive(hunk, read_hint)
bos@168:                         if not ofp:
bos@71:                             continue
bos@71:                         # first, print the command we ran
bos@71:                         if not hunk.startswith('#'):
bos@71:                             nl = hunk.endswith('\n')
bos@564:                             hunk = ('<prompt>%s</prompt> <userinput>%s</userinput>' %
bos@103:                                     (prompts[ps],
bos@564:                                      xml_escape(hunk.rstrip('\n'))))
bos@71:                             if nl: hunk += '\n'
bos@71:                         ofp.write(hunk)
bos@71:                         # then its output
bos@564:                         ofp.write(xml_escape(output))
bos@103:                     ps = newps
bos@71:                 self.status('\n')
bos@71:             except:
bos@72:                 print >> sys.stderr, '(killed)'
bos@79:                 os.kill(self.pid, signal.SIGKILL)
bos@72:                 pid, rc = os.wait()
bos@71:                 raise
bos@72:             else:
bos@71:                 try:
bos@173:                     ps, output = self.sendreceive('exit\n', read_hint)
bos@138:                     if ofp is not None:
bos@71:                         ofp.write(output)
bos@564:                         ofp.write('</screen>\n')
bos@138:                         ofp.close()
bos@160:                         err |= self.rename_output(ofp_basename, ignore)
bos@73:                     os.close(self.cfd)
bos@71:                 except IOError:
bos@71:                     pass
bos@79:                 os.kill(self.pid, signal.SIGTERM)
bos@72:                 pid, rc = os.wait()
bos@160:                 err = err or rc
bos@160:                 if err:
bos@72:                     if os.WIFEXITED(rc):
bos@72:                         print >> sys.stderr, '(exit %s)' % os.WEXITSTATUS(rc)
bos@72:                     elif os.WIFSIGNALED(rc):
bos@72:                         print >> sys.stderr, '(signal %s)' % os.WTERMSIG(rc)
bos@136:                 else:
bos@579:                     wopen(result_name(self.name + '.run'))
bos@160:                 return err
bos@72:         finally:
bos@4:             shutil.rmtree(tmpdir)
bos@3: 
bos@258: def print_help(exit, msg=None):
bos@258:     if msg:
bos@258:         print >> sys.stderr, 'Error:', msg
bos@258:     print >> sys.stderr, 'Usage: run-example [options] [test...]'
bos@258:     print >> sys.stderr, 'Options:'
bos@565:     print >> sys.stderr, '  -a --all       run all examples in this directory'
bos@258:     print >> sys.stderr, '  -h --help      print this help message'
bos@545:     print >> sys.stderr, '     --help      keep new output as desired output'
bos@258:     print >> sys.stderr, '  -v --verbose   display extra debug output'
bos@258:     sys.exit(exit)
bos@258: 
bos@3: def main(path='.'):
bos@566:     if os.path.realpath(path).split(os.sep)[-1] != 'examples':
bos@566:         print >> sys.stderr, 'Not being run from the examples directory!'
bos@566:         sys.exit(1)
bos@566: 
bos@258:     opts, args = getopt.getopt(sys.argv[1:], '?ahv',
bos@545:                                ['all', 'help', 'keep', 'verbose'])
bos@78:     verbose = False
bos@258:     run_all = False
bos@545:     keep_change = False
bos@566: 
bos@78:     for o, a in opts:
bos@258:         if o in ('-h', '-?', '--help'):
bos@258:             print_help(0)
bos@258:         if o in ('-a', '--all'):
bos@258:             run_all = True
bos@545:         if o in ('--keep',):
bos@545:             keep_change = True
bos@78:         if o in ('-v', '--verbose'):
bos@78:             verbose = True
bos@71:     errs = 0
bos@3:     if args:
bos@3:         for a in args:
bos@75:             try:
bos@75:                 st = os.lstat(a)
bos@75:             except OSError, err:
bos@75:                 print >> sys.stderr, '%s: %s' % (a, err.strerror)
bos@75:                 errs += 1
bos@75:                 continue
bos@579:             if stat.S_ISREG(st.st_mode):
bos@579:                 if st.st_mode & 0111:
bos@579:                     if shell_example(a, verbose, keep_change).run():
bos@579:                         errs += 1
bos@579:                 elif a.endswith('.lst'):
bos@579:                     static_example(a, verbose, keep_change).run()
bos@75:             else:
bos@75:                 print >> sys.stderr, '%s: not a file, or not executable' % a
bos@71:                 errs += 1
bos@258:     elif run_all:
bos@579:         names = glob.glob("*") + glob.glob("app*/*") + glob.glob("ch*/*")
bos@258:         names.sort()
bos@258:         for name in names:
bos@579:             if name == 'run-example' or name.endswith('~'): continue
bos@258:             pathname = os.path.join(path, name)
bos@258:             try:
bos@258:                 st = os.lstat(pathname)
bos@258:             except OSError, err:
bos@258:                 # could be an output file that was removed while we ran
bos@258:                 if err.errno != errno.ENOENT:
bos@258:                     raise
bos@258:                 continue
bos@579:             if stat.S_ISREG(st.st_mode):
bos@579:                 if st.st_mode & 0111:
bos@579:                     if shell_example(pathname, verbose, keep_change).run():
bos@579:                         errs += 1
bos@579:                 elif pathname.endswith('.lst'):
bos@579:                     static_example(pathname, verbose, keep_change).run()
bos@579:         print >> wopen(os.path.join(path, '.run')), time.asctime()
bos@258:     else:
bos@258:         print_help(1, msg='no test names given, and --all not provided')
bos@566: 
bos@579:     fp = wopen('auto-snippets.xml')
bos@566:     for key in sorted(example.entities.iterkeys()):
bos@566:         print >> fp, key
bos@566:     fp.close()
bos@71:     return errs
bos@3: 
bos@3: if __name__ == '__main__':
bos@258:     try:
bos@258:         sys.exit(main())
bos@258:     except KeyboardInterrupt:
bos@258:         print >> sys.stderr, 'interrupted!'
bos@258:         sys.exit(1)