1.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     1.2 +++ b/es/examples/run-example	Sat Jul 10 16:21:26 2010 +0100
     1.3 @@ -0,0 +1,391 @@
     1.4 +#!/usr/bin/env python
     1.5 +#
     1.6 +# This program takes something that resembles a shell script and runs
     1.7 +# it, spitting input (commands from the script) and output into text
     1.8 +# files, for use in examples.
     1.9 +
    1.10 +import cStringIO
    1.11 +import errno
    1.12 +import getopt
    1.13 +import os
    1.14 +import pty
    1.15 +import re
    1.16 +import select
    1.17 +import shutil
    1.18 +import signal
    1.19 +import stat
    1.20 +import sys
    1.21 +import tempfile
    1.22 +import time
    1.23 +
    1.24 +tex_subs = {
    1.25 +    '\\': '\\textbackslash{}',
    1.26 +    '{': '\\{',
    1.27 +    '}': '\\}',
    1.28 +    }
    1.29 +
    1.30 +def gensubs(s):
    1.31 +    start = 0
    1.32 +    for i, c in enumerate(s):
    1.33 +        sub = tex_subs.get(c)
    1.34 +        if sub:
    1.35 +            yield s[start:i]
    1.36 +            start = i + 1
    1.37 +            yield sub
    1.38 +    yield s[start:]
    1.39 +
    1.40 +def tex_escape(s):
    1.41 +    return ''.join(gensubs(s))
    1.42 +        
    1.43 +def maybe_unlink(name):
    1.44 +    try:
    1.45 +        os.unlink(name)
    1.46 +        return True
    1.47 +    except OSError, err:
    1.48 +        if err.errno != errno.ENOENT:
    1.49 +            raise
    1.50 +    return False
    1.51 +
    1.52 +def find_path_to(program):
    1.53 +    for p in os.environ.get('PATH', os.defpath).split(os.pathsep):
    1.54 +        name = os.path.join(p, program)
    1.55 +        if os.access(name, os.X_OK):
    1.56 +            return p
    1.57 +    return None
    1.58 +        
    1.59 +class example:
    1.60 +    shell = '/usr/bin/env bash'
    1.61 +    ps1 = '__run_example_ps1__ '
    1.62 +    ps2 = '__run_example_ps2__ '
    1.63 +    pi_re = re.compile(r'#\$\s*(name|ignore):\s*(.*)$')
    1.64 +    
    1.65 +    timeout = 10
    1.66 +
    1.67 +    def __init__(self, name, verbose):
    1.68 +        self.name = name
    1.69 +        self.verbose = verbose
    1.70 +        self.poll = select.poll()
    1.71 +
    1.72 +    def parse(self):
    1.73 +        '''yield each hunk of input from the file.'''
    1.74 +        fp = open(self.name)
    1.75 +        cfp = cStringIO.StringIO()
    1.76 +        for line in fp:
    1.77 +            cfp.write(line)
    1.78 +            if not line.rstrip().endswith('\\'):
    1.79 +                yield cfp.getvalue()
    1.80 +                cfp.seek(0)
    1.81 +                cfp.truncate()
    1.82 +        
    1.83 +    def status(self, s):
    1.84 +        sys.stdout.write(s)
    1.85 +        if not s.endswith('\n'):
    1.86 +            sys.stdout.flush()
    1.87 +
    1.88 +    def send(self, s):
    1.89 +        if self.verbose:
    1.90 +            print >> sys.stderr, '>', self.debugrepr(s)
    1.91 +        while s:
    1.92 +            count = os.write(self.cfd, s)
    1.93 +            s = s[count:]
    1.94 +
    1.95 +    def debugrepr(self, s):
    1.96 +        rs = repr(s)
    1.97 +        limit = 60
    1.98 +        if len(rs) > limit:
    1.99 +            return ('%s%s ... [%d bytes]' % (rs[:limit], rs[0], len(s)))
   1.100 +        else:
   1.101 +            return rs
   1.102 +            
   1.103 +    timeout = 5
   1.104 +
   1.105 +    def read(self, hint):
   1.106 +        events = self.poll.poll(self.timeout * 1000)
   1.107 +        if not events:
   1.108 +            print >> sys.stderr, ('[%stimed out after %d seconds]' %
   1.109 +                                  (hint, self.timeout))
   1.110 +            os.kill(self.pid, signal.SIGHUP)
   1.111 +            return ''
   1.112 +        return os.read(self.cfd, 1024)
   1.113 +        
   1.114 +    def receive(self, hint):
   1.115 +        out = cStringIO.StringIO()
   1.116 +        while True:
   1.117 +            try:
   1.118 +                if self.verbose:
   1.119 +                    sys.stderr.write('< ')
   1.120 +                s = self.read(hint)
   1.121 +            except OSError, err:
   1.122 +                if err.errno == errno.EIO:
   1.123 +                    return '', ''
   1.124 +                raise
   1.125 +            if self.verbose:
   1.126 +                print >> sys.stderr, self.debugrepr(s)
   1.127 +            out.write(s)
   1.128 +            s = out.getvalue()
   1.129 +            if s.endswith(self.ps1):
   1.130 +                return self.ps1, s.replace('\r\n', '\n')[:-len(self.ps1)]
   1.131 +            if s.endswith(self.ps2):
   1.132 +                return self.ps2, s.replace('\r\n', '\n')[:-len(self.ps2)]
   1.133 +        
   1.134 +    def sendreceive(self, s, hint):
   1.135 +        self.send(s)
   1.136 +        ps, r = self.receive(hint)
   1.137 +        if r.startswith(s):
   1.138 +            r = r[len(s):]
   1.139 +        return ps, r
   1.140 +    
   1.141 +    def run(self):
   1.142 +        ofp = None
   1.143 +        basename = os.path.basename(self.name)
   1.144 +        self.status('running %s ' % basename)
   1.145 +        tmpdir = tempfile.mkdtemp(prefix=basename)
   1.146 +
   1.147 +        # remove the marker file that we tell make to use to see if
   1.148 +        # this run succeeded
   1.149 +        maybe_unlink(self.name + '.run')
   1.150 +
   1.151 +        rcfile = os.path.join(tmpdir, '.hgrc')
   1.152 +        rcfp = open(rcfile, 'w')
   1.153 +        print >> rcfp, '[ui]'
   1.154 +        print >> rcfp, "username = Bryan O'Sullivan <bos@serpentine.com>"
   1.155 +        
   1.156 +        rcfile = os.path.join(tmpdir, '.bashrc')
   1.157 +        rcfp = open(rcfile, 'w')
   1.158 +        print >> rcfp, 'PS1="%s"' % self.ps1
   1.159 +        print >> rcfp, 'PS2="%s"' % self.ps2
   1.160 +        print >> rcfp, 'unset HISTFILE'
   1.161 +        path = ['/usr/bin', '/bin']
   1.162 +        hg = find_path_to('hg')
   1.163 +        if hg and hg not in path:
   1.164 +            path.append(hg)
   1.165 +        def re_export(envar):
   1.166 +            v = os.getenv(envar)
   1.167 +            if v is not None:
   1.168 +                print >> rcfp, 'export ' + envar + '=' + v
   1.169 +        print >> rcfp, 'export PATH=' + ':'.join(path)
   1.170 +        re_export('PYTHONPATH')
   1.171 +        print >> rcfp, 'export EXAMPLE_DIR="%s"' % os.getcwd()
   1.172 +        print >> rcfp, 'export HGMERGE=merge'
   1.173 +        print >> rcfp, 'export LANG=C'
   1.174 +        print >> rcfp, 'export LC_ALL=C'
   1.175 +        print >> rcfp, 'export TZ=GMT'
   1.176 +        print >> rcfp, 'export HGRC="%s/.hgrc"' % tmpdir
   1.177 +        print >> rcfp, 'export HGRCPATH=$HGRC'
   1.178 +        print >> rcfp, 'cd %s' % tmpdir
   1.179 +        rcfp.close()
   1.180 +        sys.stdout.flush()
   1.181 +        sys.stderr.flush()
   1.182 +        self.pid, self.cfd = pty.fork()
   1.183 +        if self.pid == 0:
   1.184 +            cmdline = ['/usr/bin/env', '-i', 'bash', '--noediting',
   1.185 +                       '--noprofile', '--norc']
   1.186 +            try:
   1.187 +                os.execv(cmdline[0], cmdline)
   1.188 +            except OSError, err:
   1.189 +                print >> sys.stderr, '%s: %s' % (cmdline[0], err.strerror)
   1.190 +                sys.stderr.flush()
   1.191 +                os._exit(0)
   1.192 +        self.poll.register(self.cfd, select.POLLIN | select.POLLERR |
   1.193 +                           select.POLLHUP)
   1.194 +
   1.195 +        prompts = {
   1.196 +            '': '',
   1.197 +            self.ps1: '$',
   1.198 +            self.ps2: '>',
   1.199 +            }
   1.200 +
   1.201 +        ignore = [
   1.202 +            r'\d+:[0-9a-f]{12}', # changeset number:hash
   1.203 +            r'[0-9a-f]{40}', # long changeset hash
   1.204 +            r'[0-9a-f]{12}', # short changeset hash
   1.205 +            r'^(?:---|\+\+\+) .*', # diff header with dates
   1.206 +            r'^date:.*', # date
   1.207 +            #r'^diff -r.*', # "diff -r" is followed by hash
   1.208 +            r'^# Date \d+ \d+', # hg patch header
   1.209 +            ]
   1.210 +
   1.211 +        err = False
   1.212 +        read_hint = ''
   1.213 +
   1.214 +        try:
   1.215 +            try:
   1.216 +                # eat first prompt string from shell
   1.217 +                self.read(read_hint)
   1.218 +                # setup env and prompt
   1.219 +                ps, output = self.sendreceive('source %s\n' % rcfile,
   1.220 +                                              read_hint)
   1.221 +                for hunk in self.parse():
   1.222 +                    # is this line a processing instruction?
   1.223 +                    m = self.pi_re.match(hunk)
   1.224 +                    if m:
   1.225 +                        pi, rest = m.groups()
   1.226 +                        if pi == 'name':
   1.227 +                            self.status('.')
   1.228 +                            out = rest
   1.229 +                            if out in ('err', 'lxo', 'out', 'run', 'tmp'):
   1.230 +                                print >> sys.stderr, ('%s: illegal section '
   1.231 +                                                      'name %r' %
   1.232 +                                                      (self.name, out))
   1.233 +                                return 1
   1.234 +                            assert os.sep not in out
   1.235 +                            if ofp is not None:
   1.236 +                                ofp.close()
   1.237 +                                err |= self.rename_output(ofp_basename, ignore)
   1.238 +                            if out:
   1.239 +                                ofp_basename = '%s.%s' % (self.name, out)
   1.240 +                                read_hint = ofp_basename + ' '
   1.241 +                                ofp = open(ofp_basename + '.tmp', 'w')
   1.242 +                            else:
   1.243 +                                ofp = None
   1.244 +                        elif pi == 'ignore':
   1.245 +                            ignore.append(rest)
   1.246 +                    elif hunk.strip():
   1.247 +                        # it's something we should execute
   1.248 +                        newps, output = self.sendreceive(hunk, read_hint)
   1.249 +                        if not ofp:
   1.250 +                            continue
   1.251 +                        # first, print the command we ran
   1.252 +                        if not hunk.startswith('#'):
   1.253 +                            nl = hunk.endswith('\n')
   1.254 +                            hunk = ('%s \\textbf{%s}' %
   1.255 +                                    (prompts[ps],
   1.256 +                                     tex_escape(hunk.rstrip('\n'))))
   1.257 +                            if nl: hunk += '\n'
   1.258 +                        ofp.write(hunk)
   1.259 +                        # then its output
   1.260 +                        ofp.write(tex_escape(output))
   1.261 +                    ps = newps
   1.262 +                self.status('\n')
   1.263 +            except:
   1.264 +                print >> sys.stderr, '(killed)'
   1.265 +                os.kill(self.pid, signal.SIGKILL)
   1.266 +                pid, rc = os.wait()
   1.267 +                raise
   1.268 +            else:
   1.269 +                try:
   1.270 +                    ps, output = self.sendreceive('exit\n', read_hint)
   1.271 +                    if ofp is not None:
   1.272 +                        ofp.write(output)
   1.273 +                        ofp.close()
   1.274 +                        err |= self.rename_output(ofp_basename, ignore)
   1.275 +                    os.close(self.cfd)
   1.276 +                except IOError:
   1.277 +                    pass
   1.278 +                os.kill(self.pid, signal.SIGTERM)
   1.279 +                pid, rc = os.wait()
   1.280 +                err = err or rc
   1.281 +                if err:
   1.282 +                    if os.WIFEXITED(rc):
   1.283 +                        print >> sys.stderr, '(exit %s)' % os.WEXITSTATUS(rc)
   1.284 +                    elif os.WIFSIGNALED(rc):
   1.285 +                        print >> sys.stderr, '(signal %s)' % os.WTERMSIG(rc)
   1.286 +                else:
   1.287 +                    open(self.name + '.run', 'w')
   1.288 +                return err
   1.289 +        finally:
   1.290 +            shutil.rmtree(tmpdir)
   1.291 +
   1.292 +    def rename_output(self, base, ignore):
   1.293 +        mangle_re = re.compile('(?:' + '|'.join(ignore) + ')')
   1.294 +        def mangle(s):
   1.295 +            return mangle_re.sub('', s)
   1.296 +        def matchfp(fp1, fp2):
   1.297 +            while True:
   1.298 +                s1 = mangle(fp1.readline())
   1.299 +                s2 = mangle(fp2.readline())
   1.300 +                if cmp(s1, s2):
   1.301 +                    break
   1.302 +                if not s1:
   1.303 +                    return True
   1.304 +            return False
   1.305 +
   1.306 +        oldname = base + '.out'
   1.307 +        tmpname = base + '.tmp'
   1.308 +        errname = base + '.err'
   1.309 +        errfp = open(errname, 'w+')
   1.310 +        for line in open(tmpname):
   1.311 +            errfp.write(mangle_re.sub('', line))
   1.312 +        os.rename(tmpname, base + '.lxo')
   1.313 +        errfp.seek(0)
   1.314 +        try:
   1.315 +            oldfp = open(oldname)
   1.316 +        except IOError, err:
   1.317 +            if err.errno != errno.ENOENT:
   1.318 +                raise
   1.319 +            os.rename(errname, oldname)
   1.320 +            return False
   1.321 +        if matchfp(oldfp, errfp):
   1.322 +            os.unlink(errname)
   1.323 +            return False
   1.324 +        else:
   1.325 +            print >> sys.stderr, '\nOutput of %s has changed!' % base
   1.326 +            os.system('diff -u %s %s 1>&2' % (oldname, errname))
   1.327 +            return True
   1.328 +
   1.329 +def print_help(exit, msg=None):
   1.330 +    if msg:
   1.331 +        print >> sys.stderr, 'Error:', msg
   1.332 +    print >> sys.stderr, 'Usage: run-example [options] [test...]'
   1.333 +    print >> sys.stderr, 'Options:'
   1.334 +    print >> sys.stderr, '  -a --all       run all tests in this directory'
   1.335 +    print >> sys.stderr, '  -h --help      print this help message'
   1.336 +    print >> sys.stderr, '  -v --verbose   display extra debug output'
   1.337 +    sys.exit(exit)
   1.338 +
   1.339 +def main(path='.'):
   1.340 +    opts, args = getopt.getopt(sys.argv[1:], '?ahv',
   1.341 +                               ['all', 'help', 'verbose'])
   1.342 +    verbose = False
   1.343 +    run_all = False
   1.344 +    for o, a in opts:
   1.345 +        if o in ('-h', '-?', '--help'):
   1.346 +            print_help(0)
   1.347 +        if o in ('-a', '--all'):
   1.348 +            run_all = True
   1.349 +        if o in ('-v', '--verbose'):
   1.350 +            verbose = True
   1.351 +    errs = 0
   1.352 +    if args:
   1.353 +        for a in args:
   1.354 +            try:
   1.355 +                st = os.lstat(a)
   1.356 +            except OSError, err:
   1.357 +                print >> sys.stderr, '%s: %s' % (a, err.strerror)
   1.358 +                errs += 1
   1.359 +                continue
   1.360 +            if stat.S_ISREG(st.st_mode) and st.st_mode & 0111:
   1.361 +                if example(a, verbose).run():
   1.362 +                    errs += 1
   1.363 +            else:
   1.364 +                print >> sys.stderr, '%s: not a file, or not executable' % a
   1.365 +                errs += 1
   1.366 +    elif run_all:
   1.367 +        names = os.listdir(path)
   1.368 +        names.sort()
   1.369 +        for name in names:
   1.370 +            if name == 'run-example' or name.startswith('.'): continue
   1.371 +            if name.endswith('.out') or name.endswith('~'): continue
   1.372 +            if name.endswith('.run'): continue
   1.373 +            pathname = os.path.join(path, name)
   1.374 +            try:
   1.375 +                st = os.lstat(pathname)
   1.376 +            except OSError, err:
   1.377 +                # could be an output file that was removed while we ran
   1.378 +                if err.errno != errno.ENOENT:
   1.379 +                    raise
   1.380 +                continue
   1.381 +            if stat.S_ISREG(st.st_mode) and st.st_mode & 0111:
   1.382 +                if example(pathname, verbose).run():
   1.383 +                    errs += 1
   1.384 +        print >> open(os.path.join(path, '.run'), 'w'), time.asctime()
   1.385 +    else:
   1.386 +        print_help(1, msg='no test names given, and --all not provided')
   1.387 +    return errs
   1.388 +
   1.389 +if __name__ == '__main__':
   1.390 +    try:
   1.391 +        sys.exit(main())
   1.392 +    except KeyboardInterrupt:
   1.393 +        print >> sys.stderr, 'interrupted!'
   1.394 +        sys.exit(1)