aboutsummaryrefslogtreecommitdiff
blob: 7bb96c85bdf596ff46d78111f04aa829b4dfbd3c (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
from __future__ import print_function
import sys, shutil, os, tempfile, hashlib, collections
import sysconfig
from os.path import join

try:
    import _multiprocessing
except ImportError:
    # The only function we need from multiprocessing is cpu_count(), which is
    # written in the pure Python part of multiprocessing. We can make it work
    # in environments that don't have _multiprocessing by adding an empty
    # module in place of _multiprocessing.
    import types
    sys.modules['_multiprocessing'] = types.ModuleType('fake _multiprocessing')
import multiprocessing

# do not use the long-running runsubprocess._run here, since building some of
# the extensions enable importing them later
os.environ['PYPY_DONT_RUN_SUBPROCESS'] = '1'

class MissingDependenciesError(Exception):
    pass


cffi_build_scripts = collections.OrderedDict({
    ("_ctypes._ctypes_cffi",
     "_ctypes/_ctypes_build.py" if sys.platform == 'darwin' else None),
    ("_ssl", "_ssl_build.py"),
    ("sqlite3", "_sqlite3_build.py"),
    ("audioop", "_audioop_build.py"),
    ("_tkinter", "_tkinter/tklib_build.py"),
    ("curses", "_curses_build.py" if sys.platform != "win32" else None),
    ("syslog", "_syslog_build.py" if sys.platform != "win32" else None),
    ("gdbm", "_gdbm_build.py"  if sys.platform != "win32" else None),
    ("grp", "_pwdgrp_build.py" if sys.platform != "win32" else None),
    ("resource", "_resource_build.py" if sys.platform != "win32" else None),
    ("xx", None),    # for testing: 'None' should be completely ignored
    })

# for distribution, we may want to fetch dependencies not provided by
# the OS, such as a recent openssl/libressl.
curdir = os.path.abspath(os.path.dirname(__file__))
deps_destdir = os.path.join(curdir, 'dest')
configure_args = ['./configure',
            '--prefix=/usr',
            '--disable-shared',
            '--enable-silent-rules',
            '--disable-dependency-tracking',
        ]
# please note the deliberate use of a mirror site: we can't use HTTPS
# without an _ssl module, but the OpenSSL download site redirect HTTP
# to HTTPS
cffi_dependencies = {
    'lzma': ('http://distfiles.macports.org/xz/xz-5.2.5.tar.bz2',
             '5117f930900b341493827d63aa910ff5e011e0b994197c3b71c08a20228a42df',
             [configure_args,
              ['make', '-s', '-j', str(multiprocessing.cpu_count())],
              ['make', 'install', 'DESTDIR={}/'.format(deps_destdir)],
             ]),
    '_ssl': ('http://distfiles.macports.org/openssl/openssl-1.1.1f.tar.gz',
             '186c6bfe6ecfba7a5b48c47f8a1673d0f3b0e5ba2e25602dd23b629975da3f35',
             [['./config', '--prefix=/usr', 'no-shared'],
              ['make', '-s', '-j', str(multiprocessing.cpu_count())],
              ['make', 'install', 'DESTDIR={}/'.format(deps_destdir)],
             ]),
    # this does not compile on the buildbot, linker is missing '_history_list'
    'gdbm': ('http://distfiles.macports.org/gdbm/gdbm-1.18.1.tar.gz',
              '86e613527e5dba544e73208f42b78b7c022d4fa5a6d5498bf18c8d6f745b91dc',
              [configure_args + ['--without-readline'],
              ['make', '-s', '-j', str(multiprocessing.cpu_count())],
              ['make', 'install', 'DESTDIR={}/'.format(deps_destdir)],
             ]),
}


def _unpack_tarfile(filename, extract_dir):
    """Unpack tar/tar.gz/tar.bz2/tar.xz `filename` to `extract_dir`
    """
    import tarfile  # late import for breaking circular dependency
    try:
        tarobj = tarfile.open(filename)
    except tarfile.TarError:
        raise ReadError(
            "%s is not a compressed or uncompressed tar file" % filename)
    try:
        tarobj.extractall(extract_dir)
    finally:
        tarobj.close()

def _sha256(filename):
    dgst = hashlib.sha256()

    with open(filename, 'rb') as fp:
        dgst.update(fp.read())
    return dgst.hexdigest()


def _build_dependency(name, patches=[]):
    import shutil
    from rpython.tool.runsubprocess import run_subprocess

    try:
        from urllib.request import urlretrieve
    except ImportError:
        from urllib import urlretrieve

    try:
        url, dgst, build_cmds = cffi_dependencies[name]
    except KeyError:
        return 0, None, None

    archive_dir = os.path.join(tempfile.gettempdir(), 'pypy-archives')

    if not os.path.isdir(archive_dir):
        os.makedirs(archive_dir)

    archive = os.path.join(archive_dir, url.rsplit('/', 1)[-1])

    # next, fetch the archive to disk, if needed
    if not os.path.exists(archive) or _sha256(archive) != dgst:
        print('fetching archive', url, file=sys.stderr)
        urlretrieve(url, archive)

    # make sure the hash matches
    if _sha256(archive) != dgst:
        return 1, '{} archive {} hash mismatch'.format(name, archive), ''

    shutil.rmtree(deps_destdir, ignore_errors=True)
    os.makedirs(deps_destdir)

    # extract the into our destination directory
    print('unpacking archive', archive, file=sys.stderr)
    _unpack_tarfile(archive, deps_destdir)

    sources = os.path.join(
        deps_destdir,
        os.path.basename(archive).rsplit('.', 2)[0],
    )

    # apply any patches
    if patches:
        for patch in patches:
            print('applying patch', patch, file=sys.stderr)
            status, stdout, stderr = run_subprocess(
                '/usr/bin/patch', ['-p1', '-i', patch], cwd=sources,
            )

            if status != 0:
                return status, stdout, stderr
    env = os.environ
    if sys.platform == 'darwin':
        target = sysconfig.get_config_var('MACOSX_DEPLOYMENT_TARGET')
        if target:
            # override the value for building support libraries
            env = os.environ.copy()
            env['MACOSX_DEPLOYMENT_TARGET'] = target
            print('setting MACOSX_DEPLOYMENT_TARGET to "{}"'.format(target))
        
    for args in build_cmds:
        print('running', ' '.join(args), 'in', sources, file=sys.stderr)
        status, stdout, stderr = run_subprocess(args[0], args[1:],
                                                cwd=sources, env=env)
        if status != 0:
            break
    return status, stdout, stderr

def create_cffi_import_libraries(pypy_c, options, basedir, only=None,
                                 embed_dependencies=False, rebuild=False):
    from rpython.tool.runsubprocess import run_subprocess

    shutil.rmtree(str(join(basedir,'lib_pypy','__pycache__')),
                  ignore_errors=True)
    # be sure pip, setuptools are installed in a fresh pypy
    # allows proper functioning of cffi on win32 with newer vc compilers
    # XXX move this to a build slave step?
    env = os.environ
    if sys.platform == 'win32':
        env = os.environ.copy()
        env['INCLUDE'] = r'..\externals\include;' + env.get('INCLUDE', '')
        env['LIB'] = r'..\externals\lib;' + env.get('LIB', '')
        env['PATH'] = r'..\externals\bin;' + env.get('PATH', '')
    status, stdout, stderr = run_subprocess(str(pypy_c), ['-c', 'import setuptools'])
    if status  != 0:
        status, stdout, stderr = run_subprocess(str(pypy_c), ['-m', 'ensurepip'])
    failures = []

    for key, module in cffi_build_scripts.items():
        if only and key not in only:
            print("* SKIPPING", key, '(not specified in --only)')
            continue
        if module is None or getattr(options, 'no_' + key, False):
            continue
        if not rebuild:
            # the key is the module name, has it already been built?
            status, stdout, stderr = run_subprocess(str(pypy_c),
                                         ['-c', 'import %s' % key], env=env)
            if status  == 0:
                print('*', ' %s already built' % key, file=sys.stderr)
                continue

        if module.endswith('.py'):
            args = [module]
            cwd = str(join(basedir,'lib_pypy'))
        else:
            args = ['-c', 'import ' + module]
            cwd = None

        print('*', ' '.join(args), file=sys.stderr)
        if embed_dependencies and key in cffi_dependencies:
            status, stdout, stderr = _build_dependency(key)
            if status != 0:
                failures.append((key, module))
                print("stdout:")
                print(stdout.decode('utf-8'))
                print("stderr:")
                print(stderr.decode('utf-8'))
                continue

            env['CPPFLAGS'] = \
                '-I{}/usr/include {}'.format(deps_destdir, env.get('CPPFLAGS', ''))
            env['LDFLAGS'] = \
                '-L{}/usr/lib {}'.format(deps_destdir, env.get('LDFLAGS', ''))

        try:
            status, stdout, stderr = run_subprocess(str(pypy_c), args,
                                                    cwd=cwd, env=env)
            if status != 0:
                print("stdout:")
                print(stdout, file=sys.stderr)
                print("stderr:")
                print(stderr, file=sys.stderr)
                raise RuntimeError('building {} failed'.format(key))
        except:
            import traceback;traceback.print_exc()
            failures.append((key, module))
        else:
            # Make sure it worked
            status, stdout, stderr = run_subprocess(str(pypy_c),
                         ['-c', "print('testing {0}'); import {0}".format(key)],
                         env=env)
            if status != 0:
                failures.append((key, module))
                print("stdout:")
                print(stdout, file=sys.stderr)
                print("stderr:")
                print(stderr, file=sys.stderr)
        if os.path.exists(deps_destdir):
            shutil.rmtree(deps_destdir, ignore_errors=True)
    return failures

if __name__ == '__main__':
    import argparse
    if '__pypy__' not in sys.builtin_module_names:
        print('Call with a pypy interpreter', file=sys.stderr)
        sys.exit(1)

    tool_dir = os.path.dirname(os.path.abspath(sys.argv[0]))
    base_dir = os.path.dirname(os.path.dirname(tool_dir))
    sys.path.insert(0, base_dir)

    class Options(object):
        pass

    parser = argparse.ArgumentParser(description='Build all cffi backends in lib_pypy')
    parser.add_argument('--exefile', dest='exefile', default=sys.executable,
                        help='instead of executing sys.executable' \
                             ' you can specify an alternative pypy vm here')
    parser.add_argument('--rebuild', dest='rebuild', action='store_true',
        help='Rebuild the module even if it already appears to have been built.')
    parser.add_argument('--only', dest='only', default=None,
                        help='Only build the modules delimited by a colon. E.g. _ssl,sqlite')
    parser.add_argument('--embed-dependencies', dest='embed_dependencies', action='store_true',
        help='embed dependencies for distribution')
    args = parser.parse_args()

    exename = join(os.getcwd(), args.exefile)
    basedir = exename

    while not os.path.exists(join(basedir,'include')):
        _basedir = os.path.dirname(basedir)
        if _basedir == basedir:
            raise ValueError('interpreter %s not inside pypy repo', 
                                 str(exename))
        basedir = _basedir
    options = Options()
    if args.only is None:
        only = None
    else:
        only = set(args.only.split(','))
    failures = create_cffi_import_libraries(exename, options, basedir, only=only,
                                            embed_dependencies=args.embed_dependencies,
                                            rebuild=args.rebuild)
    if len(failures) > 0:
        print('*** failed to build the CFFI modules %r' % (
            [f[1] for f in failures],), file=sys.stderr)
        print('''
PyPy can still be used as long as you don't need the corresponding
modules.  If you do need them, please install the missing headers and
libraries (see error messages just above) and then re-run the command:

    %s %s
''' % (sys.executable, ' '.join(sys.argv)), file=sys.stderr)
        sys.exit(1)

    if len(sys.argv) > 1 and sys.argv[1] == '--test':
        # monkey patch a failure, just to test
        print('This line should be followed by a traceback', file=sys.stderr)
        for k in cffi_build_scripts:
            setattr(options, 'no_' + k, True)
        must_fail = '_missing_build_script.py'
        assert not os.path.exists(str(join(join(basedir,'lib_pypy'),must_fail)))
        cffi_build_scripts['should_fail'] = must_fail
        failures = create_cffi_import_libraries(exename, options, basedir, only=only)
        assert len(failures) == 1