diff options
author | 2015-10-26 17:38:41 +1100 | |
---|---|---|
committer | 2015-10-26 17:38:41 +1100 | |
commit | 03721401c55137cf1cc4a57ac1bdd4fb411fce12 (patch) | |
tree | b3d3e75ac1094747da5109dee5ce988a0e63413e | |
parent | merge default into branch (diff) | |
parent | test, fix for broadcasting with operand shape first dim==1 (diff) | |
download | pypy-release-4.0.0.tar.gz pypy-release-4.0.0.tar.bz2 pypy-release-4.0.0.zip |
update release from defaultrelease-4.0.0
-rw-r--r-- | pypy/doc/release-4.0.0.rst | 10 | ||||
-rw-r--r-- | pypy/doc/test/test_whatsnew.py | 10 | ||||
-rw-r--r-- | pypy/doc/whatsnew-head.rst | 6 | ||||
-rw-r--r-- | pypy/interpreter/executioncontext.py | 11 | ||||
-rw-r--r-- | pypy/interpreter/test/test_pyframe.py | 25 | ||||
-rw-r--r-- | pypy/module/micronumpy/base.py | 11 | ||||
-rw-r--r-- | pypy/module/micronumpy/ctors.py | 9 | ||||
-rw-r--r-- | pypy/module/micronumpy/nditer.py | 27 | ||||
-rw-r--r-- | pypy/module/micronumpy/strides.py | 1 | ||||
-rw-r--r-- | pypy/module/micronumpy/test/test_subtype.py | 47 | ||||
-rw-r--r-- | pypy/module/micronumpy/test/test_ufuncs.py | 51 | ||||
-rw-r--r-- | pypy/module/micronumpy/ufuncs.py | 14 |
12 files changed, 162 insertions, 60 deletions
diff --git a/pypy/doc/release-4.0.0.rst b/pypy/doc/release-4.0.0.rst index 9c6694c0c7..098dca70e7 100644 --- a/pypy/doc/release-4.0.0.rst +++ b/pypy/doc/release-4.0.0.rst @@ -3,7 +3,7 @@ PyPy 4.0.0 ============ We're pleased and proud to unleash PyPy 4.0.0, a major update of the PyPy -python2.7.10 compatible interpreter with a Just In Time compiler. +python 2.7.10 compatible interpreter with a Just In Time compiler. We have improved `warmup time and memory overhead used for tracing`_, added `vectorization`_ for numpy and general loops where possible on x86 hardware (disabled by default), @@ -72,15 +72,17 @@ CFFI While not applicable only to PyPy, `cffi`_ is arguably our most significant contribution to the python ecosystem. Armin Rigo continued improving it, -and PyPy reaps the benefits of cffi-1.3: improved manangement of object -lifetimes, __stdcall on Win32, ffi.memmove(), ... +and PyPy reaps the benefits of `cffi-1.3`_: improved manangement of object +lifetimes, __stdcall on Win32, ffi.memmove(), and percolate ``const``, +``restrict`` keywords from cdef to C code. -.. _`warmup time and memory overhead used for tracing`: http://morepypy.blogspot.com/2015/10 +.. _`warmup time and memory overhead used for tracing`: http://morepypy.blogspot.com/2015/10/pypy-memory-and-warmup-improvements-2.html .. _`vectorization`: http://pypyvecopt.blogspot.co.at/ .. _`guards`: http://rpython.readthedocs.org/en/latest/glossary.html .. _`PyPy`: http://doc.pypy.org .. _`RPython`: https://rpython.readthedocs.org .. _`cffi`: https://cffi.readthedocs.org +.. _`cffi-1.3`: http://cffi.readthedocs.org/en/latest/whatsnew.html#v1-3-0 .. _`modules`: http://doc.pypy.org/en/latest/project-ideas.html#make-more-python-modules-pypy-friendly .. _`help`: http://doc.pypy.org/en/latest/project-ideas.html .. _`numpy`: https://bitbucket.org/pypy/numpy diff --git a/pypy/doc/test/test_whatsnew.py b/pypy/doc/test/test_whatsnew.py index 82a33592a0..ad14b7c837 100644 --- a/pypy/doc/test/test_whatsnew.py +++ b/pypy/doc/test/test_whatsnew.py @@ -101,3 +101,13 @@ def test_whatsnew(): assert not not_documented if branch == 'default': assert not not_merged + +def test_startrev_on_default(): + doc = ROOT.join('pypy', 'doc') + last_whatsnew = doc.join('whatsnew-head.rst').read() + startrev, documented = parse_doc(last_whatsnew) + errcode, wc_branch = getstatusoutput( + "hg log -r %s --template '{branch}'" % startrev) + if errcode != 0: + py.test.skip('no Mercurial repo') + assert wc_branch == 'default' diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst index 3060ee5bd5..4282089bf6 100644 --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -3,6 +3,8 @@ What's new in PyPy 4.0.+ ========================= .. this is a revision shortly after release-4.0.0 -.. startrev: 3a8f5481dab4 - +.. startrev: 9397d7c6f5aa +.. branch: lazy-fast2locals +improve the performance of simple trace functions by lazily calling +fast2locals and locals2fast only if f_locals is actually accessed. diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py index 992476a635..394cea068e 100644 --- a/pypy/interpreter/executioncontext.py +++ b/pypy/interpreter/executioncontext.py @@ -327,10 +327,14 @@ class ExecutionContext(object): w_arg = space.newtuple([operr.w_type, w_value, space.wrap(operr.get_traceback())]) - frame.fast2locals() + d = frame.getorcreatedebug() + if d.w_locals is not None: + # only update the w_locals dict if it exists + # if it does not exist yet and the tracer accesses it via + # frame.f_locals, it is filled by PyFrame.getdictscope + frame.fast2locals() self.is_tracing += 1 try: - d = frame.getorcreatedebug() try: w_result = space.call_function(w_callback, space.wrap(frame), space.wrap(event), w_arg) if space.is_w(w_result, space.w_None): @@ -343,7 +347,8 @@ class ExecutionContext(object): raise finally: self.is_tracing -= 1 - frame.locals2fast() + if d.w_locals is not None: + frame.locals2fast() # Profile cases if self.profilefunc is not None: diff --git a/pypy/interpreter/test/test_pyframe.py b/pypy/interpreter/test/test_pyframe.py index 69c8899c01..e7e7a80a58 100644 --- a/pypy/interpreter/test/test_pyframe.py +++ b/pypy/interpreter/test/test_pyframe.py @@ -1,10 +1,14 @@ from rpython.tool import udir from pypy.conftest import option +from pypy.interpreter.gateway import interp2app +def check_no_w_locals(space, w_frame): + return space.wrap(w_frame.getorcreatedebug().w_locals is None) class AppTestPyFrame: def setup_class(cls): + space = cls.space cls.w_udir = cls.space.wrap(str(udir.udir)) cls.w_tempfile1 = cls.space.wrap(str(udir.udir.join('tempfile1'))) if not option.runappdirect: @@ -17,6 +21,8 @@ class AppTestPyFrame: w_call_further.code.hidden_applevel = True # hack cls.w_call_further = w_call_further + cls.w_check_no_w_locals = space.wrap(interp2app(check_no_w_locals)) + # test for the presence of the attributes, not functionality def test_f_locals(self): @@ -493,6 +499,25 @@ class AppTestPyFrame: sys.settrace(None) assert res == 42 + def test_fast2locals_called_lazily(self): + import sys + class FrameHolder: + pass + fh = FrameHolder() + def trace(frame, what, arg): + # trivial trace function, does not access f_locals + fh.frame = frame + return trace + def f(x): + x += 1 + return x + sys.settrace(trace) + res = f(1) + sys.settrace(None) + assert res == 2 + if hasattr(self, "check_no_w_locals"): # not appdirect + assert self.check_no_w_locals(fh.frame) + def test_set_unset_f_trace(self): import sys seen = [] diff --git a/pypy/module/micronumpy/base.py b/pypy/module/micronumpy/base.py index 7f884c191f..c8518483a6 100644 --- a/pypy/module/micronumpy/base.py +++ b/pypy/module/micronumpy/base.py @@ -86,11 +86,14 @@ class W_NDimArray(W_NumpyObject): if len(strides) != len(shape): raise oefmt(space.w_ValueError, 'strides, if given, must be the same length as shape') + last = 0 for i in range(len(strides)): - if strides[i] < 0 or strides[i]*shape[i] > storage_bytes: - raise oefmt(space.w_ValueError, - 'strides is incompatible with shape of requested ' - 'array and size of buffer') + last += (shape[i] - 1) * strides[i] + if last > storage_bytes or start < 0 or \ + start + dtype.elsize > storage_bytes: + raise oefmt(space.w_ValueError, + 'strides is incompatible with shape of requested ' + 'array and size of buffer') backstrides = calc_backstrides(strides, shape) if w_base is not None: if owning: diff --git a/pypy/module/micronumpy/ctors.py b/pypy/module/micronumpy/ctors.py index 5efc69f132..e7e18e3d96 100644 --- a/pypy/module/micronumpy/ctors.py +++ b/pypy/module/micronumpy/ctors.py @@ -134,13 +134,18 @@ def _array(space, w_object, w_dtype=None, copy=True, w_order=None, subok=False): else: imp = w_object.implementation w_base = w_object + sz = w_base.get_size() * dtype.elsize if imp.base() is not None: w_base = imp.base() + if type(w_base) is W_NDimArray: + sz = w_base.get_size() * dtype.elsize + else: + # this must succeed (mmap, buffer, ...) + sz = space.int_w(space.call_method(w_base, 'size')) with imp as storage: - sz = support.product(w_object.get_shape()) * dtype.elsize return W_NDimArray.from_shape_and_storage(space, w_object.get_shape(), storage, dtype, storage_bytes=sz, - w_base=w_base, start=imp.start) + w_base=w_base, strides=imp.strides, start=imp.start) else: # not an array shape, elems_w = find_shape_and_elems(space, w_object, dtype) diff --git a/pypy/module/micronumpy/nditer.py b/pypy/module/micronumpy/nditer.py index 3d0aad712a..506c39db81 100644 --- a/pypy/module/micronumpy/nditer.py +++ b/pypy/module/micronumpy/nditer.py @@ -348,7 +348,7 @@ class W_NDIter(W_NumpyObject): _immutable_fields_ = ['ndim', ] def __init__(self, space, w_seq, w_flags, w_op_flags, w_op_dtypes, w_casting, w_op_axes, w_itershape, buffersize=0, - order=NPY.KEEPORDER): + order=NPY.KEEPORDER, allow_backward=True): self.external_loop = False self.buffered = False self.tracked_index = '' @@ -363,6 +363,7 @@ class W_NDIter(W_NumpyObject): self.done = False self.first_next = True self.op_axes = [] + self.allow_backward = allow_backward if not space.is_w(w_casting, space.w_None): self.casting = space.str_w(w_casting) else: @@ -540,18 +541,18 @@ class W_NDIter(W_NumpyObject): self.op_flags[i], self) backward = imp.order != self.order # XXX cleanup needed - if (abs(imp.strides[0]) < abs(imp.strides[-1]) and not backward) or \ - (abs(imp.strides[0]) > abs(imp.strides[-1]) and backward): - # flip the strides. Is this always true for multidimension? - strides = imp.strides[:] - backstrides = imp.backstrides[:] - shape = imp.shape[:] - strides.reverse() - backstrides.reverse() - shape.reverse() - else: - strides = imp.strides - backstrides = imp.backstrides + strides = imp.strides + backstrides = imp.backstrides + if self.allow_backward: + if ((abs(imp.strides[0]) < abs(imp.strides[-1]) and not backward) or \ + (abs(imp.strides[0]) > abs(imp.strides[-1]) and backward)): + # flip the strides. Is this always true for multidimension? + strides = imp.strides[:] + backstrides = imp.backstrides[:] + shape = imp.shape[:] + strides.reverse() + backstrides.reverse() + shape.reverse() r = calculate_broadcast_strides(strides, backstrides, imp.shape, shape, backward) iter_shape = shape diff --git a/pypy/module/micronumpy/strides.py b/pypy/module/micronumpy/strides.py index d946e5d430..02c13a4d90 100644 --- a/pypy/module/micronumpy/strides.py +++ b/pypy/module/micronumpy/strides.py @@ -140,7 +140,6 @@ def calculate_slice_strides(space, shape, start, strides, backstrides, chunks): extra_dims = len(shape) - used_dims rstrides = [0] * (size + extra_dims) rbackstrides = [0] * (size + extra_dims) - rstart = start rshape = [0] * (size + extra_dims) rstart = start i = 0 # index of the current dimension in the input array diff --git a/pypy/module/micronumpy/test/test_subtype.py b/pypy/module/micronumpy/test/test_subtype.py index 53b8219fc1..14688b2de4 100644 --- a/pypy/module/micronumpy/test/test_subtype.py +++ b/pypy/module/micronumpy/test/test_subtype.py @@ -309,13 +309,13 @@ class AppTestSupport(BaseNumpyAppTest): assert c.dtype is dtype(float) def test_array_of_subtype(self): - import numpy as N + import numpy as np # this part of numpy's matrix class causes an infinite loop # on cpython import sys if '__pypy__' not in sys.builtin_module_names: skip('does not pass on cpython') - class matrix(N.ndarray): + class matrix(np.ndarray): def __new__(subtype, data, dtype=None, copy=True): print('matrix __new__') if isinstance(data, matrix): @@ -326,11 +326,11 @@ class AppTestSupport(BaseNumpyAppTest): return data return data.astype(dtype) - if isinstance(data, N.ndarray): + if isinstance(data, np.ndarray): if dtype is None: intype = data.dtype else: - intype = N.dtype(dtype) + intype = np.dtype(dtype) new = data.view(subtype) if intype != data.dtype: return new.astype(intype) @@ -341,7 +341,7 @@ class AppTestSupport(BaseNumpyAppTest): data = _convert_from_string(data) # now convert data to an array - arr = N.array(data, dtype=dtype, copy=copy) + arr = np.array(data, dtype=dtype, copy=copy) ndim = arr.ndim shape = arr.shape if (ndim > 2): @@ -358,7 +358,7 @@ class AppTestSupport(BaseNumpyAppTest): if not (order or arr.flags.contiguous): arr = arr.copy() - ret = N.ndarray.__new__(subtype, shape, arr.dtype, + ret = np.ndarray.__new__(subtype, shape, arr.dtype, buffer=arr, order=order) return ret @@ -391,11 +391,11 @@ class AppTestSupport(BaseNumpyAppTest): self._getitem = True try: - out = N.ndarray.__getitem__(self, index) + out = np.ndarray.__getitem__(self, index) finally: self._getitem = False - if not isinstance(out, N.ndarray): + if not isinstance(out, np.ndarray): return out if out.ndim == 0: @@ -414,18 +414,18 @@ class AppTestSupport(BaseNumpyAppTest): return out a = matrix([[1., 2.], [3., 4.]]) - b = N.array([a]) + b = np.array([a]) assert (b == a).all() - b = N.array(a) + b = np.array(a) assert len(b.shape) == 2 assert (b == a).all() - b = N.array(a, copy=False) + b = np.array(a, copy=False) assert len(b.shape) == 2 assert (b == a).all() - b = N.array(a, copy=True, dtype=int) + b = np.array(a, copy=True, dtype=int) assert len(b.shape) == 2 assert (b == a).all() @@ -433,9 +433,30 @@ class AppTestSupport(BaseNumpyAppTest): assert c.base is not None c[0, 0] = 100 assert a[0, 0] == 100 - b = N.array(c, copy=True) + b = np.array(c, copy=True) assert (b == a).all() + d = np.empty([6,2], dtype=float) + d.view(int).fill(0xdeadbeef) + e = d[0::3,:] + e[...] = [[1, 2], [3, 4]] + assert e.strides == (48, 8) + f = e.view(matrix) + assert f.strides == (48, 8) + g = np.array(f, copy=False) + assert (g == [[1, 2], [3, 4]]).all() + + k = np.empty([2, 8], dtype=float) + k.view(int).fill(0xdeadbeef) + m = k[:, ::-4] + m[...] = [[1, 2], [3, 4]] + assert m.strides == (64, -32) + n = m.view(matrix) + assert n.strides == (64, -32) + p = np.array(n, copy=False) + assert (p == [[1, 2], [3, 4]]).all() + + def test_setstate_no_version(self): # Some subclasses of ndarray, like MaskedArray, do not use # version in __setstate__ diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py index 978c710492..85c6c058a1 100644 --- a/pypy/module/micronumpy/test/test_ufuncs.py +++ b/pypy/module/micronumpy/test/test_ufuncs.py @@ -202,13 +202,38 @@ class AppTestUfuncs(BaseNumpyAppTest): ao = ufunc(ai) assert ao.shape == (0, 1, 1) + def test_frompyfunc_not_contiguous(self): + import sys + from numpy import frompyfunc, dtype, arange, dot + if '__pypy__' not in sys.builtin_module_names: + skip('PyPy only frompyfunc extension') + def _dot(in0, in1, out): + print in0, '\nin1',in1,'\nin1.shape', in1.shape, 'in1.strides', in1.strides + out[...] = dot(in0, in1) + + ufunc_dot = frompyfunc(_dot, 2, 1, + signature='(m,m),(m,n)->(m,n)', + dtypes=[dtype(float), dtype(float), dtype(float)], + stack_inputs=True, + ) + a1 = arange(4, dtype=float).reshape(2,2) + # create a non-c-contiguous argument + a2 = arange(2, dtype=float).reshape(2,1) + a3 = arange(2, dtype=float).reshape(1,2).T + b1 = ufunc_dot(a1, a2, sig='dd->d') + b2 = dot(a1, a2) + assert (b1==b2).all() + print 'xxxxxxxxxxxx' + b1 = ufunc_dot(a1, a3, sig='dd->d') + b2 = dot(a1, a3) + assert (b1==b2).all() + def test_frompyfunc_needs_nditer(self): import sys from numpy import frompyfunc, dtype, arange if '__pypy__' not in sys.builtin_module_names: skip('PyPy only frompyfunc extension') def summer(in0): - print 'in summer, in0=',in0,'in0.shape=',in0.shape return in0.sum() ufunc = frompyfunc([summer], 1, 1, @@ -242,11 +267,14 @@ class AppTestUfuncs(BaseNumpyAppTest): stack_inputs=True, ) ai = arange(18, dtype=int).reshape(3,2,3) - aout = ufunc_add(ai, ai[0,:,:]) - assert aout.shape == (3, 2, 3) - aout = ufunc_sum(ai) - assert aout.shape == (3, 3) - + aout1 = ufunc_add(ai, ai[0,:,:]) + assert aout1.shape == (3, 2, 3) + aout2 = ufunc_add(ai, ai[0,:,:]) + aout3 = ufunc_sum(ai) + assert aout3.shape == (3, 3) + aout4 = ufunc_add(ai, ai[0,:,:][None, :,:]) + assert (aout1 == aout4).all() + def test_frompyfunc_fortran(self): import sys import numpy as np @@ -280,7 +308,7 @@ class AppTestUfuncs(BaseNumpyAppTest): def times2_int(in0, out0): assert in0.dtype == int assert out0.dtype == int - # hack to assing to a 0-dim array + # hack to assigning to a 0-dim array out0.real = in0 * 2 def times2_complex(in0, out0): @@ -319,6 +347,10 @@ class AppTestUfuncs(BaseNumpyAppTest): assert out0.dtype in (int, complex) assert (out0 == in0 * 2).all() + in0 = np.arange(4, dtype=int) + out0 = times2(in0, sig='D->D') + assert out0.dtype == complex + def test_frompyfunc_scalar(self): import sys import numpy as np @@ -354,7 +386,7 @@ class AppTestUfuncs(BaseNumpyAppTest): assert all(res == args[0] + args[1]) raises(TypeError, adder_ufunc, *args, blah=True) raises(TypeError, adder_ufunc, *args, extobj=True) - raises(RuntimeError, adder_ufunc, *args, sig='(d,d)->(d)', dtype=int) + raises(RuntimeError, adder_ufunc, *args, sig='dd->d', dtype=int) def test_unary_ufunc_kwargs(self): from numpy import array, sin, float16 @@ -453,7 +485,7 @@ class AppTestUfuncs(BaseNumpyAppTest): except AttributeError: pass except NotImplementedError: - print s + #print s uncallable.add(s) except TypeError: assert s not in uncallable @@ -1404,7 +1436,6 @@ class AppTestUfuncs(BaseNumpyAppTest): # dtype a = arange(0, 3, 0.5).reshape(2, 3) b = add.accumulate(a, dtype=int, axis=1) - print b assert (b == [[0, 0, 1], [1, 3, 5]]).all() assert b.dtype == int assert add.accumulate([True]*200)[-1] == 200 diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py index a373d7f0c4..de176afcaa 100644 --- a/pypy/module/micronumpy/ufuncs.py +++ b/pypy/module/micronumpy/ufuncs.py @@ -805,8 +805,6 @@ class W_UfuncGeneric(W_Ufunc): raise oefmt(space.w_TypeError, 'output arg %d must be an array, not %s', i+self.nin, str(args_w[i+self.nin])) outargs[i] = out - if sig is None: - sig = space.wrap(self.signature) _dtypes = self.dtypes if self.match_dtypes: _dtypes = [i.get_dtype() for i in inargs if isinstance(i, W_NDimArray)] @@ -879,7 +877,7 @@ class W_UfuncGeneric(W_Ufunc): w_itershape = space.newlist([space.wrap(i) for i in iter_shape]) nd_it = W_NDIter(space, space.newlist(inargs + outargs), w_flags, w_op_flags, w_op_dtypes, w_casting, w_op_axes, - w_itershape) + w_itershape, allow_backward=False) # coalesce each iterators, according to inner_dimensions for i in range(len(inargs) + len(outargs)): for j in range(self.core_num_dims[i]): @@ -942,8 +940,8 @@ class W_UfuncGeneric(W_Ufunc): raise oefmt(space.w_RuntimeError, "cannot specify both 'sig' and 'dtype'") dtype = decode_w_dtype(space, dtype_w) - sig = space.newtuple([dtype]) - order = kwargs_w.pop('dtype', None) + sig = dtype.char + order = kwargs_w.pop('order', None) if not space.is_w(order, space.w_None) and not order is None: raise oefmt(space.w_NotImplementedError, '"order" keyword not implemented') parsed_kw = [] @@ -952,7 +950,7 @@ class W_UfuncGeneric(W_Ufunc): if sig: raise oefmt(space.w_RuntimeError, "cannot specify both 'sig' and 'dtype'") - sig = kwargs_w[kw] + sig = space.str_w(kwargs_w[kw]) parsed_kw.append(kw) elif kw.startswith('where'): raise oefmt(space.w_NotImplementedError, @@ -966,7 +964,7 @@ class W_UfuncGeneric(W_Ufunc): # Find a match for the inargs.dtype in _dtypes, like # linear_search_type_resolver in numpy ufunc_type_resolutions.c # type_tup can be '', a tuple of dtypes, or a string - # of the form d,t -> D where the letters are dtype specs + # of the form 'dt->D' where the letters are dtype specs # XXX why does the next line not pass translation? # dtypes = [i.get_dtype() for i in inargs] @@ -1119,7 +1117,7 @@ class W_UfuncGeneric(W_Ufunc): for j in range(offset, len(iter_shape)): x = iter_shape[j + offset] y = dims_to_broadcast[j] - if y != 0 and x != 0 and ((x > y and x % y) or y %x): + if y > 1 and x != 0 and ((x > y and x % y) or y %x): raise oefmt(space.w_ValueError, "%s: %s operand %d has a " "mismatch in its broadcast dimension %d " "(size %d is different from %d)", |