How to use rewrite method in Pytest

Best Python code snippet using pytest

test_error_functions.py

Source:test_error_functions.py

...36    assert erf(x).as_leading_term(x) == 2*x/sqrt(pi)37    assert erf(x*y).as_leading_term(y) == 2*x*y/sqrt(pi)38    assert (erf(x*y)/erf(y)).as_leading_term(y) == x39    assert erf(1/x).as_leading_term(x) == erf(1/x)40    assert erf(z).rewrite('uppergamma') == sqrt(z**2)*(1 - erfc(sqrt(z**2)))/z41    assert erf(z).rewrite('erfc') == S.One - erfc(z)42    assert erf(z).rewrite('erfi') == -I*erfi(I*z)43    assert erf(z).rewrite('fresnels') == (1 + I)*(fresnelc(z*(1 - I)/sqrt(pi)) -44        I*fresnels(z*(1 - I)/sqrt(pi)))45    assert erf(z).rewrite('fresnelc') == (1 + I)*(fresnelc(z*(1 - I)/sqrt(pi)) -46        I*fresnels(z*(1 - I)/sqrt(pi)))47    assert erf(z).rewrite('hyper') == 2*z*hyper([S.Half], [3*S.Half], -z**2)/sqrt(pi)48    assert erf(z).rewrite('meijerg') == z*meijerg([S.Half], [], [0], [Rational(-1, 2)], z**2)/sqrt(pi)49    assert erf(z).rewrite('expint') == sqrt(z**2)/z - z*expint(S.Half, z**2)/sqrt(S.Pi)50    assert limit(exp(x)*exp(x**2)*(erf(x + 1/exp(x)) - erf(x)), x, oo) == \51        2/sqrt(pi)52    assert limit((1 - erf(z))*exp(z**2)*z, z, oo) == 1/sqrt(pi)53    assert limit((1 - erf(x))*exp(x**2)*sqrt(pi)*x, x, oo) == 154    assert limit(((1 - erf(x))*exp(x**2)*sqrt(pi)*x - 1)*2*x**2, x, oo) == -155    assert limit(erf(x)/x, x, 0) == 2/sqrt(pi)56    assert limit(x**(-4) - sqrt(pi)*erf(x**2) / (2*x**6), x, 0) == S(1)/357    assert erf(x).as_real_imag() == \58        (erf(re(x) - I*im(x))/2 + erf(re(x) + I*im(x))/2,59         -I*(-erf(re(x) - I*im(x)) + erf(re(x) + I*im(x)))/2)60    assert erf(x).as_real_imag(deep=False) == \61        (erf(re(x) - I*im(x))/2 + erf(re(x) + I*im(x))/2,62         -I*(-erf(re(x) - I*im(x)) + erf(re(x) + I*im(x)))/2)63    assert erf(w).as_real_imag() == (erf(w), 0)64    assert erf(w).as_real_imag(deep=False) == (erf(w), 0)65    # issue 1357566    assert erf(I).as_real_imag() == (0, -I*erf(I))67    raises(ArgumentIndexError, lambda: erf(x).fdiff(2))68    assert erf(x).inverse() == erfinv69def test_erf_series():70    assert erf(x).series(x, 0, 7) == 2*x/sqrt(pi) - \71        2*x**3/3/sqrt(pi) + x**5/5/sqrt(pi) + O(x**7)72def test_erf_evalf():73    assert abs( erf(Float(2.0)) - 0.995322265 ) < 1E-8 # XXX74def test__erfs():75    assert _erfs(z).diff(z) == -2/sqrt(S.Pi) + 2*z*_erfs(z)76    assert _erfs(1/z).series(z) == \77        z/sqrt(pi) - z**3/(2*sqrt(pi)) + 3*z**5/(4*sqrt(pi)) + O(z**6)78    assert expand(erf(z).rewrite('tractable').diff(z).rewrite('intractable')) \79        == erf(z).diff(z)80    assert _erfs(z).rewrite("intractable") == (-erf(z) + 1)*exp(z**2)81    raises(ArgumentIndexError, lambda: _erfs(z).fdiff(2))82def test_erfc():83    assert erfc(nan) is nan84    assert erfc(oo) == 085    assert erfc(-oo) == 286    assert erfc(0) == 187    assert erfc(I*oo) == -oo*I88    assert erfc(-I*oo) == oo*I89    assert erfc(-x) == S(2) - erfc(x)90    assert erfc(erfcinv(x)) == x91    assert erfc(I).is_real is False92    assert erfc(0).is_real is True93    assert erfc(erfinv(x)) == 1 - x94    assert conjugate(erfc(z)) == erfc(conjugate(z))95    assert erfc(x).as_leading_term(x) is S.One96    assert erfc(1/x).as_leading_term(x) == erfc(1/x)97    assert erfc(z).rewrite('erf') == 1 - erf(z)98    assert erfc(z).rewrite('erfi') == 1 + I*erfi(I*z)99    assert erfc(z).rewrite('fresnels') == 1 - (1 + I)*(fresnelc(z*(1 - I)/sqrt(pi)) -100        I*fresnels(z*(1 - I)/sqrt(pi)))101    assert erfc(z).rewrite('fresnelc') == 1 - (1 + I)*(fresnelc(z*(1 - I)/sqrt(pi)) -102        I*fresnels(z*(1 - I)/sqrt(pi)))103    assert erfc(z).rewrite('hyper') == 1 - 2*z*hyper([S.Half], [3*S.Half], -z**2)/sqrt(pi)104    assert erfc(z).rewrite('meijerg') == 1 - z*meijerg([S.Half], [], [0], [Rational(-1, 2)], z**2)/sqrt(pi)105    assert erfc(z).rewrite('uppergamma') == 1 - sqrt(z**2)*(1 - erfc(sqrt(z**2)))/z106    assert erfc(z).rewrite('expint') == S.One - sqrt(z**2)/z + z*expint(S.Half, z**2)/sqrt(S.Pi)107    assert erfc(z).rewrite('tractable') == _erfs(z)*exp(-z**2)108    assert expand_func(erf(x) + erfc(x)) is S.One109    assert erfc(x).as_real_imag() == \110        (erfc(re(x) - I*im(x))/2 + erfc(re(x) + I*im(x))/2,111         -I*(-erfc(re(x) - I*im(x)) + erfc(re(x) + I*im(x)))/2)112    assert erfc(x).as_real_imag(deep=False) == \113        (erfc(re(x) - I*im(x))/2 + erfc(re(x) + I*im(x))/2,114         -I*(-erfc(re(x) - I*im(x)) + erfc(re(x) + I*im(x)))/2)115    assert erfc(w).as_real_imag() == (erfc(w), 0)116    assert erfc(w).as_real_imag(deep=False) == (erfc(w), 0)117    raises(ArgumentIndexError, lambda: erfc(x).fdiff(2))118    assert erfc(x).inverse() == erfcinv119def test_erfc_series():120    assert erfc(x).series(x, 0, 7) == 1 - 2*x/sqrt(pi) + \121        2*x**3/3/sqrt(pi) - x**5/5/sqrt(pi) + O(x**7)122def test_erfc_evalf():123    assert abs( erfc(Float(2.0)) - 0.00467773 ) < 1E-8 # XXX124def test_erfi():125    assert erfi(nan) is nan126    assert erfi(oo) is S.Infinity127    assert erfi(-oo) is S.NegativeInfinity128    assert erfi(0) is S.Zero129    assert erfi(I*oo) == I130    assert erfi(-I*oo) == -I131    assert erfi(-x) == -erfi(x)132    assert erfi(I*erfinv(x)) == I*x133    assert erfi(I*erfcinv(x)) == I*(1 - x)134    assert erfi(I*erf2inv(0, x)) == I*x135    assert erfi(I*erf2inv(0, x, evaluate=False)) == I*x # To cover code in erfi136    assert erfi(I).is_real is False137    assert erfi(0).is_real is True138    assert conjugate(erfi(z)) == erfi(conjugate(z))139    assert erfi(z).rewrite('erf') == -I*erf(I*z)140    assert erfi(z).rewrite('erfc') == I*erfc(I*z) - I141    assert erfi(z).rewrite('fresnels') == (1 - I)*(fresnelc(z*(1 + I)/sqrt(pi)) -142        I*fresnels(z*(1 + I)/sqrt(pi)))143    assert erfi(z).rewrite('fresnelc') == (1 - I)*(fresnelc(z*(1 + I)/sqrt(pi)) -144        I*fresnels(z*(1 + I)/sqrt(pi)))145    assert erfi(z).rewrite('hyper') == 2*z*hyper([S.Half], [3*S.Half], z**2)/sqrt(pi)146    assert erfi(z).rewrite('meijerg') == z*meijerg([S.Half], [], [0], [Rational(-1, 2)], -z**2)/sqrt(pi)147    assert erfi(z).rewrite('uppergamma') == (sqrt(-z**2)/z*(uppergamma(S.Half,148        -z**2)/sqrt(S.Pi) - S.One))149    assert erfi(z).rewrite('expint') == sqrt(-z**2)/z - z*expint(S.Half, -z**2)/sqrt(S.Pi)150    assert erfi(z).rewrite('tractable') == -I*(-_erfs(I*z)*exp(z**2) + 1)151    assert expand_func(erfi(I*z)) == I*erf(z)152    assert erfi(x).as_real_imag() == \153        (erfi(re(x) - I*im(x))/2 + erfi(re(x) + I*im(x))/2,154         -I*(-erfi(re(x) - I*im(x)) + erfi(re(x) + I*im(x)))/2)155    assert erfi(x).as_real_imag(deep=False) == \156        (erfi(re(x) - I*im(x))/2 + erfi(re(x) + I*im(x))/2,157         -I*(-erfi(re(x) - I*im(x)) + erfi(re(x) + I*im(x)))/2)158    assert erfi(w).as_real_imag() == (erfi(w), 0)159    assert erfi(w).as_real_imag(deep=False) == (erfi(w), 0)160    raises(ArgumentIndexError, lambda: erfi(x).fdiff(2))161def test_erfi_series():162    assert erfi(x).series(x, 0, 7) == 2*x/sqrt(pi) + \163        2*x**3/3/sqrt(pi) + x**5/5/sqrt(pi) + O(x**7)164def test_erfi_evalf():165    assert abs( erfi(Float(2.0)) - 18.5648024145756 ) < 1E-13  # XXX166def test_erf2():167    assert erf2(0, 0) is S.Zero168    assert erf2(x, x) is S.Zero169    assert erf2(nan, 0) is nan170    assert erf2(-oo,  y) ==  erf(y) + 1171    assert erf2( oo,  y) ==  erf(y) - 1172    assert erf2(  x, oo) ==  1 - erf(x)173    assert erf2(  x,-oo) == -1 - erf(x)174    assert erf2(x, erf2inv(x, y)) == y175    assert erf2(-x, -y) == -erf2(x,y)176    assert erf2(-x,  y) == erf(y) + erf(x)177    assert erf2( x, -y) == -erf(y) - erf(x)178    assert erf2(x, y).rewrite('fresnels') == erf(y).rewrite(fresnels)-erf(x).rewrite(fresnels)179    assert erf2(x, y).rewrite('fresnelc') == erf(y).rewrite(fresnelc)-erf(x).rewrite(fresnelc)180    assert erf2(x, y).rewrite('hyper') == erf(y).rewrite(hyper)-erf(x).rewrite(hyper)181    assert erf2(x, y).rewrite('meijerg') == erf(y).rewrite(meijerg)-erf(x).rewrite(meijerg)182    assert erf2(x, y).rewrite('uppergamma') == erf(y).rewrite(uppergamma) - erf(x).rewrite(uppergamma)183    assert erf2(x, y).rewrite('expint') == erf(y).rewrite(expint)-erf(x).rewrite(expint)184    assert erf2(I, 0).is_real is False185    assert erf2(0, 0).is_real is True186    assert expand_func(erf(x) + erf2(x, y)) == erf(y)187    assert conjugate(erf2(x, y)) == erf2(conjugate(x), conjugate(y))188    assert erf2(x, y).rewrite('erf')  == erf(y) - erf(x)189    assert erf2(x, y).rewrite('erfc') == erfc(x) - erfc(y)190    assert erf2(x, y).rewrite('erfi') == I*(erfi(I*x) - erfi(I*y))191    assert erf2(x, y).diff(x) == erf2(x, y).fdiff(1)192    assert erf2(x, y).diff(y) == erf2(x, y).fdiff(2)193    assert erf2(x, y).diff(x) == -2*exp(-x**2)/sqrt(pi)194    assert erf2(x, y).diff(y) == 2*exp(-y**2)/sqrt(pi)195    raises(ArgumentIndexError, lambda: erf2(x, y).fdiff(3))196    assert erf2(x, y).is_extended_real is None197    xr, yr = symbols('xr yr', extended_real=True)198    assert erf2(xr, yr).is_extended_real is True199def test_erfinv():200    assert erfinv(0) == 0201    assert erfinv(1) is S.Infinity202    assert erfinv(nan) is S.NaN203    assert erfinv(-1) is S.NegativeInfinity204    assert erfinv(erf(w)) == w205    assert erfinv(erf(-w)) == -w206    assert erfinv(x).diff() == sqrt(pi)*exp(erfinv(x)**2)/2207    raises(ArgumentIndexError, lambda: erfinv(x).fdiff(2))208    assert erfinv(z).rewrite('erfcinv') == erfcinv(1-z)209    assert erfinv(z).inverse() == erf210def test_erfinv_evalf():211    assert abs( erfinv(Float(0.2)) - 0.179143454621292 ) < 1E-13212def test_erfcinv():213    assert erfcinv(1) == 0214    assert erfcinv(0) is S.Infinity215    assert erfcinv(nan) is S.NaN216    assert erfcinv(x).diff() == -sqrt(pi)*exp(erfcinv(x)**2)/2217    raises(ArgumentIndexError, lambda: erfcinv(x).fdiff(2))218    assert erfcinv(z).rewrite('erfinv') == erfinv(1-z)219    assert erfcinv(z).inverse() == erfc220def test_erf2inv():221    assert erf2inv(0, 0) is S.Zero222    assert erf2inv(0, 1) is S.Infinity223    assert erf2inv(1, 0) is S.One224    assert erf2inv(0, y) == erfinv(y)225    assert erf2inv(oo, y) == erfcinv(-y)226    assert erf2inv(x, 0) == x227    assert erf2inv(x, oo) == erfinv(x)228    assert erf2inv(nan, 0) is nan229    assert erf2inv(0, nan) is nan230    assert erf2inv(x, y).diff(x) == exp(-x**2 + erf2inv(x, y)**2)231    assert erf2inv(x, y).diff(y) == sqrt(pi)*exp(erf2inv(x, y)**2)/2232    raises(ArgumentIndexError, lambda: erf2inv(x, y).fdiff(3))233# NOTE we multiply by exp_polar(I*pi) and need this to be on the principal234# branch, hence take x in the lower half plane (d=0).235def mytn(expr1, expr2, expr3, x, d=0):236    from sympy.testing.randtest import verify_numerically, random_complex_number237    subs = {}238    for a in expr1.free_symbols:239        if a != x:240            subs[a] = random_complex_number()241    return expr2 == expr3 and verify_numerically(expr1.subs(subs),242                                               expr2.subs(subs), x, d=d)243def mytd(expr1, expr2, x):244    from sympy.testing.randtest import test_derivative_numerically, \245        random_complex_number246    subs = {}247    for a in expr1.free_symbols:248        if a != x:249            subs[a] = random_complex_number()250    return expr1.diff(x) == expr2 and test_derivative_numerically(expr1.subs(subs), x)251def tn_branch(func, s=None):252    from sympy import I, pi, exp_polar253    from random import uniform254    def fn(x):255        if s is None:256            return func(x)257        return func(s, x)258    c = uniform(1, 5)259    expr = fn(c*exp_polar(I*pi)) - fn(c*exp_polar(-I*pi))260    eps = 1e-15261    expr2 = fn(-c + eps*I) - fn(-c - eps*I)262    return abs(expr.n() - expr2.n()).n() < 1e-10263def test_ei():264    assert Ei(0) is S.NegativeInfinity265    assert Ei(oo) is S.Infinity266    assert Ei(-oo) is S.Zero267    assert tn_branch(Ei)268    assert mytd(Ei(x), exp(x)/x, x)269    assert mytn(Ei(x), Ei(x).rewrite(uppergamma),270                -uppergamma(0, x*polar_lift(-1)) - I*pi, x)271    assert mytn(Ei(x), Ei(x).rewrite(expint),272                -expint(1, x*polar_lift(-1)) - I*pi, x)273    assert Ei(x).rewrite(expint).rewrite(Ei) == Ei(x)274    assert Ei(x*exp_polar(2*I*pi)) == Ei(x) + 2*I*pi275    assert Ei(x*exp_polar(-2*I*pi)) == Ei(x) - 2*I*pi276    assert mytn(Ei(x), Ei(x).rewrite(Shi), Chi(x) + Shi(x), x)277    assert mytn(Ei(x*polar_lift(I)), Ei(x*polar_lift(I)).rewrite(Si),278                Ci(x) + I*Si(x) + I*pi/2, x)279    assert Ei(log(x)).rewrite(li) == li(x)280    assert Ei(2*log(x)).rewrite(li) == li(x**2)281    assert gruntz(Ei(x+exp(-x))*exp(-x)*x, x, oo) == 1282    assert Ei(x).series(x) == EulerGamma + log(x) + x + x**2/4 + \283        x**3/18 + x**4/96 + x**5/600 + O(x**6)284    assert Ei(x).series(x, 1, 3) == Ei(1) + E*(x - 1) + O((x - 1)**3, (x, 1))285    assert str(Ei(cos(2)).evalf(n=10)) == '-0.6760647401'286    raises(ArgumentIndexError, lambda: Ei(x).fdiff(2))287def test_expint():288    assert mytn(expint(x, y), expint(x, y).rewrite(uppergamma),289                y**(x - 1)*uppergamma(1 - x, y), x)290    assert mytd(291        expint(x, y), -y**(x - 1)*meijerg([], [1, 1], [0, 0, 1 - x], [], y), x)292    assert mytd(expint(x, y), -expint(x - 1, y), y)293    assert mytn(expint(1, x), expint(1, x).rewrite(Ei),294                -Ei(x*polar_lift(-1)) + I*pi, x)295    assert expint(-4, x) == exp(-x)/x + 4*exp(-x)/x**2 + 12*exp(-x)/x**3 \296        + 24*exp(-x)/x**4 + 24*exp(-x)/x**5297    assert expint(Rational(-3, 2), x) == \298        exp(-x)/x + 3*exp(-x)/(2*x**2) + 3*sqrt(pi)*erfc(sqrt(x))/(4*x**S('5/2'))299    assert tn_branch(expint, 1)300    assert tn_branch(expint, 2)301    assert tn_branch(expint, 3)302    assert tn_branch(expint, 1.7)303    assert tn_branch(expint, pi)304    assert expint(y, x*exp_polar(2*I*pi)) == \305        x**(y - 1)*(exp(2*I*pi*y) - 1)*gamma(-y + 1) + expint(y, x)306    assert expint(y, x*exp_polar(-2*I*pi)) == \307        x**(y - 1)*(exp(-2*I*pi*y) - 1)*gamma(-y + 1) + expint(y, x)308    assert expint(2, x*exp_polar(2*I*pi)) == 2*I*pi*x + expint(2, x)309    assert expint(2, x*exp_polar(-2*I*pi)) == -2*I*pi*x + expint(2, x)310    assert expint(1, x).rewrite(Ei).rewrite(expint) == expint(1, x)311    assert expint(x, y).rewrite(Ei) == expint(x, y)312    assert expint(x, y).rewrite(Ci) == expint(x, y)313    assert mytn(E1(x), E1(x).rewrite(Shi), Shi(x) - Chi(x), x)314    assert mytn(E1(polar_lift(I)*x), E1(polar_lift(I)*x).rewrite(Si),315                -Ci(x) + I*Si(x) - I*pi/2, x)316    assert mytn(expint(2, x), expint(2, x).rewrite(Ei).rewrite(expint),317                -x*E1(x) + exp(-x), x)318    assert mytn(expint(3, x), expint(3, x).rewrite(Ei).rewrite(expint),319                x**2*E1(x)/2 + (1 - x)*exp(-x)/2, x)320    assert expint(Rational(3, 2), z).nseries(z) == \321        2 + 2*z - z**2/3 + z**3/15 - z**4/84 + z**5/540 - \322        2*sqrt(pi)*sqrt(z) + O(z**6)323    assert E1(z).series(z) == -EulerGamma - log(z) + z - \324        z**2/4 + z**3/18 - z**4/96 + z**5/600 + O(z**6)325    assert expint(4, z).series(z) == Rational(1, 3) - z/2 + z**2/2 + \326        z**3*(log(z)/6 - Rational(11, 36) + EulerGamma/6 - I*pi/6) - z**4/24 + \327        z**5/240 + O(z**6)328    assert expint(z, y).series(z, 0, 2) == exp(-y)/y - z*meijerg(((), (1, 1)),329                                  ((0, 0, 1), ()), y)/y + O(z**2)330    raises(ArgumentIndexError, lambda: expint(x, y).fdiff(3))331    neg = Symbol('neg', negative=True)332    assert Ei(neg).rewrite(Si) == Shi(neg) + Chi(neg) - I*pi333def test__eis():334    assert _eis(z).diff(z) == -_eis(z) + 1/z335    assert _eis(1/z).series(z) == \336        z + z**2 + 2*z**3 + 6*z**4 + 24*z**5 + O(z**6)337    assert Ei(z).rewrite('tractable') == exp(z)*_eis(z)338    assert li(z).rewrite('tractable') == z*_eis(log(z))339    assert _eis(z).rewrite('intractable') == exp(-z)*Ei(z)340    assert expand(li(z).rewrite('tractable').diff(z).rewrite('intractable')) \341        == li(z).diff(z)342    assert expand(Ei(z).rewrite('tractable').diff(z).rewrite('intractable')) \343        == Ei(z).diff(z)344    assert _eis(z).series(z, n=3) == EulerGamma + log(z) + z*(-log(z) - \345        EulerGamma + 1) + z**2*(log(z)/2 - Rational(3, 4) + EulerGamma/2) + O(z**3*log(z))346    raises(ArgumentIndexError, lambda: _eis(z).fdiff(2))347def tn_arg(func):348    def test(arg, e1, e2):349        from random import uniform350        v = uniform(1, 5)351        v1 = func(arg*x).subs(x, v).n()352        v2 = func(e1*v + e2*1e-15).n()353        return abs(v1 - v2).n() < 1e-10354    return test(exp_polar(I*pi/2), I, 1) and \355        test(exp_polar(-I*pi/2), -I, 1) and \356        test(exp_polar(I*pi), -1, I) and \357        test(exp_polar(-I*pi), -1, -I)358def test_li():359    z = Symbol("z")360    zr = Symbol("z", real=True)361    zp = Symbol("z", positive=True)362    zn = Symbol("z", negative=True)363    assert li(0) == 0364    assert li(1) is -oo365    assert li(oo) is oo366    assert isinstance(li(z), li)367    assert unchanged(li, -zp)368    assert unchanged(li, zn)369    assert diff(li(z), z) == 1/log(z)370    assert conjugate(li(z)) == li(conjugate(z))371    assert conjugate(li(-zr)) == li(-zr)372    assert unchanged(conjugate, li(-zp))373    assert unchanged(conjugate, li(zn))374    assert li(z).rewrite(Li) == Li(z) + li(2)375    assert li(z).rewrite(Ei) == Ei(log(z))376    assert li(z).rewrite(uppergamma) == (-log(1/log(z))/2 - log(-log(z)) +377                                         log(log(z))/2 - expint(1, -log(z)))378    assert li(z).rewrite(Si) == (-log(I*log(z)) - log(1/log(z))/2 +379                                 log(log(z))/2 + Ci(I*log(z)) + Shi(log(z)))380    assert li(z).rewrite(Ci) == (-log(I*log(z)) - log(1/log(z))/2 +381                                 log(log(z))/2 + Ci(I*log(z)) + Shi(log(z)))382    assert li(z).rewrite(Shi) == (-log(1/log(z))/2 + log(log(z))/2 +383                                  Chi(log(z)) - Shi(log(z)))384    assert li(z).rewrite(Chi) == (-log(1/log(z))/2 + log(log(z))/2 +385                                  Chi(log(z)) - Shi(log(z)))386    assert li(z).rewrite(hyper) ==(log(z)*hyper((1, 1), (2, 2), log(z)) -387                                   log(1/log(z))/2 + log(log(z))/2 + EulerGamma)388    assert li(z).rewrite(meijerg) == (-log(1/log(z))/2 - log(-log(z)) + log(log(z))/2 -389                                      meijerg(((), (1,)), ((0, 0), ()), -log(z)))390    assert gruntz(1/li(z), z, oo) == 0391    raises(ArgumentIndexError, lambda: li(z).fdiff(2))392def test_Li():393    assert Li(2) == 0394    assert Li(oo) is oo395    assert isinstance(Li(z), Li)396    assert diff(Li(z), z) == 1/log(z)397    assert gruntz(1/Li(z), z, oo) == 0398    assert Li(z).rewrite(li) == li(z) - li(2)399    raises(ArgumentIndexError, lambda: Li(z).fdiff(2))400def test_si():401    assert Si(I*x) == I*Shi(x)402    assert Shi(I*x) == I*Si(x)403    assert Si(-I*x) == -I*Shi(x)404    assert Shi(-I*x) == -I*Si(x)405    assert Si(-x) == -Si(x)406    assert Shi(-x) == -Shi(x)407    assert Si(exp_polar(2*pi*I)*x) == Si(x)408    assert Si(exp_polar(-2*pi*I)*x) == Si(x)409    assert Shi(exp_polar(2*pi*I)*x) == Shi(x)410    assert Shi(exp_polar(-2*pi*I)*x) == Shi(x)411    assert Si(oo) == pi/2412    assert Si(-oo) == -pi/2413    assert Shi(oo) is oo414    assert Shi(-oo) is -oo415    assert mytd(Si(x), sin(x)/x, x)416    assert mytd(Shi(x), sinh(x)/x, x)417    assert mytn(Si(x), Si(x).rewrite(Ei),418                -I*(-Ei(x*exp_polar(-I*pi/2))/2419               + Ei(x*exp_polar(I*pi/2))/2 - I*pi) + pi/2, x)420    assert mytn(Si(x), Si(x).rewrite(expint),421                -I*(-expint(1, x*exp_polar(-I*pi/2))/2 +422                    expint(1, x*exp_polar(I*pi/2))/2) + pi/2, x)423    assert mytn(Shi(x), Shi(x).rewrite(Ei),424                Ei(x)/2 - Ei(x*exp_polar(I*pi))/2 + I*pi/2, x)425    assert mytn(Shi(x), Shi(x).rewrite(expint),426                expint(1, x)/2 - expint(1, x*exp_polar(I*pi))/2 - I*pi/2, x)427    assert tn_arg(Si)428    assert tn_arg(Shi)429    assert Si(x).nseries(x, n=8) == \430        x - x**3/18 + x**5/600 - x**7/35280 + O(x**9)431    assert Shi(x).nseries(x, n=8) == \432        x + x**3/18 + x**5/600 + x**7/35280 + O(x**9)433    assert Si(sin(x)).nseries(x, n=5) == x - 2*x**3/9 + 17*x**5/450 + O(x**6)434    assert Si(x).nseries(x, 1, n=3) == \435        Si(1) + (x - 1)*sin(1) + (x - 1)**2*(-sin(1)/2 + cos(1)/2) + O((x - 1)**3, (x, 1))436    t = Symbol('t', Dummy=True)437    assert Si(x).rewrite(sinc) == Integral(sinc(t), (t, 0, x))438def test_ci():439    m1 = exp_polar(I*pi)440    m1_ = exp_polar(-I*pi)441    pI = exp_polar(I*pi/2)442    mI = exp_polar(-I*pi/2)443    assert Ci(m1*x) == Ci(x) + I*pi444    assert Ci(m1_*x) == Ci(x) - I*pi445    assert Ci(pI*x) == Chi(x) + I*pi/2446    assert Ci(mI*x) == Chi(x) - I*pi/2447    assert Chi(m1*x) == Chi(x) + I*pi448    assert Chi(m1_*x) == Chi(x) - I*pi449    assert Chi(pI*x) == Ci(x) + I*pi/2450    assert Chi(mI*x) == Ci(x) - I*pi/2451    assert Ci(exp_polar(2*I*pi)*x) == Ci(x) + 2*I*pi452    assert Chi(exp_polar(-2*I*pi)*x) == Chi(x) - 2*I*pi453    assert Chi(exp_polar(2*I*pi)*x) == Chi(x) + 2*I*pi454    assert Ci(exp_polar(-2*I*pi)*x) == Ci(x) - 2*I*pi455    assert Ci(oo) == 0456    assert Ci(-oo) == I*pi457    assert Chi(oo) is oo458    assert Chi(-oo) is oo459    assert mytd(Ci(x), cos(x)/x, x)460    assert mytd(Chi(x), cosh(x)/x, x)461    assert mytn(Ci(x), Ci(x).rewrite(Ei),462                Ei(x*exp_polar(-I*pi/2))/2 + Ei(x*exp_polar(I*pi/2))/2, x)463    assert mytn(Chi(x), Chi(x).rewrite(Ei),464                Ei(x)/2 + Ei(x*exp_polar(I*pi))/2 - I*pi/2, x)465    assert tn_arg(Ci)466    assert tn_arg(Chi)467    from sympy import O, EulerGamma, log, limit468    assert Ci(x).nseries(x, n=4) == \469        EulerGamma + log(x) - x**2/4 + x**4/96 + O(x**5)470    assert Chi(x).nseries(x, n=4) == \471        EulerGamma + log(x) + x**2/4 + x**4/96 + O(x**5)472    assert limit(log(x) - Ci(2*x), x, 0) == -log(2) - EulerGamma473    assert Ci(x).rewrite(uppergamma) == -expint(1, x*exp_polar(-I*pi/2))/2 -\474                                        expint(1, x*exp_polar(I*pi/2))/2475    assert Ci(x).rewrite(expint) == -expint(1, x*exp_polar(-I*pi/2))/2 -\476                                        expint(1, x*exp_polar(I*pi/2))/2477    raises(ArgumentIndexError, lambda: Ci(x).fdiff(2))478def test_fresnel():479    assert fresnels(0) == 0480    assert fresnels(oo) == S.Half481    assert fresnels(-oo) == Rational(-1, 2)482    assert fresnels(I*oo) == -I*S.Half483    assert unchanged(fresnels, z)484    assert fresnels(-z) == -fresnels(z)485    assert fresnels(I*z) == -I*fresnels(z)486    assert fresnels(-I*z) == I*fresnels(z)487    assert conjugate(fresnels(z)) == fresnels(conjugate(z))488    assert fresnels(z).diff(z) == sin(pi*z**2/2)489    assert fresnels(z).rewrite(erf) == (S.One + I)/4 * (490        erf((S.One + I)/2*sqrt(pi)*z) - I*erf((S.One - I)/2*sqrt(pi)*z))491    assert fresnels(z).rewrite(hyper) == \492        pi*z**3/6 * hyper([Rational(3, 4)], [Rational(3, 2), Rational(7, 4)], -pi**2*z**4/16)493    assert fresnels(z).series(z, n=15) == \494        pi*z**3/6 - pi**3*z**7/336 + pi**5*z**11/42240 + O(z**15)495    assert fresnels(w).is_extended_real is True496    assert fresnels(w).is_finite is True497    assert fresnels(z).is_extended_real is None498    assert fresnels(z).is_finite is None499    assert fresnels(z).as_real_imag() == (fresnels(re(z) - I*im(z))/2 +500                    fresnels(re(z) + I*im(z))/2,501                    -I*(-fresnels(re(z) - I*im(z)) + fresnels(re(z) + I*im(z)))/2)502    assert fresnels(z).as_real_imag(deep=False) == (fresnels(re(z) - I*im(z))/2 +503                    fresnels(re(z) + I*im(z))/2,504                    -I*(-fresnels(re(z) - I*im(z)) + fresnels(re(z) + I*im(z)))/2)505    assert fresnels(w).as_real_imag() == (fresnels(w), 0)506    assert fresnels(w).as_real_imag(deep=True) == (fresnels(w), 0)507    assert fresnels(2 + 3*I).as_real_imag() == (508            fresnels(2 + 3*I)/2 + fresnels(2 - 3*I)/2,509            -I*(fresnels(2 + 3*I) - fresnels(2 - 3*I))/2510            )511    assert expand_func(integrate(fresnels(z), z)) == \512        z*fresnels(z) + cos(pi*z**2/2)/pi513    assert fresnels(z).rewrite(meijerg) == sqrt(2)*pi*z**Rational(9, 4) * \514        meijerg(((), (1,)), ((Rational(3, 4),),515        (Rational(1, 4), 0)), -pi**2*z**4/16)/(2*(-z)**Rational(3, 4)*(z**2)**Rational(3, 4))516    assert fresnelc(0) == 0517    assert fresnelc(oo) == S.Half518    assert fresnelc(-oo) == Rational(-1, 2)519    assert fresnelc(I*oo) == I*S.Half520    assert unchanged(fresnelc, z)521    assert fresnelc(-z) == -fresnelc(z)522    assert fresnelc(I*z) == I*fresnelc(z)523    assert fresnelc(-I*z) == -I*fresnelc(z)524    assert conjugate(fresnelc(z)) == fresnelc(conjugate(z))525    assert fresnelc(z).diff(z) == cos(pi*z**2/2)526    assert fresnelc(z).rewrite(erf) == (S.One - I)/4 * (527        erf((S.One + I)/2*sqrt(pi)*z) + I*erf((S.One - I)/2*sqrt(pi)*z))528    assert fresnelc(z).rewrite(hyper) == \529        z * hyper([Rational(1, 4)], [S.Half, Rational(5, 4)], -pi**2*z**4/16)530    assert fresnelc(w).is_extended_real is True531    assert fresnelc(z).as_real_imag() == \532        (fresnelc(re(z) - I*im(z))/2 + fresnelc(re(z) + I*im(z))/2,533         -I*(-fresnelc(re(z) - I*im(z)) + fresnelc(re(z) + I*im(z)))/2)534    assert fresnelc(z).as_real_imag(deep=False) == \535        (fresnelc(re(z) - I*im(z))/2 + fresnelc(re(z) + I*im(z))/2,536         -I*(-fresnelc(re(z) - I*im(z)) + fresnelc(re(z) + I*im(z)))/2)537    assert fresnelc(2 + 3*I).as_real_imag() == (538        fresnelc(2 - 3*I)/2 + fresnelc(2 + 3*I)/2,539         -I*(fresnelc(2 + 3*I) - fresnelc(2 - 3*I))/2540    )541    assert expand_func(integrate(fresnelc(z), z)) == \542        z*fresnelc(z) - sin(pi*z**2/2)/pi543    assert fresnelc(z).rewrite(meijerg) == sqrt(2)*pi*z**Rational(3, 4) * \544        meijerg(((), (1,)), ((Rational(1, 4),),545        (Rational(3, 4), 0)), -pi**2*z**4/16)/(2*(-z)**Rational(1, 4)*(z**2)**Rational(1, 4))546    from sympy.testing.randtest import verify_numerically547    verify_numerically(re(fresnels(z)), fresnels(z).as_real_imag()[0], z)548    verify_numerically(im(fresnels(z)), fresnels(z).as_real_imag()[1], z)549    verify_numerically(fresnels(z), fresnels(z).rewrite(hyper), z)550    verify_numerically(fresnels(z), fresnels(z).rewrite(meijerg), z)551    verify_numerically(re(fresnelc(z)), fresnelc(z).as_real_imag()[0], z)552    verify_numerically(im(fresnelc(z)), fresnelc(z).as_real_imag()[1], z)553    verify_numerically(fresnelc(z), fresnelc(z).rewrite(hyper), z)554    verify_numerically(fresnelc(z), fresnelc(z).rewrite(meijerg), z)555    raises(ArgumentIndexError, lambda: fresnels(z).fdiff(2))556    raises(ArgumentIndexError, lambda: fresnelc(z).fdiff(2))557    assert fresnels(x).taylor_term(-1, x) is S.Zero558    assert fresnelc(x).taylor_term(-1, x) is S.Zero559    assert fresnelc(x).taylor_term(1, x) == -pi**2*x**5/40560@slow561def test_fresnel_series():562    assert fresnelc(z).series(z, n=15) == \563        z - pi**2*z**5/40 + pi**4*z**9/3456 - pi**6*z**13/599040 + O(z**15)564    # issues 6510, 10102565    fs = (S.Half - sin(pi*z**2/2)/(pi**2*z**3)566        + (-1/(pi*z) + 3/(pi**3*z**5))*cos(pi*z**2/2))567    fc = (S.Half - cos(pi*z**2/2)/(pi**2*z**3)568        + (1/(pi*z) - 3/(pi**3*z**5))*sin(pi*z**2/2))...

test_rewrite.py

Source:test_rewrite.py

1# -*- coding: utf-8 -*-2# Copyright 2015 Google Inc. All Rights Reserved.3#4# Licensed under the Apache License, Version 2.0 (the "License");5# you may not use this file except in compliance with the License.6# You may obtain a copy of the License at7#8#     http://www.apache.org/licenses/LICENSE-2.09#10# Unless required by applicable law or agreed to in writing, software11# distributed under the License is distributed on an "AS IS" BASIS,12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.13# See the License for the specific language governing permissions and14# limitations under the License.15"""Integration tests for rewrite command."""16from __future__ import absolute_import17import logging18import os19import re20from boto.storage_uri import BucketStorageUri21from gslib.cs_api_map import ApiSelector22from gslib.encryption_helper import CryptoKeyWrapperFromKey23from gslib.gcs_json_api import GcsJsonApi24from gslib.project_id import PopulateProjectId25from gslib.tests.rewrite_helper import EnsureRewriteRestartCallbackHandler26from gslib.tests.rewrite_helper import EnsureRewriteResumeCallbackHandler27from gslib.tests.rewrite_helper import HaltingRewriteCallbackHandler28from gslib.tests.rewrite_helper import RewriteHaltException29import gslib.tests.testcase as testcase30from gslib.tests.testcase.integration_testcase import SkipForS331from gslib.tests.util import GenerationFromURI as urigen32from gslib.tests.util import ObjectToURI as suri33from gslib.tests.util import SetBotoConfigForTest34from gslib.tests.util import TEST_ENCRYPTION_KEY135from gslib.tests.util import TEST_ENCRYPTION_KEY236from gslib.tests.util import TEST_ENCRYPTION_KEY337from gslib.tests.util import TEST_ENCRYPTION_KEY438from gslib.tests.util import unittest39from gslib.tracker_file import DeleteTrackerFile40from gslib.tracker_file import GetRewriteTrackerFilePath41from gslib.util import DiscardMessagesQueue42from gslib.util import ONE_MIB43@SkipForS3('gsutil doesn\'t support S3 customer-supplied encryption keys.')44class TestRewrite(testcase.GsUtilIntegrationTestCase):45  """Integration tests for rewrite command."""46  def test_rewrite_missing_flag(self):47    """Tests rewrite with no transformation flag."""48    stderr = self.RunGsUtil(49        ['rewrite', '%s://some_url' % self.default_provider],50        return_stderr=True, expected_status=1)51    self.assertIn('command requires at least one transformation flag', stderr)52  def test_rewrite_generation_url(self):53    """Tests that rewrite fails on a URL that includes a generation."""54    if self.test_api == ApiSelector.XML:55      return unittest.skip('Rewrite API is only supported in JSON.')56    object_uri = self.CreateObject(contents='bar',57                                   encryption_key=TEST_ENCRYPTION_KEY1)58    generation = object_uri.generation59    stderr = self.RunGsUtil(60        ['rewrite', '-k', '%s#%s' % (suri(object_uri), generation)],61        return_stderr=True, expected_status=1)62    self.assertIn('"rewrite" called on URL with generation', stderr)63  def test_rewrite_missing_decryption_key(self):64    """Tests that rewrite fails when no decryption key matches."""65    if self.test_api == ApiSelector.XML:66      return unittest.skip('Rewrite API is only supported in JSON.')67    object_uri = self.CreateObject(object_name='foo', contents='bar',68                                   encryption_key=TEST_ENCRYPTION_KEY1)69    boto_config_for_test = [70        ('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY2),71        ('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY3)]72    with SetBotoConfigForTest(boto_config_for_test):73      stderr = self.RunGsUtil(['rewrite', '-k', suri(object_uri)],74                              return_stderr=True, expected_status=1)75      self.assertIn('No decryption key matches object %s' %76                    suri(object_uri), stderr)77  def test_rewrite_stdin_args(self):78    """Tests rewrite with arguments supplied on stdin."""79    if self.test_api == ApiSelector.XML:80      return unittest.skip('Rewrite API is only supported in JSON.')81    object_uri = self.CreateObject(contents='bar',82                                   encryption_key=TEST_ENCRYPTION_KEY1)83    stdin_arg = suri(object_uri)84    boto_config_for_test = [85        ('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY2),86        ('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY1)]87    with SetBotoConfigForTest(boto_config_for_test):88      self.RunGsUtil(['rewrite', '-k', '-I'], stdin=stdin_arg)89    self.AssertObjectUsesCSEK(stdin_arg, TEST_ENCRYPTION_KEY2)90  def test_rewrite_overwrite_acl(self):91    """Tests rewrite with the -O flag."""92    if self.test_api == ApiSelector.XML:93      return unittest.skip('Rewrite API is only supported in JSON.')94    object_uri = self.CreateObject(contents='bar',95                                   encryption_key=TEST_ENCRYPTION_KEY1)96    self.RunGsUtil(['acl', 'ch', '-u', 'AllUsers:R', suri(object_uri)])97    stdout = self.RunGsUtil(['acl', 'get', suri(object_uri)],98                            return_stdout=True)99    self.assertIn('allUsers', stdout)100    boto_config_for_test = [101        ('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY2),102        ('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY1)]103    with SetBotoConfigForTest(boto_config_for_test):104      self.RunGsUtil(['rewrite', '-k', '-O', suri(object_uri)])105    self.AssertObjectUsesCSEK(suri(object_uri), TEST_ENCRYPTION_KEY2)106    stdout = self.RunGsUtil(['acl', 'get', suri(object_uri)],107                            return_stdout=True)108    self.assertNotIn('allUsers', stdout)109  def test_rewrite_bucket_recursive(self):110    """Tests rewrite command recursively on a bucket."""111    if self.test_api == ApiSelector.XML:112      return unittest.skip('Rewrite API is only supported in JSON.')113    bucket_uri = self.CreateBucket()114    self._test_rewrite_key_rotation_bucket(115        bucket_uri, ['rewrite', '-k', '-r', suri(bucket_uri)])116  def test_parallel_rewrite_bucket_flat_wildcard(self):117    """Tests parallel rewrite command with a flat wildcard on a bucket."""118    if self.test_api == ApiSelector.XML:119      return unittest.skip('Rewrite API is only supported in JSON.')120    bucket_uri = self.CreateBucket()121    self._test_rewrite_key_rotation_bucket(122        bucket_uri, ['-m', 'rewrite', '-k', suri(bucket_uri, '**')])123  def _test_rewrite_key_rotation_bucket(self, bucket_uri, command_args):124    """Helper function for testing key rotation on a bucket.125    Args:126      bucket_uri: bucket StorageUri to use for the test.127      command_args: list of args to gsutil command.128    """129    object_contents = 'bar'130    object_uri1 = self.CreateObject(bucket_uri=bucket_uri,131                                    object_name='foo/foo',132                                    contents=object_contents,133                                    encryption_key=TEST_ENCRYPTION_KEY1)134    object_uri2 = self.CreateObject(bucket_uri=bucket_uri,135                                    object_name='foo/bar',136                                    contents=object_contents,137                                    encryption_key=TEST_ENCRYPTION_KEY2)138    object_uri3 = self.CreateObject(bucket_uri=bucket_uri,139                                    object_name='foo/baz',140                                    contents=object_contents,141                                    encryption_key=TEST_ENCRYPTION_KEY3)142    object_uri4 = self.CreateObject(bucket_uri=bucket_uri,143                                    object_name='foo/qux',144                                    contents=object_contents)145    # Rotate all keys to TEST_ENCRYPTION_KEY1.146    boto_config_for_test = [147        ('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY1),148        ('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY2),149        ('GSUtil', 'decryption_key2', TEST_ENCRYPTION_KEY3)]150    with SetBotoConfigForTest(boto_config_for_test):151      stderr = self.RunGsUtil(command_args, return_stderr=True)152      # Object one already has the correct key.153      self.assertIn('Skipping %s' % suri(object_uri1), stderr)154      # Other objects should be rotated.155      self.assertIn('Rotating', stderr)156    for object_uri_str in (suri(object_uri1), suri(object_uri2),157                           suri(object_uri3), suri(object_uri4)):158      self.AssertObjectUsesCSEK(object_uri_str, TEST_ENCRYPTION_KEY1)159    # Remove all encryption.160    boto_config_for_test2 = [161        ('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY1)]162    with SetBotoConfigForTest(boto_config_for_test2):163      stderr = self.RunGsUtil(command_args, return_stderr=True)164      self.assertIn('Decrypting', stderr)165    for object_uri_str in (suri(object_uri1), suri(object_uri2),166                           suri(object_uri3), suri(object_uri4)):167      self.AssertObjectUnencrypted(object_uri_str)168  def test_rewrite_seek_ahead(self):169    if self.test_api == ApiSelector.XML:170      return unittest.skip('Rewrite API is only supported in JSON.')171    object_uri = self.CreateObject(contents='bar',172                                   encryption_key=TEST_ENCRYPTION_KEY1)173    # Remove encryption174    boto_config_for_test = [175        ('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY1),176        ('GSUtil', 'task_estimation_threshold', '1'),177        ('GSUtil', 'task_estimation_force', 'True')]178    with SetBotoConfigForTest(boto_config_for_test):179      stderr = self.RunGsUtil(['-m', 'rewrite', '-k', suri(object_uri)],180                              return_stderr=True)181      self.assertIn(182          'Estimated work for this command: objects: 1, total size: 3', stderr)183  def test_rewrite_unintentional_key_rotation_fails(self):184    if self.test_api == ApiSelector.XML:185      return unittest.skip('Rewrite API is only supported in JSON.')186    encrypted_obj_uri = self.CreateObject(contents='bar',187                                          encryption_key=TEST_ENCRYPTION_KEY1)188    unencrypted_obj_uri = self.CreateObject(contents='bar')189    boto_config_for_test = [190        ('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY2),191        ('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY1)]192    with SetBotoConfigForTest(boto_config_for_test):193      # Executing rewrite without the -k flag should fail if your boto file has194      # a different encryption_key than was last used to encrypt the object.195      stderr = self.RunGsUtil(['rewrite', '-s', 'dra', suri(encrypted_obj_uri)],196                              return_stderr=True, expected_status=1)197      self.assertIn('EncryptionException', stderr)198      # Should also fail for a previously unencrypted object.199      stderr = self.RunGsUtil(200          ['rewrite', '-s', 'dra', suri(unencrypted_obj_uri)],201          return_stderr=True, expected_status=1)202      self.assertIn('EncryptionException', stderr)203  def test_rewrite_key_rotation_single_object(self):204    if self.test_api == ApiSelector.XML:205      return unittest.skip('Rewrite API is only supported in JSON.')206    object_uri = self.CreateObject(contents='bar',207                                   encryption_key=TEST_ENCRYPTION_KEY1)208    # Rotate key.209    boto_config_for_test = [210        ('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY2),211        ('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY1)]212    with SetBotoConfigForTest(boto_config_for_test):213      stderr = self.RunGsUtil(['rewrite', '-k', suri(object_uri)],214                              return_stderr=True)215      self.assertIn('Rotating', stderr)216    self.AssertObjectUsesCSEK(suri(object_uri), TEST_ENCRYPTION_KEY2)217    # Remove encryption.218    boto_config_for_test2 = [219        ('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY2)]220    with SetBotoConfigForTest(boto_config_for_test2):221      stderr = self.RunGsUtil(['rewrite', '-k', suri(object_uri)],222                              return_stderr=True)223      self.assertIn('Decrypting', stderr)224    self.AssertObjectUnencrypted(suri(object_uri))225  def test_rewrite_key_rotation_bucket_subdir(self):226    if self.test_api == ApiSelector.XML:227      return unittest.skip('Rewrite API is only supported in JSON.')228    bucket_uri = self.CreateBucket()229    object_contents = 'bar'230    rotate_subdir = suri(bucket_uri, 'bar')231    object_uri1 = self.CreateObject(bucket_uri=bucket_uri,232                                    object_name='foo/bar',233                                    contents=object_contents,234                                    encryption_key=TEST_ENCRYPTION_KEY1)235    object_uri2 = self.CreateObject(bucket_uri=bucket_uri,236                                    object_name='bar/foo',237                                    contents=object_contents,238                                    encryption_key=TEST_ENCRYPTION_KEY2)239    object_uri3 = self.CreateObject(bucket_uri=bucket_uri,240                                    object_name='bar/baz',241                                    contents=object_contents,242                                    encryption_key=TEST_ENCRYPTION_KEY3)243    object_uri4 = self.CreateObject(bucket_uri=bucket_uri,244                                    object_name='bar/qux',245                                    contents=object_contents)246    # Rotate subdir keys to TEST_ENCRYPTION_KEY3.247    boto_config_for_test = [248        ('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY3),249        ('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY2),250        ('GSUtil', 'decryption_key2', TEST_ENCRYPTION_KEY1)]251    self.AssertNObjectsInBucket(bucket_uri, 4)252    with SetBotoConfigForTest(boto_config_for_test):253      stderr = self.RunGsUtil(['rewrite', '-r', '-k', rotate_subdir],254                              return_stderr=True)255      self.assertIn('Rotating', stderr)  # Object 2.256      self.assertIn('Skipping %s' % suri(object_uri3), stderr)257      self.assertIn('Encrypting', stderr)  # Object 4.258    # First subdir should be unaffected.259    self.AssertObjectUsesCSEK(suri(object_uri1), TEST_ENCRYPTION_KEY1)260    for object_uri_str in (suri(object_uri2), suri(object_uri3),261                           suri(object_uri4)):262      self.AssertObjectUsesCSEK(object_uri_str, TEST_ENCRYPTION_KEY3)263    # Remove encryption in subdir.264    boto_config_for_test2 = [265        ('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY3)]266    with SetBotoConfigForTest(boto_config_for_test2):267      stderr = self.RunGsUtil(['rewrite', '-r', '-k', rotate_subdir],268                              return_stderr=True)269      self.assertIn('Decrypting', stderr)270    # First subdir should be unaffected.271    self.AssertObjectUsesCSEK(suri(object_uri1), TEST_ENCRYPTION_KEY1)272    for object_uri_str in (suri(object_uri2), suri(object_uri3),273                           suri(object_uri4)):274      self.AssertObjectUnencrypted(object_uri_str)275  def test_rewrite_with_nonkey_transform_works_when_key_is_unchanged(self):276    # Tests that when a valid transformation flag aside from "-k" is supplied,277    # the "-k" flag is not supplied, and the encryption key previously used to278    # encrypt the target object matches the encryption_key in the user's boto279    # config file (via hash comparison), that the rewrite command properly280    # passes the same tuple for decryption and encryption, in addition to281    # performing the other desired transformations.282    if self.test_api == ApiSelector.XML:283      return unittest.skip('Rewrite API is only supported in JSON.')284    object_uri = self.CreateObject(contents='bar',285                                   encryption_key=TEST_ENCRYPTION_KEY1)286    boto_config_for_test = [('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY1)]287    with SetBotoConfigForTest(boto_config_for_test):288      stderr = self.RunGsUtil(289          ['rewrite', '-s', 'nearline', suri(object_uri)], return_stderr=True)290      self.assertIn('Rewriting', stderr)291  def test_rewrite_key_rotation_with_storage_class_change(self):292    if self.test_api == ApiSelector.XML:293      return unittest.skip('Rewrite API is only supported in JSON.')294    object_uri = self.CreateObject(contents='bar',295                                   encryption_key=TEST_ENCRYPTION_KEY1)296    # Rotate key and change storage class to nearline.297    boto_config_for_test = [298        ('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY2),299        ('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY1)]300    with SetBotoConfigForTest(boto_config_for_test):301      stderr = self.RunGsUtil(302          ['rewrite', '-s', 'nearline', '-k', suri(object_uri)],303          return_stderr=True)304      self.assertIn('Rotating', stderr)305    self.AssertObjectUsesCSEK(suri(object_uri), TEST_ENCRYPTION_KEY2)306    stdout = self.RunGsUtil(['stat', suri(object_uri)], return_stdout=True)307    self.assertRegexpMatchesWithFlags(308        stdout, r'Storage class:\s+NEARLINE', flags=re.IGNORECASE,309        msg=('Storage class appears not to have been changed.'))310  def test_rewrite_with_only_storage_class_change(self):311    if self.test_api == ApiSelector.XML:312      return unittest.skip('Rewrite API is only supported in JSON.')313    object_uri = self.CreateObject(contents='bar')314    # Change storage class to nearline.315    stderr = self.RunGsUtil(['rewrite', '-s', 'nearline', suri(object_uri)],316                            return_stderr=True)317    self.assertIn('Rewriting', stderr)318    stdout = self.RunGsUtil(['stat', suri(object_uri)], return_stdout=True)319    self.assertRegexpMatchesWithFlags(320        stdout, r'Storage class:\s+NEARLINE', flags=re.IGNORECASE,321        msg=('Storage class appears not to have been changed.'))322  def test_rewrite_to_same_storage_class_is_skipped(self):323    if self.test_api == ApiSelector.XML:324      return unittest.skip('Rewrite API is only supported in JSON.')325    object_uri = self.CreateObject(contents='bar')326    stderr = self.RunGsUtil(['rewrite', '-s', 'standard', suri(object_uri)],327                            return_stderr=True)328    self.assertIn('Skipping %s' % suri(object_uri), stderr)329  def test_rewrite_with_same_key_and_storage_class_is_skipped(self):330    if self.test_api == ApiSelector.XML:331      return unittest.skip('Rewrite API is only supported in JSON.')332    object_uri = self.CreateObject(contents='foo',333                                   encryption_key=TEST_ENCRYPTION_KEY1,334                                   storage_class='standard')335    boto_config_for_test = [('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY1)]336    with SetBotoConfigForTest(boto_config_for_test):337      stderr = self.RunGsUtil(338          ['rewrite', '-k', '-s', 'standard', suri(object_uri)],339          return_stderr=True)340    self.assertIn('Skipping %s' % suri(object_uri), stderr)341  def test_rewrite_with_no_value_for_minus_s(self):342    if self.test_api == ApiSelector.XML:343      return unittest.skip('Rewrite API is only supported in JSON.')344    stderr = self.RunGsUtil(['rewrite', '-s', 'gs://some-random-name'],345                            return_stderr=True, expected_status=1)346    self.assertIn('CommandException', stderr)347    self.assertIn('expects at least one URL', stderr)348  def test_rewrite_resume(self):349    self._test_rewrite_resume_or_restart(350        TEST_ENCRYPTION_KEY1, TEST_ENCRYPTION_KEY2)351  def test_rewrite_resume_restart_source_encryption_changed(self):352    self._test_rewrite_resume_or_restart(353        TEST_ENCRYPTION_KEY1, TEST_ENCRYPTION_KEY2,354        new_dec_key=TEST_ENCRYPTION_KEY3)355  def test_rewrite_resume_restart_dest_encryption_changed(self):356    self._test_rewrite_resume_or_restart(357        TEST_ENCRYPTION_KEY1, TEST_ENCRYPTION_KEY2,358        new_enc_key=TEST_ENCRYPTION_KEY3)359  def test_rewrite_resume_restart_both_encryption_changed(self):360    self._test_rewrite_resume_or_restart(361        TEST_ENCRYPTION_KEY1, TEST_ENCRYPTION_KEY2,362        new_dec_key=TEST_ENCRYPTION_KEY3, new_enc_key=TEST_ENCRYPTION_KEY4)363  def authorize_project_to_use_testing_kms_key(364      self, key_name=testcase.KmsTestingResources.CONSTANT_KEY_NAME):365    # Make sure our keyRing and cryptoKey exist.366    keyring_fqn = self.kms_api.CreateKeyRing(367        PopulateProjectId(None), testcase.KmsTestingResources.KEYRING_NAME,368        location=testcase.KmsTestingResources.KEYRING_LOCATION)369    key_fqn = self.kms_api.CreateCryptoKey(keyring_fqn, key_name)370    # Make sure that the service account for our default project is authorized371    # to use our test KMS key.372    self.RunGsUtil(['kms', 'authorize', '-k', key_fqn])373    return key_fqn374  def test_rewrite_to_kms_then_unencrypted(self):375    if self.test_api == ApiSelector.XML:376      return unittest.skip('Rewrite API is only supported in JSON.')377    key_fqn = self.authorize_project_to_use_testing_kms_key()378    object_uri = self.CreateObject(contents='foo')379    boto_config_for_test = [('GSUtil', 'encryption_key', key_fqn)]380    with SetBotoConfigForTest(boto_config_for_test):381      stderr = self.RunGsUtil(382          ['rewrite', '-k', suri(object_uri)], return_stderr=True)383    self.assertIn('Encrypting', stderr)384    self.AssertObjectUsesCMEK(suri(object_uri), key_fqn)385    # Rewrite back to unencrypted and make sure no KMS key was used.386    boto_config_for_test = [('GSUtil', 'encryption_key', None)]387    with SetBotoConfigForTest(boto_config_for_test):388      stderr = self.RunGsUtil(389          ['rewrite', '-k', suri(object_uri)], return_stderr=True)390    self.assertIn('Decrypting', stderr)391    self.AssertObjectUnencrypted(suri(object_uri))392  def test_rewrite_to_kms_then_csek(self):393    if self.test_api == ApiSelector.XML:394      return unittest.skip('Rewrite API is only supported in JSON.')395    key_fqn = self.authorize_project_to_use_testing_kms_key()396    object_uri = self.CreateObject(contents='foo')397    boto_config_for_test = [('GSUtil', 'encryption_key', key_fqn)]398    with SetBotoConfigForTest(boto_config_for_test):399      stderr = self.RunGsUtil(400          ['rewrite', '-k', suri(object_uri)], return_stderr=True)401    self.assertIn('Encrypting', stderr)402    self.AssertObjectUsesCMEK(suri(object_uri), key_fqn)403    # Rewrite from CMEK to CSEK encryption.404    boto_config_for_test = [('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY1)]405    with SetBotoConfigForTest(boto_config_for_test):406      stderr = self.RunGsUtil(407          ['rewrite', '-k', suri(object_uri)], return_stderr=True)408    self.assertIn('Rotating', stderr)409    self.AssertObjectUsesCSEK(suri(object_uri), TEST_ENCRYPTION_KEY1)410  def test_rewrite_to_csek_then_kms(self):411    if self.test_api == ApiSelector.XML:412      return unittest.skip('Rewrite API is only supported in JSON.')413    key_fqn = self.authorize_project_to_use_testing_kms_key()414    object_uri = self.CreateObject(contents='foo')415    boto_config_for_test = [('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY1)]416    with SetBotoConfigForTest(boto_config_for_test):417      stderr = self.RunGsUtil(418          ['rewrite', '-k', suri(object_uri)], return_stderr=True)419    self.assertIn('Encrypting', stderr)420    self.AssertObjectUsesCSEK(suri(object_uri), TEST_ENCRYPTION_KEY1)421    # Rewrite from CSEK to CMEK encryption.422    boto_config_for_test = [423        ('GSUtil', 'encryption_key', key_fqn),424        ('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY1),425    ]426    with SetBotoConfigForTest(boto_config_for_test):427      stderr = self.RunGsUtil(428          ['rewrite', '-k', suri(object_uri)], return_stderr=True)429    self.assertIn('Rotating', stderr)430    self.AssertObjectUsesCMEK(suri(object_uri), key_fqn)431  def _test_rewrite_resume_or_restart(self, initial_dec_key, initial_enc_key,432                                      new_dec_key=None, new_enc_key=None):433    """Tests that the rewrite command restarts if the object's key changed.434    Args:435      initial_dec_key: Initial key the object is encrypted with, used as436          decryption key in the first rewrite call.437      initial_enc_key: Initial encryption key to rewrite the object with,438          used as encryption key in the first rewrite call.439      new_dec_key: Decryption key for the second rewrite call; if specified,440          object will be overwritten with a new encryption key in between441          the first and second rewrite calls, and this key will be used for442          the second rewrite call.443      new_enc_key: Encryption key for the second rewrite call; if specified,444          this key will be used for the second rewrite call, otherwise the445          initial key will be used.446    Returns:447      None448    """449    if self.test_api == ApiSelector.XML:450      return unittest.skip('Rewrite API is only supported in JSON.')451    bucket_uri = self.CreateBucket()452    # maxBytesPerCall must be >= 1 MiB, so create an object > 2 MiB because we453    # need 2 response from the service: 1 success, 1 failure prior to454    # completion.455    object_uri = self.CreateObject(bucket_uri=bucket_uri, object_name='foo',456                                   contents=('12'*ONE_MIB) + 'bar',457                                   prefer_json_api=True,458                                   encryption_key=initial_dec_key)459    gsutil_api = GcsJsonApi(BucketStorageUri, logging.getLogger(),460                            DiscardMessagesQueue(), self.default_provider)461    with SetBotoConfigForTest(462        [('GSUtil', 'decryption_key1', initial_dec_key)]):463      src_obj_metadata = gsutil_api.GetObjectMetadata(464          object_uri.bucket_name, object_uri.object_name,465          provider=self.default_provider, fields=['bucket', 'contentType',466                                                  'etag', 'name'])467    dst_obj_metadata = src_obj_metadata468    tracker_file_name = GetRewriteTrackerFilePath(469        src_obj_metadata.bucket, src_obj_metadata.name,470        dst_obj_metadata.bucket, dst_obj_metadata.name, self.test_api)471    decryption_tuple = CryptoKeyWrapperFromKey(initial_dec_key)472    decryption_tuple2 = CryptoKeyWrapperFromKey(new_dec_key or initial_dec_key)473    encryption_tuple = CryptoKeyWrapperFromKey(initial_enc_key)474    encryption_tuple2 = CryptoKeyWrapperFromKey(new_enc_key or initial_enc_key)475    try:476      try:477        gsutil_api.CopyObject(478            src_obj_metadata, dst_obj_metadata,479            progress_callback=HaltingRewriteCallbackHandler(ONE_MIB*2).call,480            max_bytes_per_call=ONE_MIB, decryption_tuple=decryption_tuple,481            encryption_tuple=encryption_tuple)482        self.fail('Expected RewriteHaltException.')483      except RewriteHaltException:484        pass485      # Tracker file should be left over.486      self.assertTrue(os.path.exists(tracker_file_name))487      if new_dec_key:488        # Recreate the object with a different encryption key.489        self.CreateObject(490            bucket_uri=bucket_uri, object_name='foo',491            contents=('12'*ONE_MIB) + 'bar', prefer_json_api=True,492            encryption_key=new_dec_key,493            gs_idempotent_generation=urigen(object_uri))494      with SetBotoConfigForTest([495          ('GSUtil', 'decryption_key1', new_dec_key or initial_dec_key)]):496        original_md5 = gsutil_api.GetObjectMetadata(497            src_obj_metadata.bucket, src_obj_metadata.name,498            fields=['customerEncryption', 'md5Hash']).md5Hash499      if new_dec_key or new_enc_key:500        # Keys changed, rewrite should be restarted.501        progress_callback = EnsureRewriteRestartCallbackHandler(ONE_MIB).call502      else:503        # Keys are the same, rewrite should be resumed.504        progress_callback = EnsureRewriteResumeCallbackHandler(ONE_MIB*2).call505      # Now resume. Callback ensures the appropriate resume/restart behavior.506      gsutil_api.CopyObject(507          src_obj_metadata, dst_obj_metadata,508          progress_callback=progress_callback, max_bytes_per_call=ONE_MIB,509          decryption_tuple=decryption_tuple2,510          encryption_tuple=encryption_tuple2)511      # Copy completed; tracker file should be deleted.512      self.assertFalse(os.path.exists(tracker_file_name))513      final_enc_key = new_enc_key or initial_enc_key514      with SetBotoConfigForTest([515          ('GSUtil', 'encryption_key', final_enc_key)]):516        self.assertEqual(517            original_md5,518            gsutil_api.GetObjectMetadata(dst_obj_metadata.bucket,519                                         dst_obj_metadata.name,520                                         fields=['customerEncryption',521                                                 'md5Hash']).md5Hash,522            'Error: Rewritten object\'s hash doesn\'t match source object.')523    finally:524      # Clean up if something went wrong....

rewrite.py

Source:rewrite.py

...212def do_rewrite_impl(d):213    if 'rewrite_ignore' in d.attrs:214        yield '#[allow(unused)]'215        yield 'impl Rewrite for %s {' % d.name216        yield '  fn rewrite(old: &Self, new: &Self, mut rcx: RewriteCtxtRef) -> bool {'217        yield '    // Rewrite mode: ignore'218        yield '    true'219        yield '  }'220        yield '}'221        return222    yield '#[allow(unused)]'223    yield 'impl Rewrite for %s {' % d.name224    yield '  fn rewrite(old: &Self, new: &Self, mut rcx: RewriteCtxtRef) -> bool {'225    if has_field(d, 'id'):226        yield '    trace!("{:?}: rewrite: begin (%s)", new.id);' % d.name227    for strat in get_rewrite_strategies(d):228        yield '    let mark = rcx.mark();'229        if has_field(d, 'id'):230            yield '    trace!("{:?}: rewrite: try %s", new.id);' % strat231        yield '    let ok = strategy::%s::rewrite(old, new, rcx.borrow());' % strat232        yield '    if ok {'233        if has_field(d, 'id'):234            yield '      trace!("{:?}: rewrite: %s succeeded", new.id);' % strat235        yield '      return true;'236        yield '    } else {'237        if has_field(d, 'id'):238            yield '      trace!("{:?}: rewrite: %s FAILED", new.id);' % strat239        yield '      rcx.rewind(mark);'240        yield '    }'241        yield ''242    if has_field(d, 'id'):243        yield '    trace!("{:?}: rewrite: ran out of strategies!", new.id);'244    yield '    false'245    yield '  }'246    yield '}'247@linewise248def generate_rewrite_impls(decls):249    yield '// AUTOMATICALLY GENERATED - DO NOT EDIT'250    yield '// Produced %s by process_ast.py' % (datetime.now(),)251    yield ''252    for d in decls:253        if type_needs_generated_impl(d, 'Rewrite'):254            yield do_rewrite_impl(d)255@linewise256def do_recursive_body(se, target1, target2):257    contains_expr = 'prec_contains_expr' in se.attrs258    yield 'match (%s, %s) {' % (target1, target2)259    for v, path in variants_paths(se):260        yield '  (&%s,' % struct_pattern(v, path, '1')261        yield '   &%s) => {' % struct_pattern(v, path, '2')262        for f in v.fields:263            if 'rewrite_ignore' in f.attrs:264                continue265            # Figure out what function to call to rewrite this field266            seq_rewrite_mode = f.attrs.get('seq_rewrite')267            if seq_rewrite_mode is None and 'seq_rewrite_outer_span' in f.attrs:268                seq_rewrite_mode = ''   # enabled, default mode269            if seq_rewrite_mode is None:270                mk_rewrite = lambda old, new: \271                        'Rewrite::rewrite({old}, {new}, rcx.borrow())'.format(272                                old=old, new=new)273            else:274                outer_span_expr = f.attrs.get('seq_rewrite_outer_span')275                if outer_span_expr is not None:276                    # Replace `self.foo` with `foo2`, since we want the *old*277                    # outer span.278                    outer_span_expr = rewrite_field_expr(outer_span_expr, '%s2')279                else:280                    outer_span_expr = 'DUMMY_SP'281                mk_rewrite = lambda old, new: \282                        'rewrite_seq({old}, {new}, {outer}, rcx.borrow())'.format(283                                old=old, new=new, outer=outer_span_expr)284            # Generate the code for the recursive call, including expr285            # precedence bookkeeping.286            yield '    ({'287            if 'prec_first' in f.attrs:288                yield '      let old = rcx.replace_expr_prec(%s);' % \289                        field_prec_expr(f, True)290                yield '      let ok = Rewrite::rewrite(&%s1[0], &%s2[0], ' \291                        'rcx.borrow());' % (f.name, f.name)292                yield '      rcx.replace_expr_prec(%s);' % field_prec_expr(f, False)293                rewrite_expr = mk_rewrite('&%s1[1..]' % f.name, '&%s2[1..]' % f.name)294                yield '      let ok = ok && %s;' % rewrite_expr295                yield '      rcx.replace_expr_prec(old);'296                yield '      ok'297            else:298                if contains_expr:299                    yield '      let old = rcx.replace_expr_prec(%s);' % \300                            field_prec_expr(f, False)301                rewrite_expr = mk_rewrite('%s1' % f.name, '%s2' % f.name)302                yield '      let ok = %s;' % rewrite_expr303                if contains_expr:304                    yield '      rcx.replace_expr_prec(old);'305                yield '      ok'306            yield '    }) &&'307        yield '    true'308        yield '  }'309    yield '  (_, _) => false,'310    yield '}'311@linewise312def do_recursive_impl(d):313    if 'rewrite_ignore' in d.attrs:314        yield '#[allow(unused)]'315        yield 'impl Recursive for %s {' % d.name...

update_version.py

Source:update_version.py

1#!/usr/bin/env python2# Usage: ./update_version.py <MAJOR>.<MINOR>.<MICRO> [<RC version>]3#4# Example:5# ./update_version.py 3.7.1 26#   => Version will become 3.7.1-rc-2 (beta)7# ./update_version.py 3.7.18#   => Version will become 3.7.1 (stable)9import datetime10import re11import sys12from xml.dom import minidom13if len(sys.argv) < 2 or len(sys.argv) > 3:14  print """15[ERROR] Please specify a version.16./update_version.py <MAJOR>.<MINOR>.<MICRO> [<RC version>]17Example:18./update_version.py 3.7.1 219"""20  exit(1)21NEW_VERSION = sys.argv[1]22NEW_VERSION_INFO = NEW_VERSION.split('.')23if len(NEW_VERSION_INFO) != 3:24  print """25[ERROR] Version must be in the format <MAJOR>.<MINOR>.<MICRO>26Example:27./update_version.py 3.7.328"""29  exit(1)30RC_VERSION = 031if len(sys.argv) > 2:32  RC_VERSION = int(sys.argv[2])33def Find(elem, tagname):34  for child in elem.childNodes:35    if child.nodeName == tagname:36      return child37  return None38def FindAndClone(elem, tagname):39  return Find(elem, tagname).cloneNode(True)40def ReplaceText(elem, text):41  elem.firstChild.replaceWholeText(text)42def GetFullVersion(rc_suffix = '-rc-'):43  if RC_VERSION == 0:44    return NEW_VERSION45  else:46    return '%s%s%s' % (NEW_VERSION, rc_suffix, RC_VERSION)47def RewriteXml(filename, rewriter, add_xml_prefix=True):48  document = minidom.parse(filename)49  rewriter(document)50  # document.toxml() always prepend the XML version without inserting new line.51  # We wants to preserve as much of the original formatting as possible, so we52  # will remove the default XML version and replace it with our custom one when53  # whever necessary.54  content = document.toxml().replace('<?xml version="1.0" ?>', '')55  file_handle = open(filename, 'wb')56  if add_xml_prefix:57    file_handle.write('<?xml version="1.0" encoding="UTF-8"?>\n')58  file_handle.write(content)59  file_handle.write('\n')60  file_handle.close()61def RewriteTextFile(filename, line_rewriter):62  lines = open(filename, 'r').readlines()63  updated_lines = []64  for line in lines:65    updated_lines.append(line_rewriter(line))66  if lines == updated_lines:67    print '%s was not updated. Please double check.' % filename68  f = open(filename, 'w')69  f.write(''.join(updated_lines))70  f.close()71def UpdateConfigure():72  RewriteTextFile('configure.ac',73    lambda line : re.sub(74      r'^AC_INIT\(\[Protocol Buffers\],\[.*\],\[protobuf@googlegroups.com\],\[protobuf\]\)$',75      ('AC_INIT([Protocol Buffers],[%s],[protobuf@googlegroups.com],[protobuf])'76        % GetFullVersion()),77      line))78def UpdateCpp():79  cpp_version = '%s00%s00%s' % (80    NEW_VERSION_INFO[0], NEW_VERSION_INFO[1], NEW_VERSION_INFO[2])81  def RewriteCommon(line):82    line = re.sub(83      r'^#define GOOGLE_PROTOBUF_VERSION .*$',84      '#define GOOGLE_PROTOBUF_VERSION %s' % cpp_version,85      line)86    line = re.sub(87      r'^#define PROTOBUF_VERSION .*$',88      '#define PROTOBUF_VERSION %s' % cpp_version,89      line)90    if NEW_VERSION_INFO[2] == '0':91      line = re.sub(92        r'^#define PROTOBUF_MIN_HEADER_VERSION_FOR_PROTOC .*$',93        '#define PROTOBUF_MIN_HEADER_VERSION_FOR_PROTOC %s' % cpp_version,94        line)95      line = re.sub(96        r'^#define GOOGLE_PROTOBUF_MIN_PROTOC_VERSION .*$',97        '#define GOOGLE_PROTOBUF_MIN_PROTOC_VERSION %s' % cpp_version,98        line)99      line = re.sub(100        r'^static const int kMinHeaderVersionForLibrary = .*$',101        'static const int kMinHeaderVersionForLibrary = %s;' % cpp_version,102        line)103      line = re.sub(104        r'^static const int kMinHeaderVersionForProtoc = .*$',105        'static const int kMinHeaderVersionForProtoc = %s;' % cpp_version,106        line)107    return line108  def RewritePortDef(line):109    line = re.sub(110      r'^#define PROTOBUF_VERSION .*$',111      '#define PROTOBUF_VERSION %s' % cpp_version,112      line)113    if NEW_VERSION_INFO[2] == '0':114      line = re.sub(115        r'^#define PROTOBUF_MIN_HEADER_VERSION_FOR_PROTOC .*$',116        '#define PROTOBUF_MIN_HEADER_VERSION_FOR_PROTOC %s' % cpp_version,117        line)118      line = re.sub(119        r'^#define PROTOBUF_MIN_PROTOC_VERSION .*$',120        '#define PROTOBUF_MIN_PROTOC_VERSION %s' % cpp_version,121        line)122      line = re.sub(123        r'^#define GOOGLE_PROTOBUF_MIN_LIBRARY_VERSION .*$',124        '#define GOOGLE_PROTOBUF_MIN_LIBRARY_VERSION %s' % cpp_version,125        line)126    return line127  def RewritePbH(line):128    line = re.sub(129        r'^#if PROTOBUF_VERSION < .*$',130        '#if PROTOBUF_VERSION < %s' % cpp_version,131        line)132    line = re.sub(133        r'^#if .* < PROTOBUF_MIN_PROTOC_VERSION$',134        '#if %s < PROTOBUF_MIN_PROTOC_VERSION' % cpp_version,135        line)136    return line137    138  RewriteTextFile('src/google/protobuf/stubs/common.h', RewriteCommon)139  RewriteTextFile('src/google/protobuf/port_def.inc', RewritePortDef)140  RewriteTextFile('src/google/protobuf/any.pb.h', RewritePbH)141  RewriteTextFile('src/google/protobuf/api.pb.h', RewritePbH)142  RewriteTextFile('src/google/protobuf/descriptor.pb.h', RewritePbH)143  RewriteTextFile('src/google/protobuf/duration.pb.h', RewritePbH)144  RewriteTextFile('src/google/protobuf/empty.pb.h', RewritePbH)145  RewriteTextFile('src/google/protobuf/field_mask.pb.h', RewritePbH)146  RewriteTextFile('src/google/protobuf/source_context.pb.h', RewritePbH)147  RewriteTextFile('src/google/protobuf/struct.pb.h', RewritePbH)148  RewriteTextFile('src/google/protobuf/timestamp.pb.h', RewritePbH)149  RewriteTextFile('src/google/protobuf/type.pb.h', RewritePbH)150  RewriteTextFile('src/google/protobuf/wrappers.pb.h', RewritePbH)151  RewriteTextFile('src/google/protobuf/compiler/plugin.pb.h', RewritePbH)152def UpdateCsharp():153  RewriteXml('csharp/src/Google.Protobuf/Google.Protobuf.csproj',154    lambda document : ReplaceText(155      Find(Find(document.documentElement, 'PropertyGroup'), 'VersionPrefix'),156      GetFullVersion(rc_suffix = '-rc')),157    add_xml_prefix=False)158  RewriteXml('csharp/Google.Protobuf.Tools.nuspec',159    lambda document : ReplaceText(160      Find(Find(document.documentElement, 'metadata'), 'version'),161      GetFullVersion(rc_suffix = '-rc')))162def UpdateJava():163  RewriteXml('java/pom.xml',164    lambda document : ReplaceText(165      Find(document.documentElement, 'version'), GetFullVersion()))166  RewriteXml('java/bom/pom.xml',167    lambda document : ReplaceText(168      Find(document.documentElement, 'version'), GetFullVersion()))169  RewriteXml('java/core/pom.xml',170    lambda document : ReplaceText(171      Find(Find(document.documentElement, 'parent'), 'version'),172      GetFullVersion()))173  RewriteXml('java/lite/pom.xml',174    lambda document : ReplaceText(175      Find(Find(document.documentElement, 'parent'), 'version'),176      GetFullVersion()))177  RewriteXml('java/util/pom.xml',178    lambda document : ReplaceText(179      Find(Find(document.documentElement, 'parent'), 'version'),180      GetFullVersion()))181  RewriteXml('protoc-artifacts/pom.xml',182    lambda document : ReplaceText(183      Find(document.documentElement, 'version'), GetFullVersion()))184def UpdateJavaScript():185  RewriteTextFile('js/package.json',186    lambda line : re.sub(187      r'^  "version": ".*",$',188      '  "version": "%s",' % GetFullVersion(rc_suffix = '-rc.'),189      line))190def UpdateMakefile():191  protobuf_version_offset = 11192  expected_major_version = '3'193  if NEW_VERSION_INFO[0] != expected_major_version:194    print """[ERROR] Major protobuf version has changed. Please update195update_version.py to readjust the protobuf_version_offset and196expected_major_version such that the PROTOBUF_VERSION in src/Makefile.am is197always increasing.198    """199    exit(1)200  protobuf_version_info = '%s:%s:0' % (201    int(NEW_VERSION_INFO[1]) + protobuf_version_offset, NEW_VERSION_INFO[2])202  RewriteTextFile('src/Makefile.am',203    lambda line : re.sub(204      r'^PROTOBUF_VERSION = .*$',205      'PROTOBUF_VERSION = %s' % protobuf_version_info,206      line))207def UpdateObjectiveC():208  RewriteTextFile('Protobuf.podspec',209    lambda line : re.sub(210      r"^  s.version  = '.*'$",211      "  s.version  = '%s'" % GetFullVersion(rc_suffix = '-rc'),212      line))213def UpdatePhp():214  def Callback(document):215    def CreateNode(tagname, indent, children):216      elem = document.createElement(tagname)217      indent += 1218      for child in children:219        elem.appendChild(document.createTextNode('\n' + (' ' * indent)))220        elem.appendChild(child)221      indent -= 1222      elem.appendChild(document.createTextNode('\n' + (' ' * indent)))223      return elem224    root = document.documentElement225    now = datetime.datetime.now()226    ReplaceText(Find(root, 'date'), now.strftime('%Y-%m-%d'))227    ReplaceText(Find(root, 'time'), now.strftime('%H:%M:%S'))228    version = Find(root, 'version')229    ReplaceText(Find(version, 'release'), GetFullVersion(rc_suffix = 'RC'))230    ReplaceText(Find(version, 'api'), NEW_VERSION)231    stability = Find(root, 'stability')232    ReplaceText(Find(stability, 'release'),233        'stable' if RC_VERSION == 0 else 'beta')234    ReplaceText(Find(stability, 'api'), 'stable' if RC_VERSION == 0 else 'beta')235    changelog = Find(root, 'changelog')236    for old_version in changelog.getElementsByTagName('version'):237      if Find(old_version, 'release').firstChild.nodeValue == NEW_VERSION:238        print ('[WARNING] Version %s already exists in the change log.'239          % NEW_VERSION)240        return241    changelog.appendChild(document.createTextNode(' '))242    release = CreateNode('release', 2, [243        CreateNode('version', 3, [244          FindAndClone(version, 'release'),245          FindAndClone(version, 'api')246        ]),247        CreateNode('stability', 3, [248          FindAndClone(stability, 'release'),249          FindAndClone(stability, 'api')250        ]),251        FindAndClone(root, 'date'),252        FindAndClone(root, 'time'),253        FindAndClone(root, 'license'),254        FindAndClone(root, 'notes')255      ])256    changelog.appendChild(release)257    changelog.appendChild(document.createTextNode('\n '))258  RewriteXml('php/ext/google/protobuf/package.xml', Callback)259  RewriteTextFile('php/ext/google/protobuf/protobuf.h',260    lambda line : re.sub(261      r'PHP_PROTOBUF_VERSION ".*"$',262      'PHP_PROTOBUF_VERSION "%s"' % NEW_VERSION,263      line))264  RewriteTextFile('php/ext/google/protobuf/protobuf.h',265    lambda line : re.sub(266      r"^#define PHP_PROTOBUF_VERSION .*$",267      "#define PHP_PROTOBUF_VERSION \"%s\"" % GetFullVersion(rc_suffix = 'RC'),268      line))269  RewriteTextFile('php/ext/google/protobuf/protobuf.h',270    lambda line : re.sub(271      r"^#define PHP_PROTOBUF_VERSION .*$",272      "#define PHP_PROTOBUF_VERSION \"%s\"" % GetFullVersion(rc_suffix = 'RC'),273      line))274def UpdatePython():275  RewriteTextFile('python/google/protobuf/__init__.py',276    lambda line : re.sub(277      r"^__version__ = '.*'$",278      "__version__ = '%s'" % GetFullVersion(rc_suffix = 'rc'),279      line))280def UpdateRuby():281  RewriteTextFile('ruby/google-protobuf.gemspec',282    lambda line : re.sub(283      r'^  s.version     = ".*"$',284      '  s.version     = "%s"' % GetFullVersion(rc_suffix = '.rc.'),285      line))286UpdateConfigure()287UpdateCsharp()288UpdateCpp()289UpdateJava()290UpdateJavaScript()291UpdateMakefile()292UpdateObjectiveC()293UpdatePhp()294UpdatePython()...

Pytest Tutorial

Looking for an in-depth tutorial around pytest? LambdaTest covers the detailed pytest tutorial that has everything related to the pytest, from setting up the pytest framework to automation testing. Delve deeper into pytest testing by exploring advanced use cases like parallel testing, pytest fixtures, parameterization, executing multiple test cases from a single file, and more.

Chapters

What is pytest
Pytest installation: Want to start pytest from scratch? See how to install and configure pytest for Python automation testing.
Run first test with pytest framework: Follow this step-by-step tutorial to write and run your first pytest script.
Parallel testing with pytest: A hands-on guide to parallel testing with pytest to improve the scalability of your test automation.
Generate pytest reports: Reports make it easier to understand the results of pytest-based test runs. Learn how to generate pytest reports.
Pytest Parameterized tests: Create and run your pytest scripts while avoiding code duplication and increasing test coverage with parameterization.
Pytest Fixtures: Check out how to implement pytest fixtures for your end-to-end testing needs.
Execute Multiple Test Cases: Explore different scenarios for running multiple test cases in pytest from a single file.
Stop Test Suite after N Test Failures: See how to stop your test suite after n test failures in pytest using the @pytest.mark.incremental decorator and maxfail command-line option.