How to use explain method in refurb

Best Python code snippet using refurb_python

DB.py

Source:DB.py

1#!/usr/bin/env python2#coding:utf-83import json4import os5import re6import datetime7import random8import django9import concurrent.futures10from regulation.models import law11from regulation.models import law_clause12from regulation.models import multi_version_law13from regulation.models import multi_version_law_clause14from regulation.models import explain15from regulation.models import explain_element16from regulation.models import solr_weibo_data17from regulation.models import matched_law_data18from regulation.models import matched_clause_data19from regulation.models import law_charts_data20from regulation.models import explain_charts_data21from regulation.models import random_selected_data22from regulation.models import judge_law_data23from regulation.models import real_law_data24from regulation.models import judge_clause_data25from regulation.models import real_clause_data26from regulation.models import alias27from regulation.models import stopword28def delete_db():29    # law.objects.all().delete()30    # law_clause.objects.all().delete()31    # explain.objects.all().delete()32    # explain_element.objects.all().delete()33    # solr_weibo_data.objects.all().delete()34    # solr_weibo_data.objects.filter(data_type = "law").delete()35    # solr_weibo_data.objects.all().delete()36    # matched_clause_data.objects.all().delete()37    # matched_law_data.objects.all().delete()38    # law_charts_data.objects.all().delete()39    # explain_charts_data.objects.all().delete()40    # random_selected_data.objects.all().delete()41    # real_law_data.objects.all().delete()42    # real_clause_data.objects.all().delete()43    print('delete db done!')44def find_effective_law():45    law_list = law.objects.all()46    for item in law_list:47        l = multi_version_law.objects.filter(law_name = item.law_name,timeliness = "ç°è¡ææ")48        if len(l) == 0:49            li = multi_version_law.objects.filter(law_name = item.law_name)50            if len(li)==0:51                print('ç¼ºå¤±æ³å¾:',item.law_name)52            else:53                print('æ ç°è¡ææ: ', item.law_name)54        if len(l) >1:55            print('å¤ä¸ªç°è¡ææï¼',item.law_name)56def add_lost_law():57    law_list = law.objects.all()58    multi_version_l = multi_version_law.objects.all()59    count = len(multi_version_l)60    for item in law_list:61        l = multi_version_law.objects.filter(law_name = item.law_name)62        if len(l)==0:63            print('ç¼ºå¤±æ³å¾:',item.law_name)64            count+=165            multi_version_law.objects.get_or_create(66                law_id=count,67                law_name = item.law_name,68                ful_name = item.law_name,#todo69                timeliness = item.timeliness,70                department = item.department,71                efficacyLevel = item.efficacyLevel,72                releaseDate = item.releaseDate,73                effectiveDate = item.effectiveDate,74                legalCategories = item.legalCategories,75                content = item.content,76                total = item.total77            )78            clause_list = law_clause.objects.filter(law_name=item.law_name)79            for e in clause_list:80                multi_version_law_clause.objects.get_or_create(81                    law_id = count,82                    clause_id = e.clause_id,83                    law_name = item.law_name,84                    ful_name = item.law_name,#todo85                    clause_name = e.clause_name,86                    content = e.content,87                    total=e.total88                )89def modify_ful_name():90    for count in range(567,590):91        item = multi_version_law.objects.filter(law_id = count)[0]    92        clause_list = multi_version_law_clause.objects.filter(law_id = count)93        for e in clause_list:94            e.ful_name = item.ful_name95            e.save()96def delete_law():97    for count in range(567,590):98        multi_version_law.objects.filter(law_id = count).delete()99def test_date():100    begin = datetime.date(2019,6,1)101    end = datetime.date(2019,6,2)102    d = begin103    delta = datetime.timedelta(days=1)104    while d <= end:105        l=solr_weibo_data.objects.filter(doc_date=d)106        print(len(l))107        d += delta108#ç»è®¡æ³å¾æ³æ¡ãå¸æ³è§£éççåº¦ï¼å½matched_clause_dataæ¹ååï¼éè¦éæ°ç»è®¡109def tongji(sy,sm,sd,ey,em,ed):110    print("sy = ", sy,'type = ',type(sy))111    print("sm = ", sm,'type = ',type(sm))112    print("sd = ", sd,'type = ',type(sd))113    print("ey = ", ey,'type = ',type(ey))114    print("em = ", em,'type = ',type(em))115    print("ed = ", ed,'type = ',type(ed))116    #ä»æ³å¾çº§å«å¹éç»æä¸æ¾å°çè®°å½ï¼éè¦æ¹è¿ä¸ºå¹éå½æ°æ¾å°çå¾®åæ°æ®    117    law_list = law.objects.all()118    explain_list = explain.objects.all()119    # clause_list = law_clause.objects.all()120    clause_list = multi_version_law_clause.objects.all()121    element_list = explain_element.objects.all()122    123    print('==========ç»è®¡æ³å¾==========')124    for law_obj in law_list:125        print('law = ', law_obj.law_name)126        l = matched_law_data.objects.filter(keyword_id = law_obj.law_id, data_type = 'law')127        num = len(l)128        law_obj.total = num129        origin_l = matched_law_data.objects.filter(keyword_id = law_obj.law_id, data_type = 'law', origin_tag=1)130        law_obj.origin_count = len(origin_l)131        law_obj.save()132    print('==========æ³å¾ç»è®¡ç»æ==========')133    134    print('==========ç»è®¡å¸æ³è§£é==========')135    for explain_obj in explain_list:136        print('explain = ', explain_obj.explain_name)137        l = matched_law_data.objects.filter(keyword_id = explain_obj.explain_id, data_type = 'explain')138        num = len(l)139        explain_obj.total = num140        origin_l = matched_law_data.objects.filter(keyword_id = explain_obj.explain_id, data_type = 'explain', origin_tag=1)141        explain_obj.origin_count = len(origin_l)142        explain_obj.save()143    print('==========å¸æ³ç»è®¡ç»æ==========')144    145    146    print('==========ç»è®¡æ³æ¡==========')147    for clause_obj in clause_list:148        print('ful_name = ', clause_obj.ful_name, ' clause = ', clause_obj.clause_name)149        l = matched_clause_data.objects.filter(data_type='law', keyword_id = clause_obj.law_id, element_id = clause_obj.clause_id)150        num = len(l)151        clause_obj.total = num152        origin_l = matched_clause_data.objects.filter(data_type='law', keyword_id = clause_obj.law_id, element_id = clause_obj.clause_id, origin_tag=1)153        clause_obj.origin_count = len(origin_l)154        clause_obj.save()155    print('==========æ³æ¡ç»è®¡ç»æ==========')156    157    print('==========ç»è®¡å¸æ³æ³æ¡==========')158    for element_obj in element_list:159        print('explain = ', element_obj.explain_name, ' element_obj = ', element_obj.element_name)160        l = matched_clause_data.objects.filter(data_type='explain', keyword_id = element_obj.explain_id, element_id=element_obj.element_id)161        element_obj.total = len(l)162        origin_l = matched_clause_data.objects.filter(data_type = 'explain', keyword_id=element_obj.explain_id, element_id=element_obj.element_id, origin_tag=1)163        element_obj.origin_count = len(origin_l)164        element_obj.save()165    print('==========ç»è®¡å¸æ³æ³æ¡ç»æ==========')166    167    print('==========ç»è®¡echartæ³å¾æ°æ®==========')168    law_count=0169    for law_obj in law_list:170        law_count+=1171        if law_count<128:172            continue173        print('law=', law_obj.law_name)174        begin = datetime.date(sy,sm,sd)175        end = datetime.date(ey,em,ed)176        d = begin177        delta = datetime.timedelta(days=1)178        while d < end:179            l=matched_law_data.objects.filter(data_type='law',keyword_name=law_obj.law_name,doc_date=d)180            origin_list = matched_law_data.objects.filter(data_type='law', keyword_name=law_obj.law_name,doc_date=d,origin_tag=1)181            test_list = law_charts_data.objects.filter(law_id=law_obj.law_id,law_name=law_obj.law_name,182                date=d)183            if len(test_list)>0:184                Obj = test_list[0]185                Obj.total=len(l)186                Obj.origin_count = len(origin_list)187                if len(origin_list)>0:188                    Obj.rate=len(l)/ float(len(origin_list))189                else:190                    Obj.rate=float(len(l))191                Obj.save()192            else:193                total_num = len(l)194                origin_num = len(origin_list)195                if origin_num > 0:196                    r = total_num / float(origin_num)197                else:198                    r = float(total_num)199                law_charts_data.objects.get_or_create(law_id=law_obj.law_id,law_name=law_obj.law_name,200                    date=d,total=total_num, origin_count = origin_num, rate = r)201            # print(d.strftime("%Y-%m-%d"))202            d += delta203    print('==========echartæ³å¾æ°æ®ç»è®¡ç»æ==========')204    205    print('==========ç»è®¡echartå¸æ³è§£éæ°æ®==========')206    for explain_obj in explain_list:207        print('explain=', explain_obj.explain_name)208        begin = datetime.date(sy,sm,sd)209        end = datetime.date(ey,em,ed)210        d = begin211        delta = datetime.timedelta(days=1)212        while d < end:213            l=matched_law_data.objects.filter(data_type='explain', keyword_name=explain_obj.explain_name, doc_date=d)214            origin_list = matched_law_data.objects.filter(data_type='explain', keyword_name=explain_obj.explain_name, doc_date=d, origin_tag=1)215            test_list = explain_charts_data.objects.filter(explain_id=explain_obj.explain_id,explain_name=explain_obj.explain_name,date=d)216            #å¦æå·²åå¨217            if len(test_list)>0:218                Obj = test_list[0]219                Obj.total=len(l)220                Obj.origin_count = len(origin_list)221                if len(origin_list)>0:222                    Obj.rate = len(l) / float(len(origin_list))223                else:224                    Obj.rate = float(len(l))225                Obj.save()226            else:227                total_num = len(l)228                origin_num = len(origin_list)229                if origin_num > 0:230                    r = total_num / float(origin_num)231                else:232                    r = float(total_num)233                explain_charts_data.objects.get_or_create(explain_id=explain_obj.explain_id,explain_name=explain_obj.explain_name,234                    date=d,total=total_num, origin_count = origin_num, rate = r)235            # print(d.strftime("%Y-%m-%d"))236            d += delta237    print('==========echartå¸æ³æ°æ®ç»è®¡ç»æ==========')238    239#æ·»å å¾®åé¾æ¥å°ä¸¤ä¸ªæ°æ®åºä¸240def add_url():241    l1=matched_clause_data.objects.all()242    l2=matched_law_data.objects.all()243    count=0244    for item in l1:245        count+=1246        if count % 1000 == 0:247            print('total = ',len(l1),'l1 count = ',count)248        temp_list=solr_weibo_data.objects.filter(weibo_id=item.weibo_id)249        item.weibo_link=temp_list[0].weibo_link250        item.save()251    count=0252    for item in l2:253        count+=1254        if count % 1000 == 0:255            print('total = ',len(l2),' l2 count = ',count)256        temp_list=solr_weibo_data.objects.filter(weibo_id=item.weibo_id)257        item.weibo_link=temp_list[0].weibo_link258        item.save()259def join():260    l=matched_clause_data.objects.all()261    print("total_len=",len(l))262    count=0263    for item in l:264        count+=1265        if count%1000==0:266            print('count=',count)267        temp_list=solr_weibo_data.objects.filter(weibo_id=item.weibo_id)268        if len(temp_list)>0:269            if len(temp_list)!=1:270                print(len(temp_list))271                print('weibo_id=',item.weibo_id)272                # for e in temp_list:273                #     print(e.author_name)274            temp=temp_list[0]275            item.user_type=temp.user_type276            item.author_name=temp.author_name277            item.tou_xiang=temp.tou_xiang278            item.doc_time=temp.doc_time279            item.doc_date=temp.doc_date280            item.doc_text=temp.doc_text281            item.weibo_source=temp.weibo_source282            item.opinion=temp.opinion283            item.month=temp.month284            item.save()285def add_alias():286    alias.objects.all().delete()287    law_list = law.objects.all()288    for item in law_list:289        if item.law_name =="ä¸åäººæ°å±åå½æ°´æ³" or item.law_name =="ä¸åäººæ°å±åå½ç»è®¡æ³":290            continue291        if item.law_name.startswith('ä¸åäººæ°å±åå½'):292            alias.objects.get_or_create(293                data_type="law",294                keyword_id=item.law_id,295                formal_name=item.law_name,296                alias_name=item.law_name[7:]297            )298    explain_list = explain.objects.all()299    for item in explain_list:300        if item.explain_name.startswith('æé«äººæ°æ³é¢'):301            alias.objects.get_or_create(302                data_type="explain",303                keyword_id=item.explain_id,304                formal_name=item.explain_name,305                alias_name=item.explain_name[6:]306            )307def select_random_weibo(num):308    print('delete ...')309    random_selected_data.objects.all().delete()310    real_law_data.objects.all().delete()311    real_clause_data.objects.all().delete()312    print('loading...')313    l = solr_weibo_data.objects.all()314    Len = len(l)315    print('Len = ',Len)316    index_list = []317    count = 0318    while count<num:319        # r = random.randint(496745,496745+Len-1)320        r = random.randint(0,Len-1)321        if r not in index_list:322            index_list.append(r)323            count+=1324    index_list = sorted(index_list)325    print('index_list = ',index_list)326    for index in index_list:327        temp_list = l[index:index+1]328        # temp_list = solr_weibo_data.objects.filter(id = index)329        if len(temp_list) >0:330            temp = temp_list[0]331            random_selected_data.objects.get_or_create(332                weibo_link = temp.weibo_link,333                weibo_id = temp.weibo_id,334                tou_xiang = temp.tou_xiang,335                author_name = temp.author_name,336                doc_text = temp.doc_text,337                weibo_source = temp.weibo_source,338                doc_date = temp.doc_date339            )340def trim_title():341    l = law.objects.all()342    for item in l:343        law_id = item.law_id344        name = item.law_name345        clauses = law_clause.objects.filter(law_id = law_id)346        for element in clauses:347            element.law_name = name348            element.save()349    l = explain.objects.all()350    for item in l:351        explain_id = item.explain_id352        name = item.explain_name353        elements = explain_element.objects.filter(explain_id = explain_id)354        for e in elements:355            e.explain_name = name356            e.save()357    l = law.objects.all()358    for item in l:359        law_id = item.law_id360        name = item.law_name361        data_list = matched_clause_data.objects.filter(keyword_id = law_id, data_type = 'law')362        for element in data_list:363            element.keyword_name = name364            element.save()365    l = explain.objects.all()366    for item in l:367        explain_id = item.explain_id368        name = item.explain_name369        data_list = matched_clause_data.objects.filter(keyword_id = explain_id, data_type = 'explain')370        for d in data_list:371            d.keyword_name = name372            d.save()373    # pass374def test():375    print('test    test !')376    # l = law_clause.objects.all()377    # for item in l:378    #     print('len = ', len(item.law_name))379    #     print('law_name =' + item.law_name+'|')380    #     new_law_name = item.law_name.strip()381    #     print('new_law_name =' + new_law_name + '|')382    #     item.law_name = new_law_name383    #     item.save()384    # l = explain_element.objects.all()385    # for item in l:386        # print('len = ', len(item.element_name))387        # print('element_name ='+item.element_name+'|')388        # new_element_name = item.element_name.strip()389        # print('new_element_name ='+new_element_name+'|')390        # item.explain_name = item.explain_name.strip()391        # item.element_name = new_element_name392        # item.save()393def add_clause_from_text(law_id,law_name, ful_name):394    multi_version_law_clause.objects.filter(law_id=law_id,law_name=law_name,ful_name=ful_name).delete()395    396    # law_clause.objects.filter(law_id=law_id,law_name=law_name).delete()397    f=open('./law_text.txt',"r")398    count = 0399    for line in f:400        if line=="\n":401            continue402        O = re.compile('(ç¬¬.{1,2}è)')403        fl = O.findall(line[0:6])404        if len(fl)>0:405            continue406        Obj = re.compile('(ç¬¬.{1,5}ç¼)')407        fl = Obj.findall(line[0:6])408        if len(fl)>0:409            continue410        OBJ = re.compile('(ç¬¬.{1,5}ç« )')411        fl = OBJ.findall(line[0:6])412        if len(fl)>0:413            continue414        if count < 100:415            obj = re.compile('(ç¬¬.{1,3}æ¡)')416            find_list = obj.findall(line[0:5])417        else:418            obj = re.compile('(ç¬¬.{1,5}æ¡)')419            find_list = obj.findall(line[0:7])420        if len(find_list)>0 and line.startswith('ç¬¬'):421            # print("find_list = ",find_list)422            if find_list[0] =="ç¬¬ä¸æ¡":423                count += 1424                clause_name = find_list[0]425                content = line + " "426            else:427                # print('law_id = ',law_id)428                # print('law_name = ',law_name)429                print('clause_id = ',count)430                print('clause_name ='+clause_name+"|")431                # print('content = ', content)432                multi_version_law_clause.objects.get_or_create(433                    law_id = law_id,434                    law_name=law_name,435                    ful_name=ful_name,436                    clause_id=count,437                    clause_name=clause_name,438                    content=content,439                    total=0440                )441                # law_clause.objects.get_or_create(442                #     law_id = law_id,443                #     law_name=law_name,444                #     clause_id=count,445                #     clause_name=clause_name,446                #     content=content,447                #     total=0448                # )449                count += 1450                clause_name = find_list[0]451                content = line + " "        452        else:453            content += line454    # print('law_id = ',law_id)455    # print('law_name = ',law_name)456    print('clause_id = ',count)457    print('clause_name ='+clause_name+"|")458    # print('content = ', content)459    multi_version_law_clause.objects.get_or_create(460        law_id = law_id,461        law_name=law_name,462        ful_name=ful_name,463        clause_id=count,464        clause_name=clause_name,465        content=content,466        total=0467    )468    # law_clause.objects.get_or_create(469    #     law_id = law_id,470    #     law_name=law_name,471    #     clause_id=count,472    #     clause_name=clause_name,473    #     content=content,474    #     total=0475    # )476    f.close()477    478def add_law(id,name,ful_name,timeliness,department,effic):479    multi_version_law.objects.get_or_create(480        law_id = id,481        law_name = name,482        ful_name = ful_name,483        timeliness = timeliness,484        department = department,485        efficacyLevel = effic,486        releaseDate = 1093622400001,487        effectiveDate = 1093622400001,488        legalCategories = "æ ",489        content = "æ ",490        total = 0491    )492    # law.objects.get_or_create(493    #     law_id = id,494    #     law_name = name,495    #     timeliness = timeliness,496    #     department = department,497    #     efficacyLevel = effic,498    #     releaseDate = 698947200000,499    #     effectiveDate = 698947200000,500    #     legalCategories = "æ ",501    #     content = "æ ",502    #     total = 0503    # )504def get_timeliness():505    l = law.objects.filter(timeliness = " å·²è¢«ä¿®æ¹")506    print(len(l))507    for item in l:508        print(item.law_name)509    510def modify_timeliness(name):511    l = law.objects.filter(law_name = name)512    if len(l)>0:513        item = l[0]514        item.timeliness = "ç°è¡ææ"515        item.save()516def multi_sort_law_and_clause():517    origin_law_list = multi_version_law.objects.all()518    for e in origin_law_list:519        e.department = e.department.strip(' ')520        e.efficacyLevel = e.efficacyLevel.strip(' ')521        e.timeliness = e.timeliness.strip(' ')522        e.save()523    law_list = multi_version_law.objects.all()524    clause_list = multi_version_law_clause.objects.all()525    law_list = sorted(law_list, key=lambda x:(x.efficacyLevel,x.department,x.ful_name))526    for item in law_list:527        print(item.law_id,item.law_name,item.ful_name)528    529    for e in clause_list:530        e.law_id = e.law_id + 1000531        e.save()532    multi_version_law.objects.all().delete()533    534    count = 0535    for item in law_list:536        count+=1537        multi_version_law.objects.get_or_create(538            law_id=count,539            law_name=item.law_name,540            ful_name=item.ful_name,541            timeliness=item.timeliness,542            department=item.department,543            efficacyLevel=item.efficacyLevel,544            releaseDate=item.releaseDate,545            effectiveDate=item.effectiveDate,546            legalCategories=item.legalCategories,547            content=item.content,548            total=item.total549        )550        l = multi_version_law_clause.objects.filter(law_id = item.law_id+1000,law_name=item.law_name)551        for e in l:552            multi_version_law_clause.objects.get_or_create(553                law_id = count,554                clause_id = e.clause_id,555                ful_name = e.ful_name,556                law_name = e.law_name,557                clause_name = e.clause_name,558                content = e.content,559                total = e.total560            )561    multi_version_law_clause.objects.filter(law_id__gt=1000).delete()562    563def move_to_law_and_clause():564    law.objects.all().delete()565    law_clause.objects.all().delete()566    valid_law_list = multi_version_law.objects.filter(timeliness = 'ç°è¡ææ')567    nearly_law_list = multi_version_law.objects.filter(timeliness = 'å°æªçæ')568    count = 0569    for Law in valid_law_list:570        count += 1571        law.objects.get_or_create(572            law_id = count,573            law_name = Law.law_name,574            timeliness = Law.timeliness,575            department = Law.department,576            efficacyLevel = Law.efficacyLevel,577            releaseDate = Law.releaseDate,578            effectiveDate= Law.effectiveDate,579            legalCategories = Law.legalCategories,580            content = Law.content,581            total = Law.total582        )583        clause_list = multi_version_law_clause.objects.filter(law_id = Law.law_id)584        for e in clause_list:585            law_clause.objects.get_or_create(586                law_id = count,587                clause_id = e.clause_id,588                law_name = e.law_name,589                clause_name = e.clause_name,590                content = e.content,591                total = e.total592            )593    for Law in nearly_law_list:594        count += 1595        law.objects.get_or_create(596            law_id = count,597            law_name = Law.law_name,598            timeliness = Law.timeliness,599            department = Law.department,600            efficacyLevel = Law.efficacyLevel,601            releaseDate = Law.releaseDate,602            effectiveDate= Law.effectiveDate,603            legalCategories = Law.legalCategories,604            content = Law.content,605            total = Law.total606        )607        clause_list = multi_version_law_clause.objects.filter(law_id = Law.law_id)608        for e in clause_list:609            law_clause.objects.get_or_create(610                law_id = count,611                clause_id = e.clause_id,612                law_name = e.law_name,613                clause_name = e.clause_name,614                content = e.content,615                total = e.total616            )617    618#å»é¤æ³æ¡åå®¹çï¼æ³å®èæ³ï¼619def trim_clause_content():620    clause_list = law_clause.objects.all()621    multi_clause_list = multi_version_law_clause.objects.all()622    print('load    completed')623    for item in clause_list:624        if "ï¼æ³å®èæ³:" in item.content:625            print('old content = ',item.content)626            item.content = item.content.split('ï¼æ³å®èæ³:')[0]627            print('new content = ',item.content)628            item.save()629    for e in multi_clause_list:630        if "ï¼æ³å®èæ³:" in e.content:631            print('old content = ',e.content)632            e.content = e.content.split('ï¼æ³å®èæ³:')[0]633            print('new content = ',e.content)634            e.save()635def trim_clause_name():636    ml = multi_version_law_clause.objects.all()637    for e in ml:638        if ' ' in e.clause_name:639            print(e.clause_name)640            s=""641            for ch in e.clause_name:642                if ch !=' ':643                    s+=ch644            print('s = ',s)645            e.clause_name = s646            e.save()647    l = law_clause.objects.all()648    for item in l:649        if ' ' in item.clause_name:650            print(item.clause_name)651            s=""652            for ch in item.clause_name:653                if ch !=' ':654                    s+=ch655            print('s = ',s)656            item.clause_name = s...

Data.py

Source:Data.py

1#!/usr/bin/env python2#coding:utf-83import json4import os5import re6import datetime7import random8import django9from regulation.models import law10from regulation.models import law_clause11from regulation.models import multi_version_law12from regulation.models import multi_version_law_clause13from regulation.models import explain14from regulation.models import explain_element15from regulation.models import solr_weibo_data16from regulation.models import matched_law_data17from regulation.models import matched_clause_data 18from regulation.models import law_charts_data19from regulation.models import explain_charts_data20from regulation.models import random_selected_data21from regulation.models import judge_law_data22from regulation.models import real_law_data23from regulation.models import judge_clause_data24from regulation.models import real_clause_data25from regulation.models import alias26from regulation.models import stopword27from regulation.models import timestamp28def rename_file():29    path='./data/explain/'30    for file in os.listdir(path):31        index=file[:3]32        old_name=file[3:-8]33        houzhui=file[-8:]34        # print("old_name=",old_name)35        temp_name=old_name.split("(")[0]36        # print("temp_name=",temp_name)37        if temp_name[-2:]=="è§£é":38            new_name=temp_name39        else:40            new_name=old_name41        if new_name!=old_name:42            print("new_name=",new_name)43            os.rename(path+index+old_name+houzhui, path+index+new_name+houzhui)44def sort_data_file():45    origin = open('law_clause.json',"r",encoding='utf-8')46    output = open('trimed_sorted_law_clause.json',"w")47    law_list = []48    l=[]49    for line in origin:50        data = json.loads(line)51        source=data['_source']52        timeliness=source['timeliness']53        department = source['department']54        releaseDate = source['releaseDate']55        effectiveDate = source['effectiveDate']56        efficacyLevel = source['efficacyLevel']57        legalCategories = source['legalCategories']58        if legalCategories == "":59            legalCategories = "æ "60        ful_name=(source['title']).split("\n")[0]61        t = (ful_name,timeliness,department,releaseDate,effectiveDate,efficacyLevel,legalCategories)62        if t not in l:63            l.append(t)64            law_list.append((ful_name,line))65        else:66            print('error:',t)67    law_list=sorted(law_list,key=lambda x:x[0],reverse=False)68    for item in law_list:69        output.write(item[1])70    output.close()71def lawdata2db():72    ful_name_list = []73    filepath = 'trimed_sorted_law_clause.json'74    f = open(filepath, 'r', encoding = 'utf-8')75    law_id = 076    count=077    for line in f:78        count+=179        data = json.loads(line)80        source = data['_source']81        ful_name = (source['title']).split("\n")[0]82        law_name = ful_name.split("(")[0]83        law_name = law_name.split("[")[0]84        85        if ful_name not in ful_name_list:86            ful_name_list.append(ful_name)87            law_id += 188        else:89            print('except = '+ ful_name)90            continue91        content = source['content']92        department = source['department']93        releaseDate = source['releaseDate']94        effectiveDate = source['effectiveDate']95        timeliness = source['timeliness'].strip(' ')96        efficacyLevel = source['efficacyLevel']97        legalCategories = source['legalCategories']98        if legalCategories == "":99            legalCategories = "æ "100        lawsRegulations = source['lawsRegulations']101        clause_id = 0102        for item in lawsRegulations:103            clause_id += 1104            element_content = item['element']105            element_list = element_content[1:-1].split('\n')106            clause_content = ""107            for x in element_list:108                clause_content += (x.strip(', []') + '\n')109            clause_name = element_list[0].strip(', []').split(' ')[0]110            clause_name = clause_name.strip()111            # print('law_id =', law_id )112            # print('clause_id =', clause_id)113            # print('law_name = ' + law_name)114            print('clause_name =' + clause_name+'|')115            # print('clause_content = ' + clause_content)116            multi_version_law_clause.objects.get_or_create(law_id = law_id, clause_id = clause_id, 117                law_name = law_name,ful_name=ful_name, clause_name = clause_name, content = clause_content)118        print('law_id = ', law_id)119        print('law_name = ' + law_name)120        print('ful_name = '+ful_name)121        # print('legalNumber = ', legalNumber)122        # print('timeliness = ', timeliness)123        # print('department = ', department)124        # print('efficacyLevel = ', efficacyLevel)125        # print('releaseDate = ', releaseDate)126        # print('effectiveDate = ', effectiveDate)127        # print('legalCategories = ', legalCategories)128        # print('content = ', content)129        multi_version_law.objects.get_or_create(law_id = law_id, law_name = law_name, ful_name=ful_name, timeliness = timeliness,130            department = department, efficacyLevel = efficacyLevel, releaseDate = releaseDate, 131            effectiveDate = effectiveDate, legalCategories = legalCategories, content = content)132        133    134def explaindata2db():135    explain_name_list = []136    filepath = './data/judicial_explain.json'137    f = open(filepath, 'r', encoding = 'utf-8')138    explain_id = 0139    for line in f:140        data = json.loads(line)141        source = data['_source']142        explain_name = source['title']143        explain_name = (source['title']).split("\n")[0]144        explain_name = explain_name.split("[")[0]145        temp_name = explain_name.split("(")[0]146        if temp_name[-2:]=="è§£é":147            explain_name = temp_name148        if explain_name not in explain_name_list:149            explain_name_list.append(explain_name)150            explain_id += 1151        else:152            continue153        content = source['content']154        155        department = source['department']156        releaseDate = source['releaseDate']157        effectiveDate = source['effectiveDate']158        timeliness = source['timeliness']159        efficacyLevel = source['efficacyLevel']160        legalCategories = source['legalCategories']161        if legalCategories == "":162            legalCategories = "NAN"163        questionExplains = source['questionExplain']164        clause_id = 0165        for item in questionExplains:166            clause_id += 1167            element_content = item['element']168            element_list = element_content[1:-1].split('\n')169            clause_content = ""170            for x in element_list:171                clause_content += (x.strip(', []') + '\n')172            clause_name = element_list[0].strip(', []').split(' ')[0]173            clause_name = clause_name.strip()174            # print('explain_id =', explain_id )175            # print('element_id =', clause_id)176            # print('explain_name = ' + explain_name)177            print('element_name =' + clause_name+'|')178            # print('element_content = ' + clause_content)179            explain_element.objects.get_or_create(explain_id = explain_id, element_id = clause_id, 180                explain_name = explain_name, element_name = clause_name, content = clause_content)181        print('explain_id = ', explain_id)182        print('explain_name = ', explain_name)183        # print('timeliness = ', timeliness)184        # print('department = ', department)185        # print('efficacyLevel = ', efficacyLevel)186        # print('releaseDate = ', releaseDate)187        # print('effectiveDate = ', effectiveDate)188        # print('legalCategories = ', legalCategories)189        # print('content = ', content)190        explain.objects.get_or_create(explain_id = explain_id, explain_name = explain_name, timeliness = timeliness,191            department = department, efficacyLevel = efficacyLevel, releaseDate = releaseDate, 192            effectiveDate = effectiveDate, legalCategories = legalCategories, content = content)193def generate_regulation_query_cmd():194    output = open('./cmd_law.txt', 'wb')195    l = law.objects.all()196    law_name_list = []197    for item in l:198        law_name = item.law_name199        law_name_list.append(law_name)200    count = 0201    for name in law_name_list:202        count += 1203        index = "%03d" % count204        205        p1 = "query \"\\\"".encode('utf-8')206        p2 = name.encode('utf-8')207        for i in range(11):208            month1 = "%02d" % (i + 1)209            month2 = "%02d" % (i + 2)210            print('month1=' + month1)211            print('month2=' + month2)212            p3 = "\\\" AND Time:[2019-{m1}-01T00:00:00Z TO 2019-{m2}-01T00:00:00Z]\"".format(m1=month1, m2=month2).encode('utf-8')213            p4 = " -f ./law/".encode('utf-8')214            p5 = ".bz2".encode('utf-8')215            id = index.encode('utf-8')216            m1 = month1.encode('utf-8')217            m2 = month2.encode('utf-8')218            p6 = '\n'.encode('utf-8')219            cmd = p1 + p2 + p3 + p4 + id + p2 + m1 + m2 + p5 + p6220            output.write(cmd)221    output.close()222def generate_explain_query_cmd():223    output = open('./cmd_explain.txt', 'wb')224    l = explain.objects.all()225    explain_name_list = []226    for item in l:227        explain_name = item.explain_name228        explain_name_list.append(explain_name)229    count = 0230    for name in explain_name_list:231        count += 1232        index = "%03d" % count233        234        p1 = "query \"\\\"".encode('utf-8')235        p2 = name.encode('utf-8')236        for i in range(11):237            month1 = "%02d" % (i + 1)238            month2 = "%02d" % (i + 2)239            print('month1=' + month1)240            print('month2=' + month2)241            p3 = "\\\" AND Time:[2019-{m1}-01T00:00:00Z TO 2019-{m2}-01T00:00:00Z]\"".format(m1=month1, m2=month2).encode('utf-8')242            p4 = " -f ./explain/".encode('utf-8')243            p5 = ".bz2".encode('utf-8')244            id = index.encode('utf-8')245            m1 = month1.encode('utf-8')246            m2 = month2.encode('utf-8')247            p6 = '\n'.encode('utf-8')248            cmd = p1 + p2 + p3 + p4 + id + p2 + m1 + m2 + p5 + p6249            output.write(cmd)250    output.close()251#çææ°çæ¥è¯¢å½ä»¤æä»¶ï¼ä¸¤ä¸ªæä»¶äºåä¸252def gen_cmd():253    #è·åç°å¨çæ¥æ254    today = datetime.date.today()255    ey=today.year256    em=today.month257    ed=today.day258    #è·åä¸æ¬¡æ´æ°æ°æ®çæ¥æ259    date_list = timestamp.objects.all()260    if len(date_list)==0:261        print('timestamp empty!')262        return263    start_date=date_list[0]264    sy=start_date.year265    sm=start_date.month266    sd=start_date.day267    date1=sy*10000+sm*100+sd268    date2=ey*10000+em*100+ed269    if date2<=date1:270        print('date1=',date1)271        print('date2=',date2)272        print("already up to date!")273        return 274    275    output=open('/Users/suzhan/Desktop/test/judicial/justice/search_dir/search_cmd.txt',"wb")276    keyword_list=[]277    formal_name_list=[]278    law_list=law.objects.all()279    explain_list=explain.objects.all()280    alias_list=alias.objects.all()281    #åæå«åå å°å³é®è¯åè¡¨ä¸282    for e in alias_list:283        if e.alias_name not in keyword_list:284            keyword_list.append(e.alias_name)285        if e.formal_name not in formal_name_list:286            formal_name_list.append(e.formal_name)287    #åæå©ä¸çæ²¡æå«åçå å°åè¡¨ä¸288    for e in explain_list:289        if e.explain_name not in formal_name_list:290            keyword_list.append(e.explain_name)291            formal_name_list.append(e.explain_name)292    293    for e in law_list:294        if e.law_name not in formal_name_list:295            keyword_list.append(e.law_name)296            formal_name_list.append(e.law_name)297    count=0298    for name in keyword_list:299        count += 1300        index = "%03d" % count301        302        p1 = "query \"\\\"".encode('utf-8')303        p2 = name.encode('utf-8')304        305        month1 = "%02d" % (sm)306        month2 = "%02d" % (em)307        day1 = "%02d" % (sd)308        day2 = "%02d" % (ed)309        print('year1=',sy,'month1=' + month1,'day1='+day1)310        print('year2=',ey,'month2=' + month2,'day2='+day2)311        p3 = "\\\" AND Time:[{y1}-{m1}-{d1}T00:00:00Z TO {y2}-{m2}-{d2}T00:00:00Z]\"".format(y1=sy,m1=month1,d1=day1,y2=ey,m2=month2,d2=day2).encode('utf-8')312        p4 = " -f ./search_data/".encode('utf-8')313        p5 = ".bz2".encode('utf-8')314        id = index.encode('utf-8')315        m1 = month1.encode('utf-8')316        m2 = month2.encode('utf-8')317        p6 = '\n'.encode('utf-8')318        cmd = p1 + p2 + p3 + p4 + id + p2 + m1 + m2 + p5 + p6319        output.write(cmd)320    output.close()321    start_date.year=2019322    start_date.month=1323    start_date.day=1324    # start_date.year=ey325    # start_date.month=em326    # start_date.day=ed327    start_date.save()328    print('gen_cmd done!')329    330def weibodata2db(flag):331    if flag==1:332        data_dir = './data/law/'333    elif flag==2:334        data_dir = './data/explain/'335    fl = os.listdir(data_dir)336    337    for fname in fl:338        keyword_id = int(fname[0:3])339        keyword_name = fname[3:-8]340        month = int(fname[-8:-6])341        path = os.path.join(data_dir, fname)342        fsize = os.path.getsize(path)343        if fsize > 0:344            file = open(path, 'r', encoding = 'utf-8')345            print('keyword_id = ', keyword_id)346            print('keyword_name = ', keyword_name)347            for line in file:348                data = json.loads(line)349                if flag==1:350                    data_type = 'law'351                elif flag==2:352                    data_type = 'explain'353                weibo_id = data['Id']354                if 'DataS_user_type' in data:355                    user_type = data['DataS_user_type']356                else:357                    user_type = 'æªç¥'358                Url = data['Url']359                author_name = data['DataS_nick_name']360                tou_xiang = data['DataS_tou_xiang']361                doc_time = data['Time']362                doc_date = data['Date']363                doc_text = data['Text']364                if 'DataS_r_weibo_content' in data:365                    weibo_source = data['DataS_r_weibo_content']366                else:367                    weibo_source = "NULL"368                if 'Opinion' in data:369                    opinion = data['Opinion']370                else:371                    opinion = 0372                # if 'DataS_r_time' in data:373                #     source_date = data['DataS_r_time']374                # else:375                #     source_date = '1970-01-01 00:00:01'376                # print('doc_time = ', doc_time)377                # print('weibo_id = ', weibo_id)378                solr_weibo_data.objects.get_or_create(weibo_link=Url,keyword_id = keyword_id, weibo_id = weibo_id, 379                    user_type = user_type,author_name = author_name, tou_xiang = tou_xiang, 380                    doc_time = doc_time, doc_date = doc_date,doc_text = doc_text, weibo_source = weibo_source, ...

test_expressions.py

Source:test_expressions.py

...21        content = sa.Column(sa.UnicodeText)22    return Article23@pytest.mark.usefixtures('postgresql_dsn')24class TestExplain(object):25    def test_render_explain(self, session, assert_startswith, Article):26        assert_startswith(27            explain(session.query(Article)),28            'EXPLAIN SELECT'29        )30    def test_render_explain_with_analyze(31        self,32        session,33        assert_startswith,34        Article35    ):36        assert_startswith(37            explain(session.query(Article), analyze=True),38            'EXPLAIN (ANALYZE true) SELECT'39        )40    def test_with_string_as_stmt_param(self, assert_startswith):41        assert_startswith(42            explain('SELECT 1 FROM article'),43            'EXPLAIN SELECT'44        )45    def test_format(self, assert_startswith):46        assert_startswith(47            explain('SELECT 1 FROM article', format='json'),48            'EXPLAIN (FORMAT json) SELECT'49        )50    def test_timing(self, assert_startswith):51        assert_startswith(52            explain('SELECT 1 FROM article', analyze=True, timing=False),53            'EXPLAIN (ANALYZE true, TIMING false) SELECT'54        )55    def test_verbose(self, assert_startswith):56        assert_startswith(57            explain('SELECT 1 FROM article', verbose=True),58            'EXPLAIN (VERBOSE true) SELECT'59        )60    def test_buffers(self, assert_startswith):61        assert_startswith(62            explain('SELECT 1 FROM article', analyze=True, buffers=True),63            'EXPLAIN (ANALYZE true, BUFFERS true) SELECT'64        )65    def test_costs(self, assert_startswith):66        assert_startswith(67            explain('SELECT 1 FROM article', costs=False),68            'EXPLAIN (COSTS false) SELECT'69        )70class TestExplainAnalyze(object):71    def test_render_explain_analyze(self, session, Article):72        assert str(73            explain_analyze(session.query(Article))74            .compile(75                dialect=postgresql.dialect()76            )77        ).startswith('EXPLAIN (ANALYZE true) SELECT')78class TestAsterisk(object):79    def test_with_table_object(self):80        Base = sa.ext.declarative.declarative_base()81        class Article(Base):...

multi_core_test.py

Source:multi_core_test.py

1#!/usr/bin/env python2#coding:utf-83import json4import os5import re6import datetime7import random8import django9import concurrent.futures10from regulation.models import law11from regulation.models import law_clause12from regulation.models import multi_version_law13from regulation.models import multi_version_law_clause14from regulation.models import explain15from regulation.models import explain_element16from regulation.models import solr_weibo_data17from regulation.models import matched_law_data18from regulation.models import matched_clause_data19def f(clause_obj):20    print('ful_name = ', clause_obj.ful_name, ' clause = ', clause_obj.clause_name)21    l = matched_clause_data.objects.filter(data_type='law', keyword_id = clause_obj.law_id, element_id = clause_obj.clause_id, origin_tag=0)22    num = len(l)23    clause_obj.total = num24    origin_l = matched_law_data.objects.filter(data_type='law', keyword_id = clause_obj.law_id, element_id = clause_obj.clause_id, origin_tag=1)25    clause_obj.origin_count = len(origin_l)26    clause_obj.save()27def law_tongji(law_obj, parm):28    print('param = ',param)29    # print('law = ', law_obj.law_name)30    l = matched_law_data.objects.filter(keyword_id = law_obj.law_id, data_type = 'law', origin_tag=0)31    num = len(l)32    law_obj.total = num33    origin_l = matched_law_data.objects.filter(keyword_id = law_obj.law_id, data_type = 'law', origin_tag=1)34    law_obj.origin_count = len(origin_l)35    law_obj.save()36def explain_tongji(explain_obj):37    # print('explain = ', explain_obj.explain_name)38    l = matched_law_data.objects.filter(keyword_id = explain_obj.explain_id, data_type = 'explain', origin_tag=0)39    num = len(l)40    explain_obj.total = num41    origin_l = matched_law_data.objects.filter(keyword_id = explain_obj.explain_id, data_type = 'explain', origin_tag=1)42    explain_obj.origin_count = len(origin_l)43    explain_obj.save()44def try_multi_core():45    with concurrent.futures.ProcessPoolExecutor() as executor:46        print('law')47        law_list = law.objects.all()48        executor.map(law_tongji,law_list,1)49        # clause_list = multi_version_law_clause.objects.all()50        # executor.map(f,clause_list)51def try_multi_core_1():52    with concurrent.futures.ProcessPoolExecutor() as executor:53        print('explain')54        explain_list = explain.objects.all()...

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.