How to use update_method method in localstack

Best Python code snippet using localstack_python

wc.py

Source:wc.py

1from wrangler import dw2import sys3if(len(sys.argv) < 3):4	sys.exit('Error: Please include an input and output file.  Example python script.py input.csv output.csv')5w = dw.DataWrangler()6# Split data repeatedly on newline  into  rows7w.add(dw.Split(column=["data"],8               table=0,9               status="active",10               drop=True,11               result="row",12               update=False,13               insert_position="right",14               row=None,15               on="\n",16               before=None,17               after=None,18               ignore_between=None,19               which=1,20               max=0,21               positions=None,22               quote_character=None))23# Delete  rows where data = '! Team !! Titles !! Runners-up...24w.add(dw.Filter(column=[],25                table=0,26                status="active",27                drop=False,28                row=dw.Row(column=[],29             table=0,30             status="active",31             drop=False,32             conditions=[dw.Eq(column=[],33            table=0,34            status="active",35            drop=False,36            lcol="data",37            value="! Team !! Titles !! Runners-up !! Third place !! Fourth place !! Top 4 <br/> finishes",38            op_str="=")])))39# Delete  rows where data = '|-'40w.add(dw.Filter(column=[],41                table=0,42                status="active",43                drop=False,44                row=dw.Row(column=[],45             table=0,46             status="active",47             drop=False,48             conditions=[dw.Eq(column=[],49            table=0,50            status="active",51            drop=False,52            lcol="data",53            value="|-",54            op_str="=")])))55# Extract from data between 'fb|' and '}'56w.add(dw.Extract(column=["data"],57                 table=0,58                 status="active",59                 drop=False,60                 result="column",61                 update=False,62                 insert_position="right",63                 row=None,64                 on=".*",65                 before="}",66                 after="fb\\|",67                 ignore_between=None,68                 which=1,69                 max=1,70                 positions=None))71# Extract from data on '.+' between '(' and ')'72w.add(dw.Extract(column=["data"],73                 table=0,74                 status="active",75                 drop=False,76                 result="column",77                 update=False,78                 insert_position="right",79                 row=None,80                 on=".+",81                 before="\\)",82                 after="\\(",83                 ignore_between=None,84                 which=1,85                 max=1,86                 positions=None))87# Extract from data on 'align' between '|' and '|'88w.add(dw.Extract(column=["data"],89                 table=0,90                 status="active",91                 drop=False,92                 result="column",93                 update=False,94                 insert_position="right",95                 row=None,96                 on="align",97                 before="\\|",98                 after="\\|",99                 ignore_between=None,100                 which=1,101                 max=1,102                 positions=None))103# Merge extract2, extract1  with glue  104w.add(dw.Merge(column=["extract2","extract1"],105               table=0,106               status="active",107               drop=False,108               result="column",109               update=False,110               insert_position="right",111               row=None,112               glue=""))113# Drop extract2114w.add(dw.Drop(column=["extract2"],115              table=0,116              status="active",117              drop=True))118# Drop extract1119w.add(dw.Drop(column=["extract1"],120              table=0,121              status="active",122              drop=True))123# Drop data124w.add(dw.Drop(column=["data"],125              table=0,126              status="active",127              drop=True))128# Delete empty rows129w.add(dw.Filter(column=[],130                table=0,131                status="active",132                drop=False,133                row=dw.Row(column=[],134             table=0,135             status="active",136             drop=False,137             conditions=[dw.Empty(column=[],138               table=0,139               status="active",140               drop=False,141               percent_valid=0,142               num_valid=0)])))143# Fill extract  with values from above144w.add(dw.Fill(column=["extract"],145              table=0,146              status="active",147              drop=False,148              direction="down",149              method="copy",150              row=None))151# Delete  rows where merge is null152w.add(dw.Filter(column=[],153                table=0,154                status="active",155                drop=False,156                row=dw.Row(column=[],157             table=0,158             status="active",159             drop=False,160             conditions=[dw.IsNull(column=[],161                table=0,162                status="active",163                drop=False,164                lcol="merge",165                value=None,166                op_str="is null")])))167# Extract from merge168w.add(dw.Extract(column=["merge"],169                 table=0,170                 status="active",171                 drop=False,172                 result="column",173                 update=False,174                 insert_position="right",175                 row=None,176                 on=None,177                 before=None,178                 after=None,179                 ignore_between=None,180                 which=1,181                 max=1,182                 positions=None))183# Edit extract3 row 1  to ' 1 '184w.add(dw.Edit(column=["extract3"],185              table=0,186              status="active",187              drop=False,188              result="column",189              update=True,190              insert_position="right",191              row=dw.Row(column=[],192             table=0,193             status="active",194             drop=False,195             conditions=[dw.RowIndex(column=[],196                  table=0,197                  status="active",198                  drop=False,199                  indices=[0])]),200              on=None,201              before=None,202              after=None,203              ignore_between=None,204              which=1,205              max=1,206              positions=None,207              to="1",208              update_method=None))209# Edit extract3 row 2  to ' 2 '210w.add(dw.Edit(column=["extract3"],211              table=0,212              status="active",213              drop=False,214              result="column",215              update=True,216              insert_position="right",217              row=dw.Row(column=[],218             table=0,219             status="active",220             drop=False,221             conditions=[dw.RowIndex(column=[],222                  table=0,223                  status="active",224                  drop=False,225                  indices=[1])]),226              on=None,227              before=None,228              after=None,229              ignore_between=None,230              which=1,231              max=1,232              positions=None,233              to="2",234              update_method=None))235# Edit extract3 row 3  to ' 3 '236w.add(dw.Edit(column=["extract3"],237              table=0,238              status="active",239              drop=False,240              result="column",241              update=True,242              insert_position="right",243              row=dw.Row(column=[],244             table=0,245             status="active",246             drop=False,247             conditions=[dw.RowIndex(column=[],248                  table=0,249                  status="active",250                  drop=False,251                  indices=[2])]),252              on=None,253              before=None,254              after=None,255              ignore_between=None,256              which=1,257              max=1,258              positions=None,259              to="3",260              update_method=None))261# Edit extract3 row 4  to ' 4 '262w.add(dw.Edit(column=["extract3"],263              table=0,264              status="active",265              drop=False,266              result="column",267              update=True,268              insert_position="right",269              row=dw.Row(column=[],270             table=0,271             status="active",272             drop=False,273             conditions=[dw.RowIndex(column=[],274                  table=0,275                  status="active",276                  drop=False,277                  indices=[3])]),278              on=None,279              before=None,280              after=None,281              ignore_between=None,282              which=1,283              max=1,284              positions=None,285              to="4",286              update_method=None))287# Edit extract3 row 5  to ' 1 '288w.add(dw.Edit(column=["extract3"],289              table=0,290              status="active",291              drop=False,292              result="column",293              update=True,294              insert_position="right",295              row=dw.Row(column=[],296             table=0,297             status="active",298             drop=False,299             conditions=[dw.RowIndex(column=[],300                  table=0,301                  status="active",302                  drop=False,303                  indices=[4])]),304              on=None,305              before=None,306              after=None,307              ignore_between=None,308              which=1,309              max=1,310              positions=None,311              to="1",312              update_method=None))313# Edit extract3 row 6  to ' 2 '314w.add(dw.Edit(column=["extract3"],315              table=0,316              status="active",317              drop=False,318              result="column",319              update=True,320              insert_position="right",321              row=dw.Row(column=[],322             table=0,323             status="active",324             drop=False,325             conditions=[dw.RowIndex(column=[],326                  table=0,327                  status="active",328                  drop=False,329                  indices=[5])]),330              on=None,331              before=None,332              after=None,333              ignore_between=None,334              which=1,335              max=1,336              positions=None,337              to="2",338              update_method=None))339# Edit extract3 row 7  to ' 3 '340w.add(dw.Edit(column=["extract3"],341              table=0,342              status="active",343              drop=False,344              result="column",345              update=True,346              insert_position="right",347              row=dw.Row(column=[],348             table=0,349             status="active",350             drop=False,351             conditions=[dw.RowIndex(column=[],352                  table=0,353                  status="active",354                  drop=False,355                  indices=[6])]),356              on=None,357              before=None,358              after=None,359              ignore_between=None,360              which=1,361              max=1,362              positions=None,363              to="3",364              update_method=None))365# Edit extract3 row 8  to ' 4 '366w.add(dw.Edit(column=["extract3"],367              table=0,368              status="active",369              drop=False,370              result="column",371              update=True,372              insert_position="right",373              row=dw.Row(column=[],374             table=0,375             status="active",376             drop=False,377             conditions=[dw.RowIndex(column=[],378                  table=0,379                  status="active",380                  drop=False,381                  indices=[7])]),382              on=None,383              before=None,384              after=None,385              ignore_between=None,386              which=1,387              max=1,388              positions=None,389              to="4",390              update_method=None))391# Edit extract3 row 9  to ' 1 '392w.add(dw.Edit(column=["extract3"],393              table=0,394              status="active",395              drop=False,396              result="column",397              update=True,398              insert_position="right",399              row=dw.Row(column=[],400             table=0,401             status="active",402             drop=False,403             conditions=[dw.RowIndex(column=[],404                  table=0,405                  status="active",406                  drop=False,407                  indices=[8])]),408              on=None,409              before=None,410              after=None,411              ignore_between=None,412              which=1,413              max=1,414              positions=None,415              to="1",416              update_method=None))417# Edit extract3 row 10  to ' 2 '418w.add(dw.Edit(column=["extract3"],419              table=0,420              status="active",421              drop=False,422              result="column",423              update=True,424              insert_position="right",425              row=dw.Row(column=[],426             table=0,427             status="active",428             drop=False,429             conditions=[dw.RowIndex(column=[],430                  table=0,431                  status="active",432                  drop=False,433                  indices=[9])]),434              on=None,435              before=None,436              after=None,437              ignore_between=None,438              which=1,439              max=1,440              positions=None,441              to="2",442              update_method=None))443# Edit extract3 row 11  to ' 3 '444w.add(dw.Edit(column=["extract3"],445              table=0,446              status="active",447              drop=False,448              result="column",449              update=True,450              insert_position="right",451              row=dw.Row(column=[],452             table=0,453             status="active",454             drop=False,455             conditions=[dw.RowIndex(column=[],456                  table=0,457                  status="active",458                  drop=False,459                  indices=[10])]),460              on=None,461              before=None,462              after=None,463              ignore_between=None,464              which=1,465              max=1,466              positions=None,467              to="3",468              update_method=None))469# Edit extract3 row 12  to ' 4 '470w.add(dw.Edit(column=["extract3"],471              table=0,472              status="active",473              drop=False,474              result="column",475              update=True,476              insert_position="right",477              row=dw.Row(column=[],478             table=0,479             status="active",480             drop=False,481             conditions=[dw.RowIndex(column=[],482                  table=0,483                  status="active",484                  drop=False,485                  indices=[11])]),486              on=None,487              before=None,488              after=None,489              ignore_between=None,490              which=1,491              max=1,492              positions=None,493              to="4",494              update_method=None))495# Edit extract3 row 13  to ' 1 '496w.add(dw.Edit(column=["extract3"],497              table=0,498              status="active",499              drop=False,500              result="column",501              update=True,502              insert_position="right",503              row=dw.Row(column=[],504             table=0,505             status="active",506             drop=False,507             conditions=[dw.RowIndex(column=[],508                  table=0,509                  status="active",510                  drop=False,511                  indices=[12])]),512              on=None,513              before=None,514              after=None,515              ignore_between=None,516              which=1,517              max=1,518              positions=None,519              to="1",520              update_method=None))521# Edit extract3 row 14  to ' 2 '522w.add(dw.Edit(column=["extract3"],523              table=0,524              status="active",525              drop=False,526              result="column",527              update=True,528              insert_position="right",529              row=dw.Row(column=[],530             table=0,531             status="active",532             drop=False,533             conditions=[dw.RowIndex(column=[],534                  table=0,535                  status="active",536                  drop=False,537                  indices=[13])]),538              on=None,539              before=None,540              after=None,541              ignore_between=None,542              which=1,543              max=1,544              positions=None,545              to="2",546              update_method=None))547# Edit extract3 row 17  to ' 1 '548w.add(dw.Edit(column=["extract3"],549              table=0,550              status="active",551              drop=False,552              result="column",553              update=True,554              insert_position="right",555              row=dw.Row(column=[],556             table=0,557             status="active",558             drop=False,559             conditions=[dw.RowIndex(column=[],560                  table=0,561                  status="active",562                  drop=False,563                  indices=[16])]),564              on=None,565              before=None,566              after=None,567              ignore_between=None,568              which=1,569              max=1,570              positions=None,571              to="1",572              update_method="UPPER"))573# Edit extract3 row 20  to ' 4 '574w.add(dw.Edit(column=["extract3"],575              table=0,576              status="active",577              drop=False,578              result="column",579              update=True,580              insert_position="right",581              row=dw.Row(column=[],582             table=0,583             status="active",584             drop=False,585             conditions=[dw.RowIndex(column=[],586                  table=0,587                  status="active",588                  drop=False,589                  indices=[19])]),590              on=None,591              before=None,592              after=None,593              ignore_between=None,594              which=1,595              max=1,596              positions=None,597              to="4",598              update_method=None))599# Edit extract3 row 28  to ' 4 '600w.add(dw.Edit(column=["extract3"],601              table=0,602              status="active",603              drop=False,604              result="column",605              update=True,606              insert_position="right",607              row=dw.Row(column=[],608             table=0,609             status="active",610             drop=False,611             conditions=[dw.RowIndex(column=[],612                  table=0,613                  status="active",614                  drop=False,615                  indices=[27])]),616              on=None,617              before=None,618              after=None,619              ignore_between=None,620              which=1,621              max=1,622              positions=None,623              to="4",624              update_method=None))625# Edit extract3 row 29  to ' 1 '626w.add(dw.Edit(column=["extract3"],627              table=0,628              status="active",629              drop=False,630              result="column",631              update=True,632              insert_position="right",633              row=dw.Row(column=[],634             table=0,635             status="active",636             drop=False,637             conditions=[dw.RowIndex(column=[],638                  table=0,639                  status="active",640                  drop=False,641                  indices=[28])]),642              on=None,643              before=None,644              after=None,645              ignore_between=None,646              which=1,647              max=1,648              positions=None,649              to="1",650              update_method=None))651# Edit extract3 row 32  to ' 4 '652w.add(dw.Edit(column=["extract3"],653              table=0,654              status="active",655              drop=False,656              result="column",657              update=True,658              insert_position="right",659              row=dw.Row(column=[],660             table=0,661             status="active",662             drop=False,663             conditions=[dw.RowIndex(column=[],664                  table=0,665                  status="active",666                  drop=False,667                  indices=[31])]),668              on=None,669              before=None,670              after=None,671              ignore_between=None,672              which=1,673              max=1,674              positions=None,675              to="4",676              update_method=None))677# Edit extract3 row 34  to ' 2 '678w.add(dw.Edit(column=["extract3"],679              table=0,680              status="active",681              drop=False,682              result="column",683              update=True,684              insert_position="right",685              row=dw.Row(column=[],686             table=0,687             status="active",688             drop=False,689             conditions=[dw.RowIndex(column=[],690                  table=0,691                  status="active",692                  drop=False,693                  indices=[33])]),694              on=None,695              before=None,696              after=None,697              ignore_between=None,698              which=1,699              max=1,700              positions=None,701              to="2",702              update_method=None))703# Edit extract3 row 36  to ' 4 '704w.add(dw.Edit(column=["extract3"],705              table=0,706              status="active",707              drop=False,708              result="column",709              update=True,710              insert_position="right",711              row=dw.Row(column=[],712             table=0,713             status="active",714             drop=False,715             conditions=[dw.RowIndex(column=[],716                  table=0,717                  status="active",718                  drop=False,719                  indices=[35])]),720              on=None,721              before=None,722              after=None,723              ignore_between=None,724              which=1,725              max=1,726              positions=None,727              to="4",728              update_method=None))729# Edit extract3 row 38  to ' 2 '730w.add(dw.Edit(column=["extract3"],731              table=0,732              status="active",733              drop=False,734              result="column",735              update=True,736              insert_position="right",737              row=dw.Row(column=[],738             table=0,739             status="active",740             drop=False,741             conditions=[dw.RowIndex(column=[],742                  table=0,743                  status="active",744                  drop=False,745                  indices=[37])]),746              on=None,747              before=None,748              after=None,749              ignore_between=None,750              which=1,751              max=1,752              positions=None,753              to="2",754              update_method=None))755# Edit extract3 row 42  to ' 2 '756w.add(dw.Edit(column=["extract3"],757              table=0,758              status="active",759              drop=False,760              result="column",761              update=True,762              insert_position="right",763              row=dw.Row(column=[],764             table=0,765             status="active",766             drop=False,767             conditions=[dw.RowIndex(column=[],768                  table=0,769                  status="active",770                  drop=False,771                  indices=[41])]),772              on=None,773              before=None,774              after=None,775              ignore_between=None,776              which=1,777              max=1,778              positions=None,779              to="2",780              update_method=None))781# Edit extract3 row 46  to ' 2 '782w.add(dw.Edit(column=["extract3"],783              table=0,784              status="active",785              drop=False,786              result="column",787              update=True,788              insert_position="right",789              row=dw.Row(column=[],790             table=0,791             status="active",792             drop=False,793             conditions=[dw.RowIndex(column=[],794                  table=0,795                  status="active",796                  drop=False,797                  indices=[45])]),798              on=None,799              before=None,800              after=None,801              ignore_between=None,802              which=1,803              max=1,804              positions=None,805              to="2",806              update_method=None))807# Edit extract3 row 47  to ' 3 '808w.add(dw.Edit(column=["extract3"],809              table=0,810              status="active",811              drop=False,812              result="column",813              update=True,814              insert_position="right",815              row=dw.Row(column=[],816             table=0,817             status="active",818             drop=False,819             conditions=[dw.RowIndex(column=[],820                  table=0,821                  status="active",822                  drop=False,823                  indices=[46])]),824              on=None,825              before=None,826              after=None,827              ignore_between=None,828              which=1,829              max=1,830              positions=None,831              to="3",832              update_method=None))833# Edit extract3 row 48  to ' 4 '834w.add(dw.Edit(column=["extract3"],835              table=0,836              status="active",837              drop=False,838              result="column",839              update=True,840              insert_position="right",841              row=dw.Row(column=[],842             table=0,843             status="active",844             drop=False,845             conditions=[dw.RowIndex(column=[],846                  table=0,847                  status="active",848                  drop=False,849                  indices=[47])]),850              on=None,851              before=None,852              after=None,853              ignore_between=None,854              which=1,855              max=1,856              positions=None,857              to="4",858              update_method=None))859# Edit extract3 row 51  to ' 3 '860w.add(dw.Edit(column=["extract3"],861              table=0,862              status="active",863              drop=False,864              result="column",865              update=True,866              insert_position="right",867              row=dw.Row(column=[],868             table=0,869             status="active",870             drop=False,871             conditions=[dw.RowIndex(column=[],872                  table=0,873                  status="active",874                  drop=False,875                  indices=[50])]),876              on=None,877              before=None,878              after=None,879              ignore_between=None,880              which=1,881              max=1,882              positions=None,883              to="3",884              update_method=None))885# Edit extract3 row 55  to ' 3 '886w.add(dw.Edit(column=["extract3"],887              table=0,888              status="active",889              drop=False,890              result="column",891              update=True,892              insert_position="right",893              row=dw.Row(column=[],894             table=0,895             status="active",896             drop=False,897             conditions=[dw.RowIndex(column=[],898                  table=0,899                  status="active",900                  drop=False,901                  indices=[54])]),902              on=None,903              before=None,904              after=None,905              ignore_between=None,906              which=1,907              max=1,908              positions=None,909              to="3",910              update_method=None))911# Edit extract3 row 56  to ' 4 '912w.add(dw.Edit(column=["extract3"],913              table=0,914              status="active",915              drop=False,916              result="column",917              update=True,918              insert_position="right",919              row=dw.Row(column=[],920             table=0,921             status="active",922             drop=False,923             conditions=[dw.RowIndex(column=[],924                  table=0,925                  status="active",926                  drop=False,927                  indices=[55])]),928              on=None,929              before=None,930              after=None,931              ignore_between=None,932              which=1,933              max=1,934              positions=None,935              to="4",936              update_method=None))937# Edit extract3 row 59  to ' 3 '938w.add(dw.Edit(column=["extract3"],939              table=0,940              status="active",941              drop=False,942              result="column",943              update=True,944              insert_position="right",945              row=dw.Row(column=[],946             table=0,947             status="active",948             drop=False,949             conditions=[dw.RowIndex(column=[],950                  table=0,951                  status="active",952                  drop=False,953                  indices=[58])]),954              on=None,955              before=None,956              after=None,957              ignore_between=None,958              which=1,959              max=1,960              positions=None,961              to="3",962              update_method=None))963# Edit extract3 row 60  to ' 4 '964w.add(dw.Edit(column=["extract3"],965              table=0,966              status="active",967              drop=False,968              result="column",969              update=True,970              insert_position="right",971              row=dw.Row(column=[],972             table=0,973             status="active",974             drop=False,975             conditions=[dw.RowIndex(column=[],976                  table=0,977                  status="active",978                  drop=False,979                  indices=[59])]),980              on=None,981              before=None,982              after=None,983              ignore_between=None,984              which=1,985              max=1,986              positions=None,987              to="4",988              update_method=None))989# Edit extract3 row 63  to ' 3 '990w.add(dw.Edit(column=["extract3"],991              table=0,992              status="active",993              drop=False,994              result="column",995              update=True,996              insert_position="right",997              row=dw.Row(column=[],998             table=0,999             status="active",1000             drop=False,1001             conditions=[dw.RowIndex(column=[],1002                  table=0,1003                  status="active",1004                  drop=False,1005                  indices=[62])]),1006              on=None,1007              before=None,1008              after=None,1009              ignore_between=None,1010              which=1,1011              max=1,1012              positions=None,1013              to="3",1014              update_method=None))1015# Edit extract3 row 67  to ' 3 '1016w.add(dw.Edit(column=["extract3"],1017              table=0,1018              status="active",1019              drop=False,1020              result="column",1021              update=True,1022              insert_position="right",1023              row=dw.Row(column=[],1024             table=0,1025             status="active",1026             drop=False,1027             conditions=[dw.RowIndex(column=[],1028                  table=0,1029                  status="active",1030                  drop=False,1031                  indices=[66])]),1032              on=None,1033              before=None,1034              after=None,1035              ignore_between=None,1036              which=1,1037              max=1,1038              positions=None,1039              to="3",1040              update_method=None))1041# Edit extract3 row 71  to ' 3 '1042w.add(dw.Edit(column=["extract3"],1043              table=0,1044              status="active",1045              drop=False,1046              result="column",1047              update=True,1048              insert_position="right",1049              row=dw.Row(column=[],1050             table=0,1051             status="active",1052             drop=False,1053             conditions=[dw.RowIndex(column=[],1054                  table=0,1055                  status="active",1056                  drop=False,1057                  indices=[70])]),1058              on=None,1059              before=None,1060              after=None,1061              ignore_between=None,1062              which=1,1063              max=1,1064              positions=None,1065              to="3",1066              update_method=None))1067# Edit extract3 row 75  to ' 3 '1068w.add(dw.Edit(column=["extract3"],1069              table=0,1070              status="active",1071              drop=False,1072              result="column",1073              update=True,1074              insert_position="right",1075              row=dw.Row(column=[],1076             table=0,1077             status="active",1078             drop=False,1079             conditions=[dw.RowIndex(column=[],1080                  table=0,1081                  status="active",1082                  drop=False,1083                  indices=[74])]),1084              on=None,1085              before=None,1086              after=None,1087              ignore_between=None,1088              which=1,1089              max=1,1090              positions=None,1091              to="3",1092              update_method=None))1093# Edit extract3 row 80  to ' 4 '1094w.add(dw.Edit(column=["extract3"],1095              table=0,1096              status="active",1097              drop=False,1098              result="column",1099              update=True,1100              insert_position="right",1101              row=dw.Row(column=[],1102             table=0,1103             status="active",1104             drop=False,1105             conditions=[dw.RowIndex(column=[],1106                  table=0,1107                  status="active",1108                  drop=False,1109                  indices=[79])]),1110              on=None,1111              before=None,1112              after=None,1113              ignore_between=None,1114              which=1,1115              max=1,1116              positions=None,1117              to="4",1118              update_method=None))1119# Edit extract3 row 84  to ' 4 '1120w.add(dw.Edit(column=["extract3"],1121              table=0,1122              status="active",1123              drop=False,1124              result="column",1125              update=True,1126              insert_position="right",1127              row=dw.Row(column=[],1128             table=0,1129             status="active",1130             drop=False,1131             conditions=[dw.RowIndex(column=[],1132                  table=0,1133                  status="active",1134                  drop=False,1135                  indices=[83])]),1136              on=None,1137              before=None,1138              after=None,1139              ignore_between=None,1140              which=1,1141              max=1,1142              positions=None,1143              to="4",1144              update_method=None))1145# Edit extract3 row 88  to ' 4 '1146w.add(dw.Edit(column=["extract3"],1147              table=0,1148              status="active",1149              drop=False,1150              result="column",1151              update=True,1152              insert_position="right",1153              row=dw.Row(column=[],1154             table=0,1155             status="active",1156             drop=False,1157             conditions=[dw.RowIndex(column=[],1158                  table=0,1159                  status="active",1160                  drop=False,1161                  indices=[87])]),1162              on=None,1163              before=None,1164              after=None,1165              ignore_between=None,1166              which=1,1167              max=1,1168              positions=None,1169              to="4",1170              update_method=None))1171# Edit extract3 row 92  to ' 4 '1172w.add(dw.Edit(column=["extract3"],1173              table=0,1174              status="active",1175              drop=False,1176              result="column",1177              update=True,1178              insert_position="right",1179              row=dw.Row(column=[],1180             table=0,1181             status="active",1182             drop=False,1183             conditions=[dw.RowIndex(column=[],1184                  table=0,1185                  status="active",1186                  drop=False,1187                  indices=[91])]),1188              on=None,1189              before=None,1190              after=None,1191              ignore_between=None,1192              which=1,1193              max=1,1194              positions=None,1195              to="4",1196              update_method=None))1197# Edit extract3 row 96  to ' 4 '1198w.add(dw.Edit(column=["extract3"],1199              table=0,1200              status="active",1201              drop=False,1202              result="column",1203              update=True,1204              insert_position="right",1205              row=dw.Row(column=[],1206             table=0,1207             status="active",1208             drop=False,1209             conditions=[dw.RowIndex(column=[],1210                  table=0,1211                  status="active",1212                  drop=False,1213                  indices=[95])]),1214              on=None,1215              before=None,1216              after=None,1217              ignore_between=None,1218              which=1,1219              max=1,1220              positions=None,1221              to="4",1222              update_method=None))1223# Delete  rows where merge = 'align'1224w.add(dw.Filter(column=[],1225                table=0,1226                status="active",1227                drop=False,1228                row=dw.Row(column=[],1229             table=0,1230             status="active",1231             drop=False,1232             conditions=[dw.Eq(column=[],1233            table=0,1234            status="active",1235            drop=False,1236            lcol="merge",1237            value="align",1238            op_str="=")])))1239# Edit extract3 row 19  to ' 3 '1240w.add(dw.Edit(column=["extract3"],1241              table=0,1242              status="active",1243              drop=False,1244              result="column",1245              update=True,1246              insert_position="right",1247              row=dw.Row(column=[],1248             table=0,1249             status="active",1250             drop=False,1251             conditions=[dw.RowIndex(column=[],1252                  table=0,1253                  status="active",1254                  drop=False,1255                  indices=[18])]),1256              on=None,1257              before=None,1258              after=None,1259              ignore_between=None,1260              which=1,1261              max=1,1262              positions=None,1263              to="3",1264              update_method=None))1265# Edit extract3 row 17  to ' 1 '1266w.add(dw.Edit(column=["extract3"],1267              table=0,1268              status="active",1269              drop=False,1270              result="column",1271              update=True,1272              insert_position="right",1273              row=dw.Row(column=[],1274             table=0,1275             status="active",1276             drop=False,1277             conditions=[dw.RowIndex(column=[],1278                  table=0,1279                  status="active",1280                  drop=False,1281                  indices=[16])]),1282              on=None,1283              before=None,1284              after=None,1285              ignore_between=None,1286              which=1,1287              max=1,1288              positions=None,1289              to="1",1290              update_method=None))1291# Edit extract3 row 21  to ' 1 '1292w.add(dw.Edit(column=["extract3"],1293              table=0,1294              status="active",1295              drop=False,1296              result="column",1297              update=True,1298              insert_position="right",1299              row=dw.Row(column=[],1300             table=0,1301             status="active",1302             drop=False,1303             conditions=[dw.RowIndex(column=[],1304                  table=0,1305                  status="active",1306                  drop=False,1307                  indices=[20])]),1308              on=None,1309              before=None,1310              after=None,1311              ignore_between=None,1312              which=1,1313              max=1,1314              positions=None,1315              to="1",1316              update_method=None))1317# Edit extract3 row 20  to ' 4 '1318w.add(dw.Edit(column=["extract3"],1319              table=0,1320              status="active",1321              drop=False,1322              result="column",1323              update=True,1324              insert_position="right",1325              row=dw.Row(column=[],1326             table=0,1327             status="active",1328             drop=False,1329             conditions=[dw.RowIndex(column=[],1330                  table=0,1331                  status="active",1332                  drop=False,1333                  indices=[19])]),1334              on=None,1335              before=None,1336              after=None,1337              ignore_between=None,1338              which=1,1339              max=1,1340              positions=None,1341              to="4",1342              update_method=None))1343# Edit extract3 row 18  to ' 2 '1344w.add(dw.Edit(column=["extract3"],1345              table=0,1346              status="active",1347              drop=False,1348              result="column",1349              update=True,1350              insert_position="right",1351              row=dw.Row(column=[],1352             table=0,1353             status="active",1354             drop=False,1355             conditions=[dw.RowIndex(column=[],1356                  table=0,1357                  status="active",1358                  drop=False,1359                  indices=[17])]),1360              on=None,1361              before=None,1362              after=None,1363              ignore_between=None,1364              which=1,1365              max=1,1366              positions=None,1367              to="2",1368              update_method=None))1369# Merge extract3, extract  with glue  , 1370w.add(dw.Merge(column=["extract3","extract"],1371               table=0,1372               status="active",1373               drop=False,1374               result="column",1375               update=False,1376               insert_position="right",1377               row=None,1378               glue=", "))1379# Split merge repeatedly after ','  into  rows1380w.add(dw.Split(column=["merge"],1381               table=0,1382               status="active",1383               drop=True,1384               result="row",1385               update=False,1386               insert_position="right",1387               row=None,1388               on=None,1389               before=None,1390               after=",",1391               ignore_between=None,1392               which=1,1393               max="0",1394               positions=None,1395               quote_character=None))1396# Extract from merge on ' any number ' after '|'1397w.add(dw.Extract(column=["merge"],1398                 table=0,1399                 status="active",1400                 drop=False,1401                 result="column",1402                 update=False,1403                 insert_position="right",1404                 row=None,1405                 on="[0-9]+",1406                 before=None,1407                 after="\\|",1408                 ignore_between=None,1409                 which=1,1410                 max=1,1411                 positions=None))1412# Drop merge, extract3, extract1413w.add(dw.Drop(column=["merge","extract3","extract"],1414              table=0,1415              status="active",1416              drop=True))1417# Merge extract4, merge1  with glue  , 1418w.add(dw.Merge(column=["extract4","merge1"],1419               table=0,1420               status="active",1421               drop=False,1422               result="column",1423               update=False,1424               insert_position="right",1425               row=None,1426               glue=", "))1427# Drop extract41428w.add(dw.Drop(column=["extract4"],1429              table=0,1430              status="active",1431              drop=True))1432# Drop merge11433w.add(dw.Drop(column=["merge1"],1434              table=0,1435              status="active",1436              drop=True))...

managers.py

Source:managers.py

1import datetime2from django.db.models import F, Func, Value3from django.db.models.functions import Upper, Lower, Concat4from django.db import models5from operator import itemgetter6from collections import OrderedDict, defaultdict7from core.services import get_admin_account, delimit_filtervalue8from .services import check_lexical_convention, generate_kwargs_from_parsed_rule9class GET_VALUE(F):10    ADD = '->'11class ARRAY_APPEND(Func):12    function = 'array_append'13class ARRAY_REMOVE(Func):14    function = 'array_remove'15class UNNEST(Func):16    function = 'unnest'17class SKEYS(Func):18    function = 'skeys'19class REPLACE(Func):20    function = 'replace'21class DELETE(Func):22    function = 'delete'23def lowercase(text):24    return text.lower()25def uppercase(text):26    return text.upper()27def capitalize(text):28    return text.capitalize()29class YaraRuleQueryset(models.query.QuerySet):30    def active(self):31        return self.filter(status=self.model.ACTIVE_STATUS)32    def inactive(self):33        return self.filter(status=self.model.INACTIVE_STATUS)34    def pending(self):35        return self.filter(status=self.model.PENDING_STATUS)36    def rejected(self):37        return self.filter(status=self.model.REJECTED_STATUS)38    def has_dependencies(self):39        return self.filter(dependencies__len__gt=0)40    def has_missing_dependencies(self):41        missing_dependencies = self.missing_dependency_list()42        return self.has_dependencies().filter(dependencies__overlap=missing_dependencies)43    def category_list(self):44        return self.order_by('category').values_list('category', flat=True).distinct()45    def source_list(self):46        return self.order_by('source').values_list('source', flat=True).distinct()47    def submitter_list(self):48        return self.order_by('submitter').values_list('submitter__username', flat=True).distinct()49    def dependency_list(self):50        dependencies = list(self.annotate(dependency_elements=UNNEST('dependencies')).values_list('dependency_elements', flat=True).distinct())51        dependencies.sort()52        return dependencies53    def missing_dependency_list(self):54        dependency_list = self.dependency_list()55        available_dependencies = self.filter(name__in=dependency_list).values_list('name', flat=True).distinct()56        missing_dependencies = list(set(dependency_list) - set(available_dependencies))57        return missing_dependencies58    def tag_list(self):59        tags = list(self.annotate(tag_elements=UNNEST('tags')).values_list('tag_elements', flat=True).distinct())60        tags.sort(key=str.lower)61        return tags62    def metakey_list(self):63        metadata_keys = list(self.annotate(metadata_elements=SKEYS('metadata')).values_list('metadata_elements', flat=True).distinct())64        metadata_keys.sort(key=str.lower)65        return metadata_keys66    def import_list(self):67        imports = list(self.annotate(import_elements=UNNEST('imports')).values_list('import_elements', flat=True).distinct())68        imports.sort(key=str.lower)69        return imports70    def scope_list(self):71        scopes = list(self.annotate(scope_elements=UNNEST('scopes')).values_list('scope_elements', flat=True).distinct())72        scopes.sort(key=str.lower)73        return scopes74    def active_count(self):75        return self.active().count()76    def inactive_count(self):77        return self.inactive().count()78    def pending_count(self):79        return self.pending().count()80    def rejected_count(self):81        return self.rejected().count()82    def has_dependencies_count(self):83        return self.has_dependencies().count()84    def has_missing_dependencies_count(self):85        return self.has_missing_dependencies().count()86    def category_count(self):87        category_count = [(entry['category'], entry['count'])88                          for entry in self.order_by('category')89                          .values('category').annotate(count=models.Count('category'))]90        category_count.sort(key=itemgetter(0))91        ordered_category_count = OrderedDict((item[0], item[1]) for item in category_count)92        try:93            del ordered_category_count['']94        except KeyError:95            pass96        return ordered_category_count97    def source_count(self):98        source_count = [(entry['source'], entry['count'])99                        for entry in self.order_by('source')100                        .values('source').annotate(count=models.Count('source'))]101        source_count.sort(key=itemgetter(0))102        ordered_source_count = OrderedDict((item[0], item[1]) for item in source_count)103        try:104            del ordered_source_count['']105        except KeyError:106            pass107        return ordered_source_count108    def tag_count(self):109        tags = [(key, self.filter(tags__contains=[key]).count()) for key in self.tag_list()]110        tags.sort(key=itemgetter(0))111        tag_count = OrderedDict((item[0], item[1]) for item in tags)112        return tag_count113    def import_count(self):114        imports = [(key, self.filter(imports__contains=[key]).count()) for key in self.import_list()]115        imports.sort(key=itemgetter(0))116        imports = OrderedDict((item[0], item[1]) for item in imports)117        return imports118    def metakey_count(self):119        metadata_keys = [(key, self.filter(metadata__has_key=key).count()) for key in self.metakey_list()]120        metadata_keys.sort(key=itemgetter(0))121        metadata_keys = OrderedDict((item[0], item[1]) for item in metadata_keys)122        return metadata_keys123    def dependency_count(self):124        dependencies = [(key, self.filter(dependencies__contains=[key]).count()) for key in self.dependency_list()]125        dependencies.sort(key=itemgetter(0))126        dependency_count = OrderedDict((item[0], item[1]) for item in dependencies)127        return dependency_count128    def name_conflict_count(self):129        name_conflicts = [(entry['name'], entry['count'])130                          for entry in self.order_by('name').values('name')131                          .annotate(count=models.Count('name')).filter(count__gt=1)]132        name_conflicts.sort(key=itemgetter(0))133        name_conflicts = OrderedDict((item[0], item[1]) for item in name_conflicts)134        return name_conflicts135    def logic_collision_count(self):136        logic_conflicts = [(entry['logic_hash'], entry['count'])137                           for entry in self.order_by('logic_hash').values('logic_hash')138                           .annotate(count=models.Count('logic_hash')).filter(count__gt=1)]139        logic_conflicts.sort(key=itemgetter(0))140        logic_conflicts = OrderedDict((item[0], item[1]) for item in logic_conflicts)141        return logic_conflicts142    def missing_dependency_count(self):143        missing_dependencies = [(key, self.filter(dependencies__contains=[key]).count()) for key in self.missing_dependency_list()]144        missing_dependencies.sort(key=itemgetter(0))145        missing_dependencies = OrderedDict((item[0], item[1]) for item in missing_dependencies)146        return missing_dependencies147    # Bulk Update Methods148    def bulk_update(self, update_params):149        # Pass this to methods that return feedback in order to get unified message150        update_feedback = { 'errors': [], 'warnings': [], 'changes': [] }151        # Dynamically generate update method key value pairs152        metakey_list = self.metakey_list()153        metakey_update_methods = {'lowercase': {}, 'uppercase': {}, 'capitalize': {}, 'rename': {}}154        for metakey_value in metakey_list:155            metakey_update_methods['lowercase']['lowercase_metakey_{}'.format(metakey_value)] = metakey_value156            metakey_update_methods['uppercase']['uppercase_metakey_{}'.format(metakey_value)] = metakey_value157            metakey_update_methods['capitalize']['capitalize_metakey_{}'.format(metakey_value)] = metakey_value158            metakey_update_methods['rename']['rename_metakey_{}'.format(metakey_value)] = metakey_value159        # Perform updates for each update method specified160        for update_method in update_params:161            if update_method == 'update_category':162                category = update_params.get(update_method)163                self.update_category(category)164            elif update_method == 'update_source':165                source = update_params.get(update_method)166                self.update_source(source)167            elif update_method == 'update_status':168                status = update_params.get(update_method)169                self.update_status(status)170            elif update_method == 'add_tags':171                update_value = update_params.get(update_method)172                self.add_tags(update_value, update_feedback=update_feedback)173            elif update_method == 'remove_tags':174                update_value = update_params.get(update_method)175                self.remove_tags(update_value)176            elif update_method == 'remove_scopes':177                update_value = update_params.get(update_method)178                self.remove_scopes(update_value)179            elif update_method.startswith('set_metadata_'):180                metadata_key = update_method[update_method.index('set_metadata_') + 13:]181                metadata_value = update_params.get(update_method)182                self.set_metadata(metadata_key, metadata_value)183            elif update_method == 'remove_metadata':184                update_value = update_params.get(update_method)185                self.remove_metadata(update_value)186            elif update_method == 'lowercase_name':187                update_value = update_params.get(update_method)188                self.change_name_case('lowercase', modifier=update_value)189            elif update_method == 'uppercase_name':190                update_value = update_params.get(update_method)191                self.change_name_case('uppercase', modifier=update_value)192            elif update_method == 'append_name':193                update_value = update_params.get(update_method)194                self.append_name(update_value)195            elif update_method == 'prepend_name':196                update_value = update_params.get(update_method)197                self.prepend_name(update_value)198            elif update_method == 'remove_name':199                update_value = update_params.get(update_method)200                self.remove_name(update_value)201            elif update_method in metakey_update_methods['lowercase']:202                update_value = update_params.get(update_method)203                metakey = metakey_update_methods['lowercase'][update_method]204                self.change_metakey_case(metakey, 'lowercase', modifier=update_value)205            elif update_method in metakey_update_methods['uppercase']:206                update_value = update_params.get(update_method)207                metakey = metakey_update_methods['uppercase'][update_method]208                self.change_metakey_case(metakey, 'uppercase', modifier=update_value)209            elif update_method in metakey_update_methods['capitalize']:210                update_value = update_params.get(update_method)211                metakey = metakey_update_methods['capitalize'][update_method]212                self.change_metakey_case(metakey, 'capitalize', modifier=update_value)213            elif update_method in metakey_update_methods['rename']:214                update_value = update_params.get(update_method)215                metakey = metakey_update_methods['rename'][update_method]216                self.rename_metakey(metakey, update_value)217            else:218                continue219        return update_feedback220    def update_category(self, category):221        queryset_owners = self.values_list('owner', flat=True).distinct()222        if len(queryset_owners) == 1:223            group = self[:1].get().owner224            if category in group.groupmeta.category_options:225                self.update(category=category)226    def update_source(self, source):227        queryset_owners = self.values_list('owner', flat=True).distinct()228        if len(queryset_owners) == 1:229            group = self[:1].get().owner230            if source in group.groupmeta.source_options:231                self.update(source=source)232    def update_status(self, status):233        if status in ('active', 'inactive', 'pending', 'rejected'):234            self.update(status=status)235    def add_tags(self, tag_elements, update_feedback=None):236        if isinstance(tag_elements, str):237            tag_elements = delimit_filtervalue(tag_elements)238        if not update_feedback:239            update_feedback = { 'warnings': [], 'changes': [] }240        for tag_value in tag_elements:241            if check_lexical_convention(tag_value):242                self.exclude(tags__overlap=[tag_value]).update(tags=ARRAY_APPEND('tags', Value(tag_value)))243                msg = 'Added Tag: {}'.format(tag_value)244                update_feedback['changes'].append(msg)245            else:246                msg = 'Skipped Invalid Tag: {}'.format(tag_value)247                update_feedback['warnings'].append(msg)248        return update_feedback249    def remove_tags(self, tag_elements):250        if isinstance(tag_elements, str):251            tag_elements = delimit_filtervalue(tag_elements)252        for tag_value in tag_elements:253            self.filter(tags__overlap=[tag_value]).update(tags=ARRAY_REMOVE('tags', Value(tag_value)))254    def remove_scopes(self, scope_elements):255        if isinstance(scope_elements, str):256            scope_elements = delimit_filtervalue(scope_elements)257        for scope_value in scope_elements:258            print(scope_value)259            self.filter(scopes__overlap=[scope_value]).update(scopes=ARRAY_REMOVE('scopes', Value(scope_value)))260    def change_name_case(self, operation, modifier=None):261        available_operations = {'lowercase': lowercase,262                                'uppercase': uppercase}263        edit = available_operations.get(operation, None)264        if edit and modifier:265            modification = edit(modifier)266            self.update(name=REPLACE('name', Value(modifier), Value(modification)))267        elif operation == 'lowercase':268            self.update(name=Lower('name'))269        elif operation == 'uppercase':270            self.update(name=Upper('name'))271    def remove_metadata(self, metadata_elements):272        if isinstance(metadata_elements, str):273            metadata_elements = delimit_filtervalue(metadata_elements)274        for metadata_value in metadata_elements:275            self.filter(metadata__has_key=metadata_value).update(metadata=DELETE('metadata', Value(metadata_value)))276    def change_metakey_case(self, metakey, operation, modifier=None):277        available_operations = {'lowercase': lowercase,278                                'uppercase': uppercase,279                                'capitalize': capitalize}280        edit = available_operations.get(operation, None)281        if edit:282            if modifier:283                new_metakey = metakey.replace(modifier, edit(modifier))284            else:285                new_metakey = edit(metakey)286            # Copy old metadata into an hstore container with new key value287            TEMP_HSTORE = Func(Value(new_metakey), GET_VALUE('metadata') + Value(metakey), function='hstore')288            # Delete old key entry from original hstore289            META_HSTORE = Func(F('metadata'), Value(metakey), function='delete')290            # Combine the two hstores using internal 'hs_concat' function291            CONCAT_HSTORE = Func(TEMP_HSTORE, META_HSTORE, function='hs_concat')292            self.filter(metadata__has_key=metakey).update(metadata=CONCAT_HSTORE)293    def rename_metakey(self, old_metakey, new_metakey):294        if check_lexical_convention(new_metakey):295            # Copy old metadata into an hstore container with new key value296            TEMP_HSTORE = Func(Value(new_metakey), GET_VALUE('metadata') + Value(old_metakey), function='hstore')297            # Delete old key entry from original hstore298            META_HSTORE = Func(F('metadata'), Value(old_metakey), function='delete')299            # Combine the two hstores using internal 'hs_concat' function300            CONCAT_HSTORE = Func(TEMP_HSTORE, META_HSTORE, function='hs_concat')301            self.filter(metadata__has_key=old_metakey).update(metadata=CONCAT_HSTORE)302    def append_name(self, modifier):303        invalid_modifications = []304        # Ensure name manipulation does not create an invalid rule name305        for entry_id, entry_name in self.values_list('id', 'name'):306            new_name = entry_name + modifier307            if not check_lexical_convention(new_name):308                invalid_modifications.append(entry_id)309        self.exclude(id__in=invalid_modifications).update(name=Concat('name', Value(modifier)))310    def prepend_name(self, modifier):311        invalid_modifications = []312        # Ensure name manipulation does not create an invalid rule name313        for entry_id, entry_name in self.values_list('id', 'name'):314            new_name = modifier + entry_name315            if not check_lexical_convention(new_name):316                invalid_modifications.append(entry_id)317        self.exclude(id__in=invalid_modifications).update(name=Concat(Value(modifier), 'name'))318    def remove_name(self, pattern):319        invalid_modifications = []320        # Ensure name manipulation does not create an invalid rule name321        for entry_id, entry_name in self.values_list('id', 'name'):322            new_name = entry_name.replace(pattern, '')323            if not check_lexical_convention(new_name):324                invalid_modifications.append(entry_id)325        self.exclude(id__in=invalid_modifications).update(name=REPLACE('name', Value(pattern), Value('')))326    def set_metadata(self, metakey, metavalue):327        if check_lexical_convention(metakey) and \328        (metavalue.isdigit() or metavalue in ('true', 'false') or \329        (metavalue.startswith('\"') and metavalue.endswith('\"'))):330            # Copy old metadata into an hstore container with new key value331            TEMP_HSTORE = Func(Value(metakey), Value(metavalue), function='hstore')332            # Delete old key entry from original hstore333            META_HSTORE = Func(F('metadata'), Value(metakey), function='delete')334            # Combine the two hstores using internal 'hs_concat' function335            CONCAT_HSTORE = Func(TEMP_HSTORE, META_HSTORE, function='hs_concat')336            self.update(metadata=CONCAT_HSTORE)337    def deconflict_logic(self, update_feedback=None):338        if not update_feedback:339            update_feedback = { 'warnings': [], 'changes': [] }340        deconflict_count = 0341        logic_mapping = defaultdict(list)342        # Group rules with same logic343        for rule in self:344            logic_mapping[rule.logic_hash].append(rule)345        for logic_hash, rules in logic_mapping.items():346            # Check if there was actually a collision347            if len(rules) == 1:348                continue349            newrule = None350            for rule in rules:351                if not newrule:352                    newrule = rule353                else:354                    for tag in rule.tags:355                        if tag not in newrule.tags:356                            newrule.tags.append(tag)357                    for scope in rule.scopes:358                        if scope not in newrule.scopes:359                            newrule.scopes.append(scope)360                    for imp in rule.imports:361                        if imp not in newrule.imports:362                            newrule.imports.append(imp)363                    for key, value in rule.metadata.items():364                        if key not in newrule.metadata:365                            newrule.metadata[key] = value366                    for comment in rule.yararulecomment_set.all():367                        comment.rule = newrule368                        comment.save()369                    rule.delete()370                    deconflict_count += 1371            newrule.save()372        if deconflict_count == 0:373            update_feedback['warnings'].append('No rules to deconflict')374        elif deconflict_count == 1:375            update_feedback['changes'].append('Deconflicted 1 rule')376        else:377            msg = 'Deconflicted {} Rules'.format(deconflict_count)378            update_feedback['changes'].append(msg)379        return update_feedback380class YaraRuleManager(models.Manager):381    def get_queryset(self):382        return YaraRuleQueryset(self.model, using=self._db)383    def category_options(self, group):384        return group.groupmeta.category_options385    def source_options(self, group):386        return group.groupmeta.source_options387    def process_parsed_rules(self, rules, source, category, submitter, owner, status='active',388                             add_tags=None, add_metadata=None, prepend_name=None, append_name=None,389                             force_source=False, force_category=False):390        # Container for results391        feedback = {'errors': [],392                    'warnings': [],393                    'rule_upload_count': 0,394                    'rule_collision_count': 0}395        # Ensure specified source is valid396        if not owner.groupmeta.source_required and not source:397            pass398        elif owner.groupmeta.source_required and not source:399            feedback['errors'].append('No Source Specified')400        elif source not in owner.groupmeta.source_options:401            if force_source:402                owner.groupmeta.source_options.append(source)403                owner.groupmeta.save()404            else:405                feedback['errors'].append('Invalid Source Specified: {}'.format(source))406        # Ensure specified category is valid407        if not owner.groupmeta.category_required and not category:408            pass409        elif owner.groupmeta.category_required and not category:410            feedback['errors'].append('No Category Specified')411        elif category not in owner.groupmeta.category_options:412            if force_category:413                owner.groupmeta.category_options.append(category)414                owner.groupmeta.save()415            else:416                feedback['errors'].append('Invalid Category Specified: {}'.format(category))417        # Rules must have a non-anonymous submitter and must not have pre-processing errors418        if not submitter.is_anonymous and not feedback['errors']:419            prepend_conflicts = 0420            append_conflicts = 0421            for rule in rules:422                rule_kwargs = generate_kwargs_from_parsed_rule(rule)423                rule_kwargs['owner'] = owner424                rule_kwargs['submitter'] = submitter425                rule_kwargs['source'] = source426                rule_kwargs['category'] = category427                rule_kwargs['status'] = status428                # Pop comments from kwargs so they don't get processed prematurely429                comments = rule_kwargs.pop('comments')430                # Process Modifications431                if add_tags:432                    if isinstance(add_tags, str):433                        add_tags = delimit_filtervalue(add_tags)434                    for tag_value in add_tags:435                        if check_lexical_convention(tag_value):436                            if tag_value not in rule_kwargs['tags']:437                                rule_kwargs['tags'].append(tag_value)438                        else:439                            msg = 'Skipped Invalid Tag: {}'.format(tag_value)440                            if msg not in feedback['warnings']:441                                feedback['warnings'].append(msg)442                if add_metadata:443                    for metakey, metavalue in add_metadata.items():444                        if check_lexical_convention(metakey) and \445                        (metavalue.isdigit() or metavalue in ('true', 'false') or \446                        (metavalue.startswith('\"') and metavalue.endswith('\"'))):447                            rule_kwargs['metadata'][metakey] = metavalue448                        else:449                            msg = 'Skipped Invalid Metadata: {}'.format(metakey)450                            if msg not in feedback['warnings']:451                                feedback['warnings'].append(msg)452                if prepend_name:453                    new_name = prepend_name + rule_kwargs['name']454                    if check_lexical_convention(new_name):455                        rule_kwargs['name'] = new_name456                    else:457                        prepend_conflicts += 1458                if append_name:459                    new_name = rule_kwargs['name'] + append_name460                    if check_lexical_convention(new_name):461                        rule_kwargs['name'] = new_name462                    else:463                        append_conflicts += 1464                # Check for rules with exact same detection logic465                if self.filter(owner=owner, logic_hash=rule_kwargs['logic_hash']).exists():466                    feedback['rule_collision_count'] += 1467                else:468                    new_rule = self.create(**rule_kwargs)469                    new_rule.save()470                    # Process extracted comments471                    new_rule.yararulecomment_set.model.objects.process_extracted_comments(new_rule, comments)472                    feedback['rule_upload_count'] += 1473            # Check to see if any name manipulation conflicts occurred for feedback474            if prepend_conflicts:475                msg = 'Unable To Prepend {} Rule Names'.format(prepend_conflicts)476                feedback['warnings'].append(msg)477            if append_conflicts:478                msg = 'Unable To Append {} Rule Names'.format(append_conflicts)479                feedback['warnings'].append(msg)480        return feedback481class YaraRuleCommentManager(models.Manager):482    def process_extracted_comments(self, rule, comments):483        # Generate comments from parsed comment data484        for comment in comments:485            comment_data = {'created': datetime.datetime.now(),486                            'modified': datetime.datetime.now(),487                            'poster': get_admin_account(),488                            'content': comment, 'rule': rule}...

test_utils.py

Source:test_utils.py

1from itertools import product2import numpy as np3import pytest4from alibi_detect.cd.utils import update_reference5n = [3, 50]6n_features = [1, 10]7update_method = [8    None,9    'last',10    'reservoir_sampling'11]12tests_update = list(product(n, n_features, update_method))13n_tests_update = len(tests_update)14@pytest.fixture15def update_params(request):16    return tests_update[request.param]17@pytest.mark.parametrize('update_params', list(range(n_tests_update)), indirect=True)18def test_update_reference(update_params):19    n, n_features, update_method = update_params20    n_ref = np.random.randint(1, n)21    n_test = np.random.randint(1, 2 * n)22    X_ref = np.random.rand(n_ref * n_features).reshape(n_ref, n_features)23    X = np.random.rand(n_test * n_features).reshape(n_test, n_features)24    if update_method in ['last', 'reservoir_sampling']:25        update_method = {update_method: n}26    X_ref_new = update_reference(X_ref, X, n, update_method)27    assert X_ref_new.shape[0] <= n28    if isinstance(update_method, dict):29        if list(update_method.keys())[0] == 'last':...

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.