How to use run_project method in Lemoncheesecake

Best Python code snippet using lemoncheesecake

qsub_test.py

Source:qsub_test.py

1import testutils2# ---------------------------------------------------------------------------------3def test_qsub_all_options_1():4 """5 qsub test run: all_options_16 """7 args = """-v -A myproj --attrs=a=1:b=2 --cwd /​tmp -d --debuglog=/​tmp/​d --dependencies=1:2:3 -e /​tmp/​e --env v1=1:v2=2 --geometry 198x198x198x198 -h -i /​bin/​ls -M myemal@gmail.com -n10 -o /​tmp/​o -O tmp --proccount 10 -qqueue --run_users user1:user2:user3 --run_project -t 10 --mode script --kernel kernel -K kopts /​bin/​ls"""8 cmdout = \9"""110"""11 cmderr = \12"""13qsub.py -v -A myproj --attrs=a=1:b=2 --cwd /​tmp -d --debuglog=/​tmp/​d --dependencies=1:2:3 -e /​tmp/​e --env v1=1:v2=2 --geometry 198x198x198x198 -h -i /​bin/​ls -M myemal@gmail.com -n10 -o /​tmp/​o -O tmp --proccount 10 -qqueue --run_users user1:user2:user3 --run_project -t 10 --mode script --kernel kernel -K kopts /​bin/​ls14component: "queue-manager.get_jobs", defer: True15 get_jobs(16 [{'jobid': 1}, {'jobid': 3}, {'jobid': 2}],17 )18component: "system.validate_job", defer: False19 validate_job(20 {'kernel': 'kernel', 'verbose': True, 'held': True, 'notify': 'myemal@gmail.com', 'ion_kerneloptions': False, 'project': 'myproj', 'preemptable': False, 'forcenoval': False, 'umask': False, 'version': False, 'env': 'v1=1:v2=2', 'cwd': '/​tmp', 'run_project': True, 'outputprefix': 'tmp', 'kerneloptions': 'kopts', 'time': '10', 'jobname': False, 'debug': True, 'dependencies': '1:2:3', 'debuglog': '/​tmp/​d', 'qsub_host': 'foo.bar', 'ion_kernel': 'default', 'proccount': '10', 'disable_preboot': False, 'geometry': '198x198x198x198', 'queue': 'queue', 'mode': 'script', 'error': '/​tmp/​e', 'nodecount': '10', 'output': '/​tmp/​o', 'inputfile': '/​bin/​ls', 'attrs': {'a': '1', 'b': '2'}, 'user_list': 'user1:user2:user3', 'interactive': False},21 )22component: "system.get_implementation", defer: False23 get_implementation(24 )25fd 1 not associated with a terminal device26component: "queue-manager.add_jobs", defer: False27 add_jobs(28 [{'kernel': 'kernel', 'errorpath': '/​tmp/​e', 'outputpath': '/​tmp/​o', 'tag': 'job', 'notify': 'myemal@gmail.com', 'outputdir': '/​tmp', 'queue': 'queue', 'envs': {'v1': '1', 'v2': '2'}, 'umask': 18, 'submithost': 'foo.bar', 'nodes': 10, 'cwd': '/​tmp', 'run_project': True, 'ttysession': None, 'kerneloptions': 'kopts', 'args': [], 'cobalt_log_file': '/​tmp/​d', 'user': 'gooduser', 'path': '/​tmp', 'ion_kernel': 'default', 'procs': '10', 'walltime': '10', 'geometry': [198, 198, 198, 198, 2], 'user_hold': True, 'jobid': '*', 'project': 'myproj', 'script_preboot': True, 'command': '/​bin/​ls', 'mode': 'script', 'all_dependencies': '1:2:3', 'attrs': {'a': '1', 'b': '2'}, 'user_list': ['gooduser', 'user1', 'user2', 'user3'], 'inputfile': '/​bin/​ls'}],29 )30Environment Vars: {'v1': '1', 'v2': '2'}31"""32 stubout = \33"""34GET_JOBS35jobid:136jobid type: <type 'int'>37jobid:338jobid type: <type 'int'>39jobid:240jobid type: <type 'int'>41GET_IMPLEMENTATION42ADD_JOBS43all_dependencies:1:2:344all_dependencies type: <type 'str'>45args:[]46args type: <type 'list'>47attrs:{'a': '1', 'b': '2'}48attrs type: <type 'dict'>49cobalt_log_file:/​tmp/​d50cobalt_log_file type: <type 'str'>51command:/​bin/​ls52command type: <type 'str'>53cwd:/​tmp54cwd type: <type 'str'>55envs:{'v1': '1', 'v2': '2'}56envs type: <type 'dict'>57errorpath:/​tmp/​e58errorpath type: <type 'str'>59geometry:[198, 198, 198, 198, 2]60geometry type: <type 'list'>61inputfile:/​bin/​ls62inputfile type: <type 'str'>63ion_kernel:default64ion_kernel type: <type 'str'>65jobid:*66jobid type: <type 'str'>67kernel:kernel68kernel type: <type 'str'>69kerneloptions:kopts70kerneloptions type: <type 'str'>71mode:script72mode type: <type 'str'>73nodes:1074nodes type: <type 'int'>75notify:myemal@gmail.com76notify type: <type 'str'>77outputdir:/​tmp78outputdir type: <type 'str'>79outputpath:/​tmp/​o80outputpath type: <type 'str'>81path:/​tmp82path type: <type 'str'>83procs:1084procs type: <type 'str'>85project:myproj86project type: <type 'str'>87queue:queue88queue type: <type 'str'>89run_project:True90run_project type: <type 'bool'>91script_preboot:True92script_preboot type: <type 'bool'>93submithost:foo.bar94submithost type: <type 'str'>95tag:job96tag type: <type 'str'>97ttysession:None98ttysession type: <type 'NoneType'>99umask:18100umask type: <type 'int'>101user:gooduser102user type: <type 'str'>103user_hold:True104user_hold type: <type 'bool'>105user_list:['gooduser', 'user1', 'user2', 'user3']106user_list type: <type 'list'>107walltime:10108walltime type: <type 'str'>109VALIDATE_JOB110attrs:{'a': '1', 'b': '2'}111attrs type: <type 'dict'>112cwd:/​tmp113cwd type: <type 'str'>114debug:True115debug type: <type 'bool'>116debuglog:/​tmp/​d117debuglog type: <type 'str'>118dependencies:1:2:3119dependencies type: <type 'str'>120disable_preboot:False121disable_preboot type: <type 'bool'>122env:v1=1:v2=2123env type: <type 'str'>124error:/​tmp/​e125error type: <type 'str'>126forcenoval:False127forcenoval type: <type 'bool'>128geometry:198x198x198x198129geometry type: <type 'str'>130held:True131held type: <type 'bool'>132inputfile:/​bin/​ls133inputfile type: <type 'str'>134interactive:False135interactive type: <type 'bool'>136ion_kernel:default137ion_kernel type: <type 'str'>138ion_kerneloptions:False139ion_kerneloptions type: <type 'bool'>140jobname:False141jobname type: <type 'bool'>142kernel:kernel143kernel type: <type 'str'>144kerneloptions:kopts145kerneloptions type: <type 'str'>146mode:script147mode type: <type 'str'>148nodecount:10149nodecount type: <type 'str'>150notify:myemal@gmail.com151notify type: <type 'str'>152output:/​tmp/​o153output type: <type 'str'>154outputprefix:tmp155outputprefix type: <type 'str'>156preemptable:False157preemptable type: <type 'bool'>158proccount:10159proccount type: <type 'str'>160project:myproj161project type: <type 'str'>162qsub_host:foo.bar163qsub_host type: <type 'str'>164queue:queue165queue type: <type 'str'>166run_project:True167run_project type: <type 'bool'>168time:10169time type: <type 'str'>170umask:False171umask type: <type 'bool'>172user_list:user1:user2:user3173user_list type: <type 'str'>174verbose:True175verbose type: <type 'bool'>176version:False177version type: <type 'bool'>178"""179 stubout_file = "stub.out"180 expected_results = ( 181 0, # Expected return status 182 cmdout, # Expected command output183 stubout, # Expected stub functions output184 cmderr, # Expected command error output 185 ) 186 testutils.save_testhook("")187 results = testutils.run_cmd('qsub.py',args,stubout_file) 188 result = testutils.validate_results(results,expected_results)189 testutils.remove_testhook()190 correct = 1191 assert result == correct, "Result:\n%s" % result192# ---------------------------------------------------------------------------------193def test_qsub_misc_1():194 """195 qsub test run: misc_1196 """197 args = """--mode script -n 512 --env BG_COREDUMPDISABLED=1 --proccount 512 -t 30 -q testing /​bin/​ls"""198 cmdout = \199"""1200"""201 cmderr = ''202 stubout = \203"""204GET_IMPLEMENTATION205ADD_JOBS206args:[]207args type: <type 'list'>208attrs:{}209attrs type: <type 'dict'>210command:/​bin/​ls211command type: <type 'str'>212cwd:/​tmp213cwd type: <type 'str'>214envs:{'BG_COREDUMPDISABLED': '1'}215envs type: <type 'dict'>216ion_kernel:default217ion_kernel type: <type 'str'>218jobid:*219jobid type: <type 'str'>220kernel:default221kernel type: <type 'str'>222mode:script223mode type: <type 'str'>224nodes:512225nodes type: <type 'int'>226outputdir:/​tmp227outputdir type: <type 'str'>228path:/​tmp229path type: <type 'str'>230procs:512231procs type: <type 'str'>232queue:testing233queue type: <type 'str'>234run_project:False235run_project type: <type 'bool'>236script_preboot:True237script_preboot type: <type 'bool'>238submithost:foo.bar239submithost type: <type 'str'>240tag:job241tag type: <type 'str'>242ttysession:None243ttysession type: <type 'NoneType'>244umask:18245umask type: <type 'int'>246user:gooduser247user type: <type 'str'>248user_list:['gooduser']249user_list type: <type 'list'>250walltime:30251walltime type: <type 'str'>252VALIDATE_JOB253attrs:{}254attrs type: <type 'dict'>255cwd:/​tmp256cwd type: <type 'str'>257debug:False258debug type: <type 'bool'>259debuglog:False260debuglog type: <type 'bool'>261dependencies:False262dependencies type: <type 'bool'>263disable_preboot:False264disable_preboot type: <type 'bool'>265env:BG_COREDUMPDISABLED=1266env type: <type 'str'>267error:False268error type: <type 'bool'>269forcenoval:False270forcenoval type: <type 'bool'>271geometry:False272geometry type: <type 'bool'>273held:False274held type: <type 'bool'>275inputfile:False276inputfile type: <type 'bool'>277interactive:False278interactive type: <type 'bool'>279ion_kernel:default280ion_kernel type: <type 'str'>281ion_kerneloptions:False282ion_kerneloptions type: <type 'bool'>283jobname:False284jobname type: <type 'bool'>285kernel:default286kernel type: <type 'str'>287kerneloptions:False288kerneloptions type: <type 'bool'>289mode:script290mode type: <type 'str'>291nodecount:512292nodecount type: <type 'str'>293notify:False294notify type: <type 'bool'>295output:False296output type: <type 'bool'>297outputprefix:False298outputprefix type: <type 'bool'>299preemptable:False300preemptable type: <type 'bool'>301proccount:512302proccount type: <type 'str'>303project:False304project type: <type 'bool'>305qsub_host:foo.bar306qsub_host type: <type 'str'>307queue:testing308queue type: <type 'str'>309run_project:False310run_project type: <type 'bool'>311time:30312time type: <type 'str'>313umask:False314umask type: <type 'bool'>315user_list:False316user_list type: <type 'bool'>317verbose:False318verbose type: <type 'bool'>319version:False320version type: <type 'bool'>321"""322 stubout_file = "stub.out"323 expected_results = ( 324 0, # Expected return status 325 cmdout, # Expected command output326 stubout, # Expected stub functions output327 cmderr, # Expected command error output 328 ) 329 testutils.save_testhook("")330 results = testutils.run_cmd('qsub.py',args,stubout_file) 331 result = testutils.validate_results(results,expected_results)332 testutils.remove_testhook()333 correct = 1334 assert result == correct, "Result:\n%s" % result335# ---------------------------------------------------------------------------------336def test_qsub_no_options_passed():337 """338 qsub test run: no_options_passed339 """340 args = """/​bin/​ls"""341 cmdout = \342"""Usage: qsub.py --help343Usage: qsub.py [options] <executable> [<excutable options>]344Refer to man pages for JOBID EXPANSION and SCRIPT JOB DIRECTIVES.345"""346 cmderr = \347"""No required options provided348"""349 stubout = ''350 stubout_file = "stub.out"351 expected_results = ( 352 256, # Expected return status 353 cmdout, # Expected command output354 stubout, # Expected stub functions output355 cmderr, # Expected command error output 356 ) 357 testutils.save_testhook("")358 results = testutils.run_cmd('qsub.py',args,stubout_file) 359 result = testutils.validate_results(results,expected_results)360 testutils.remove_testhook()361 correct = 1362 assert result == correct, "Result:\n%s" % result363# ---------------------------------------------------------------------------------364def test_qsub_non_existant_option():365 """366 qsub test run: non_existant_option367 """368 args = """-z -t10 -n10 /​bin/​ls"""369 cmdout = ''370 cmderr = \371"""Usage: qsub.py --help372Usage: qsub.py [options] <executable> [<excutable options>]373Refer to man pages for JOBID EXPANSION and SCRIPT JOB DIRECTIVES.374qsub.py: error: no such option: -z375"""376 stubout = ''377 stubout_file = "stub.out"378 expected_results = ( 379 512, # Expected return status 380 cmdout, # Expected command output381 stubout, # Expected stub functions output382 cmderr, # Expected command error output 383 ) 384 testutils.save_testhook("")385 results = testutils.run_cmd('qsub.py',args,stubout_file) 386 result = testutils.validate_results(results,expected_results)387 testutils.remove_testhook()388 correct = 1389 assert result == correct, "Result:\n%s" % result390# ---------------------------------------------------------------------------------391def test_qsub_debug_flag_only_1():392 """393 qsub test run: debug_flag_only_1394 """395 args = """-d"""396 cmdout = \397"""Usage: qsub.py --help398Usage: qsub.py [options] <executable> [<excutable options>]399Refer to man pages for JOBID EXPANSION and SCRIPT JOB DIRECTIVES.400"""401 cmderr = \402"""403qsub.py -d404No executable or script specified405"""406 stubout = ''407 stubout_file = "stub.out"408 expected_results = ( 409 256, # Expected return status 410 cmdout, # Expected command output411 stubout, # Expected stub functions output412 cmderr, # Expected command error output 413 ) 414 testutils.save_testhook("")415 results = testutils.run_cmd('qsub.py',args,stubout_file) 416 result = testutils.validate_results(results,expected_results)417 testutils.remove_testhook()418 correct = 1419 assert result == correct, "Result:\n%s" % result420# ---------------------------------------------------------------------------------421def test_qsub_debug_flag_only_2():422 """423 qsub test run: debug_flag_only_2424 """425 args = """-debug"""426 cmdout = \427"""Usage: qsub.py --help428Usage: qsub.py [options] <executable> [<excutable options>]429Refer to man pages for JOBID EXPANSION and SCRIPT JOB DIRECTIVES.430"""431 cmderr = \432"""433qsub.py -debug434No executable or script specified435"""436 stubout = ''437 stubout_file = "stub.out"438 expected_results = ( 439 256, # Expected return status 440 cmdout, # Expected command output441 stubout, # Expected stub functions output442 cmderr, # Expected command error output 443 ) 444 testutils.save_testhook("")445 results = testutils.run_cmd('qsub.py',args,stubout_file) 446 result = testutils.validate_results(results,expected_results)447 testutils.remove_testhook()448 correct = 1449 assert result == correct, "Result:\n%s" % result450# ---------------------------------------------------------------------------------451def test_qsub_verbose_flag_only():452 """453 qsub test run: verbose_flag_only454 """455 args = """-v"""456 cmdout = \457"""Usage: qsub.py --help458Usage: qsub.py [options] <executable> [<excutable options>]459Refer to man pages for JOBID EXPANSION and SCRIPT JOB DIRECTIVES.460"""461 cmderr = \462"""No executable or script specified463"""464 stubout = ''465 stubout_file = "stub.out"466 expected_results = ( 467 256, # Expected return status 468 cmdout, # Expected command output469 stubout, # Expected stub functions output470 cmderr, # Expected command error output 471 ) 472 testutils.save_testhook("")473 results = testutils.run_cmd('qsub.py',args,stubout_file) 474 result = testutils.validate_results(results,expected_results)475 testutils.remove_testhook()476 correct = 1477 assert result == correct, "Result:\n%s" % result478# ---------------------------------------------------------------------------------479def test_qsub_non_integer_nodecount():480 """481 qsub test run: non_integer_nodecount482 """483 args = """--mode smp -t50 -nfive --geometry 40x40x50x50 /​bin/​ls"""484 cmdout = ''485 cmderr = \486"""Specifed mode 'smp' not valid, valid modes are487co488vn489script490interactive491"""492 stubout = ''493 stubout_file = "stub.out"494 expected_results = ( 495 256, # Expected return status 496 cmdout, # Expected command output497 stubout, # Expected stub functions output498 cmderr, # Expected command error output 499 ) 500 testutils.save_testhook("")501 results = testutils.run_cmd('qsub.py',args,stubout_file) 502 result = testutils.validate_results(results,expected_results)503 testutils.remove_testhook()504 correct = 1505 assert result == correct, "Result:\n%s" % result506# ---------------------------------------------------------------------------------507def test_qsub_non_realistic_nodecount():508 """509 qsub test run: non_realistic_nodecount510 """511 args = """--mode smp -t50 -n2048 --geometry 40x40x50x50x1 /​bin/​ls"""512 cmdout = ''513 cmderr = \514"""Specifed mode 'smp' not valid, valid modes are515co516vn517script518interactive519"""520 stubout = ''521 stubout_file = "stub.out"522 expected_results = ( 523 256, # Expected return status 524 cmdout, # Expected command output525 stubout, # Expected stub functions output526 cmderr, # Expected command error output 527 ) 528 testutils.save_testhook("")529 results = testutils.run_cmd('qsub.py',args,stubout_file) 530 result = testutils.validate_results(results,expected_results)531 testutils.remove_testhook()532 correct = 1533 assert result == correct, "Result:\n%s" % result534# ---------------------------------------------------------------------------------535def test_qsub_invalid_geometry_1():536 """537 qsub test run: invalid_geometry_1538 """539 args = """--mode script -t50 -n10 --geometry x /​bin/​ls"""540 cmdout = ''541 cmderr = \542"""Invalid geometry entered: 543"""544 stubout = ''545 stubout_file = "stub.out"546 expected_results = ( 547 256, # Expected return status 548 cmdout, # Expected command output549 stubout, # Expected stub functions output550 cmderr, # Expected command error output 551 ) 552 testutils.save_testhook("")553 results = testutils.run_cmd('qsub.py',args,stubout_file) 554 result = testutils.validate_results(results,expected_results)555 testutils.remove_testhook()556 correct = 1557 assert result == correct, "Result:\n%s" % result558# ---------------------------------------------------------------------------------559def test_qsub_invalid_geometry_2():560 """561 qsub test run: invalid_geometry_2562 """563 args = """--mode script -t50 -n10 --geometry 1x2x3x4 /​bin/​ls"""564 cmdout = \565"""1566"""567 cmderr = ''568 stubout = \569"""570GET_IMPLEMENTATION571ADD_JOBS572args:[]573args type: <type 'list'>574attrs:{}575attrs type: <type 'dict'>576command:/​bin/​ls577command type: <type 'str'>578cwd:/​tmp579cwd type: <type 'str'>580geometry:[1, 2, 3, 4, 2]581geometry type: <type 'list'>582ion_kernel:default583ion_kernel type: <type 'str'>584jobid:*585jobid type: <type 'str'>586kernel:default587kernel type: <type 'str'>588mode:script589mode type: <type 'str'>590nodes:10591nodes type: <type 'int'>592outputdir:/​tmp593outputdir type: <type 'str'>594path:/​tmp595path type: <type 'str'>596procs:512597procs type: <type 'str'>598queue:default599queue type: <type 'str'>600run_project:False601run_project type: <type 'bool'>602script_preboot:True603script_preboot type: <type 'bool'>604submithost:foo.bar605submithost type: <type 'str'>606tag:job607tag type: <type 'str'>608ttysession:None609ttysession type: <type 'NoneType'>610umask:18611umask type: <type 'int'>612user:gooduser613user type: <type 'str'>614user_list:['gooduser']615user_list type: <type 'list'>616walltime:50617walltime type: <type 'str'>618VALIDATE_JOB619attrs:{}620attrs type: <type 'dict'>621cwd:/​tmp622cwd type: <type 'str'>623debug:False624debug type: <type 'bool'>625debuglog:False626debuglog type: <type 'bool'>627dependencies:False628dependencies type: <type 'bool'>629disable_preboot:False630disable_preboot type: <type 'bool'>631env:False632env type: <type 'bool'>633error:False634error type: <type 'bool'>635forcenoval:False636forcenoval type: <type 'bool'>637geometry:1x2x3x4638geometry type: <type 'str'>639held:False640held type: <type 'bool'>641inputfile:False642inputfile type: <type 'bool'>643interactive:False644interactive type: <type 'bool'>645ion_kernel:default646ion_kernel type: <type 'str'>647ion_kerneloptions:False648ion_kerneloptions type: <type 'bool'>649jobname:False650jobname type: <type 'bool'>651kernel:default652kernel type: <type 'str'>653kerneloptions:False654kerneloptions type: <type 'bool'>655mode:script656mode type: <type 'str'>657nodecount:10658nodecount type: <type 'str'>659notify:False660notify type: <type 'bool'>661output:False662output type: <type 'bool'>663outputprefix:False664outputprefix type: <type 'bool'>665preemptable:False666preemptable type: <type 'bool'>667proccount:False668proccount type: <type 'bool'>669project:False670project type: <type 'bool'>671qsub_host:foo.bar672qsub_host type: <type 'str'>673queue:default674queue type: <type 'str'>675run_project:False676run_project type: <type 'bool'>677time:50678time type: <type 'str'>679umask:False680umask type: <type 'bool'>681user_list:False682user_list type: <type 'bool'>683verbose:False684verbose type: <type 'bool'>685version:False686version type: <type 'bool'>687"""688 stubout_file = "stub.out"689 expected_results = ( 690 0, # Expected return status 691 cmdout, # Expected command output692 stubout, # Expected stub functions output693 cmderr, # Expected command error output 694 ) 695 testutils.save_testhook("")696 results = testutils.run_cmd('qsub.py',args,stubout_file) 697 result = testutils.validate_results(results,expected_results)698 testutils.remove_testhook()699 correct = 1700 assert result == correct, "Result:\n%s" % result701# ---------------------------------------------------------------------------------702def test_qsub_invalid_geometry_3():703 """704 qsub test run: invalid_geometry_3705 """706 args = """--mode script -t50 -n10 --geometry 1x2x3x4 /​bin/​ls"""707 cmdout = \708"""1709"""710 cmderr = ''711 stubout = \712"""713GET_IMPLEMENTATION714ADD_JOBS715args:[]716args type: <type 'list'>717attrs:{}718attrs type: <type 'dict'>719command:/​bin/​ls720command type: <type 'str'>721cwd:/​tmp722cwd type: <type 'str'>723geometry:[1, 2, 3, 4, 2]724geometry type: <type 'list'>725ion_kernel:default726ion_kernel type: <type 'str'>727jobid:*728jobid type: <type 'str'>729kernel:default730kernel type: <type 'str'>731mode:script732mode type: <type 'str'>733nodes:10734nodes type: <type 'int'>735outputdir:/​tmp736outputdir type: <type 'str'>737path:/​tmp738path type: <type 'str'>739procs:512740procs type: <type 'str'>741queue:default742queue type: <type 'str'>743run_project:False744run_project type: <type 'bool'>745script_preboot:True746script_preboot type: <type 'bool'>747submithost:foo.bar748submithost type: <type 'str'>749tag:job750tag type: <type 'str'>751ttysession:None752ttysession type: <type 'NoneType'>753umask:18754umask type: <type 'int'>755user:gooduser756user type: <type 'str'>757user_list:['gooduser']758user_list type: <type 'list'>759walltime:50760walltime type: <type 'str'>761VALIDATE_JOB762attrs:{}763attrs type: <type 'dict'>764cwd:/​tmp765cwd type: <type 'str'>766debug:False767debug type: <type 'bool'>768debuglog:False769debuglog type: <type 'bool'>770dependencies:False771dependencies type: <type 'bool'>772disable_preboot:False773disable_preboot type: <type 'bool'>774env:False775env type: <type 'bool'>776error:False777error type: <type 'bool'>778forcenoval:False779forcenoval type: <type 'bool'>780geometry:1x2x3x4781geometry type: <type 'str'>782held:False783held type: <type 'bool'>784inputfile:False785inputfile type: <type 'bool'>786interactive:False787interactive type: <type 'bool'>788ion_kernel:default789ion_kernel type: <type 'str'>790ion_kerneloptions:False791ion_kerneloptions type: <type 'bool'>792jobname:False793jobname type: <type 'bool'>794kernel:default795kernel type: <type 'str'>796kerneloptions:False797kerneloptions type: <type 'bool'>798mode:script799mode type: <type 'str'>800nodecount:10801nodecount type: <type 'str'>802notify:False803notify type: <type 'bool'>804output:False805output type: <type 'bool'>806outputprefix:False807outputprefix type: <type 'bool'>808preemptable:False809preemptable type: <type 'bool'>810proccount:False811proccount type: <type 'bool'>812project:False813project type: <type 'bool'>814qsub_host:foo.bar815qsub_host type: <type 'str'>816queue:default817queue type: <type 'str'>818run_project:False819run_project type: <type 'bool'>820time:50821time type: <type 'str'>822umask:False823umask type: <type 'bool'>824user_list:False825user_list type: <type 'bool'>826verbose:False827verbose type: <type 'bool'>828version:False829version type: <type 'bool'>830"""831 stubout_file = "stub.out"832 expected_results = ( 833 0, # Expected return status 834 cmdout, # Expected command output835 stubout, # Expected stub functions output836 cmderr, # Expected command error output 837 ) 838 testutils.save_testhook("")839 results = testutils.run_cmd('qsub.py',args,stubout_file) 840 result = testutils.validate_results(results,expected_results)841 testutils.remove_testhook()842 correct = 1843 assert result == correct, "Result:\n%s" % result844# ---------------------------------------------------------------------------------845def test_qsub_invalid_geometry_4():846 """847 qsub test run: invalid_geometry_4848 """849 args = """--mode script -t50 -n10 --geometry 48x48x48x48x2 /​bin/​ls"""850 cmdout = \851"""1852"""853 cmderr = ''854 stubout = \855"""856GET_IMPLEMENTATION857ADD_JOBS858args:[]859args type: <type 'list'>860attrs:{}861attrs type: <type 'dict'>862command:/​bin/​ls863command type: <type 'str'>864cwd:/​tmp865cwd type: <type 'str'>866geometry:[48, 48, 48, 48, 2]867geometry type: <type 'list'>868ion_kernel:default869ion_kernel type: <type 'str'>870jobid:*871jobid type: <type 'str'>872kernel:default873kernel type: <type 'str'>874mode:script875mode type: <type 'str'>876nodes:10877nodes type: <type 'int'>878outputdir:/​tmp879outputdir type: <type 'str'>880path:/​tmp881path type: <type 'str'>882procs:512883procs type: <type 'str'>884queue:default885queue type: <type 'str'>886run_project:False887run_project type: <type 'bool'>888script_preboot:True889script_preboot type: <type 'bool'>890submithost:foo.bar891submithost type: <type 'str'>892tag:job893tag type: <type 'str'>894ttysession:None895ttysession type: <type 'NoneType'>896umask:18897umask type: <type 'int'>898user:gooduser899user type: <type 'str'>900user_list:['gooduser']901user_list type: <type 'list'>902walltime:50903walltime type: <type 'str'>904VALIDATE_JOB905attrs:{}906attrs type: <type 'dict'>907cwd:/​tmp908cwd type: <type 'str'>909debug:False910debug type: <type 'bool'>911debuglog:False912debuglog type: <type 'bool'>913dependencies:False914dependencies type: <type 'bool'>915disable_preboot:False916disable_preboot type: <type 'bool'>917env:False918env type: <type 'bool'>919error:False920error type: <type 'bool'>921forcenoval:False922forcenoval type: <type 'bool'>923geometry:48x48x48x48x2924geometry type: <type 'str'>925held:False926held type: <type 'bool'>927inputfile:False928inputfile type: <type 'bool'>929interactive:False930interactive type: <type 'bool'>931ion_kernel:default932ion_kernel type: <type 'str'>933ion_kerneloptions:False934ion_kerneloptions type: <type 'bool'>935jobname:False936jobname type: <type 'bool'>937kernel:default938kernel type: <type 'str'>939kerneloptions:False940kerneloptions type: <type 'bool'>941mode:script942mode type: <type 'str'>943nodecount:10944nodecount type: <type 'str'>945notify:False946notify type: <type 'bool'>947output:False948output type: <type 'bool'>949outputprefix:False950outputprefix type: <type 'bool'>951preemptable:False952preemptable type: <type 'bool'>953proccount:False954proccount type: <type 'bool'>955project:False956project type: <type 'bool'>957qsub_host:foo.bar958qsub_host type: <type 'str'>959queue:default960queue type: <type 'str'>961run_project:False962run_project type: <type 'bool'>963time:50964time type: <type 'str'>965umask:False966umask type: <type 'bool'>967user_list:False968user_list type: <type 'bool'>969verbose:False970verbose type: <type 'bool'>971version:False972version type: <type 'bool'>973"""974 stubout_file = "stub.out"975 expected_results = ( 976 0, # Expected return status 977 cmdout, # Expected command output978 stubout, # Expected stub functions output979 cmderr, # Expected command error output 980 ) 981 testutils.save_testhook("")982 results = testutils.run_cmd('qsub.py',args,stubout_file) 983 result = testutils.validate_results(results,expected_results)984 testutils.remove_testhook()985 correct = 1986 assert result == correct, "Result:\n%s" % result987# ---------------------------------------------------------------------------------988def test_qsub_invalid_geometry_5():989 """990 qsub test run: invalid_geometry_5991 """992 args = """--mode script -t50 -n10 --geometry 48x48x48x48x3 /​bin/​ls"""993 cmdout = ''994 cmderr = \995"""Invalid geometry entered: 996"""997 stubout = ''998 stubout_file = "stub.out"999 expected_results = ( 1000 256, # Expected return status 1001 cmdout, # Expected command output1002 stubout, # Expected stub functions output1003 cmderr, # Expected command error output 1004 ) 1005 testutils.save_testhook("")1006 results = testutils.run_cmd('qsub.py',args,stubout_file) 1007 result = testutils.validate_results(results,expected_results)1008 testutils.remove_testhook()1009 correct = 11010 assert result == correct, "Result:\n%s" % result1011# ---------------------------------------------------------------------------------1012def test_qsub_invalid_geometry_6():1013 """1014 qsub test run: invalid_geometry_61015 """1016 args = """--mode script -t50 -n10 --geometry 128x64x32x4 /​bin/​ls"""1017 cmdout = \1018"""11019"""1020 cmderr = ''1021 stubout = \1022"""1023GET_IMPLEMENTATION1024ADD_JOBS1025args:[]1026args type: <type 'list'>1027attrs:{}1028attrs type: <type 'dict'>1029command:/​bin/​ls1030command type: <type 'str'>1031cwd:/​tmp1032cwd type: <type 'str'>1033geometry:[128, 64, 32, 4, 2]1034geometry type: <type 'list'>1035ion_kernel:default1036ion_kernel type: <type 'str'>1037jobid:*1038jobid type: <type 'str'>1039kernel:default1040kernel type: <type 'str'>1041mode:script1042mode type: <type 'str'>1043nodes:101044nodes type: <type 'int'>1045outputdir:/​tmp1046outputdir type: <type 'str'>1047path:/​tmp1048path type: <type 'str'>1049procs:5121050procs type: <type 'str'>1051queue:default1052queue type: <type 'str'>1053run_project:False1054run_project type: <type 'bool'>1055script_preboot:True1056script_preboot type: <type 'bool'>1057submithost:foo.bar1058submithost type: <type 'str'>1059tag:job1060tag type: <type 'str'>1061ttysession:None1062ttysession type: <type 'NoneType'>1063umask:181064umask type: <type 'int'>1065user:gooduser1066user type: <type 'str'>1067user_list:['gooduser']1068user_list type: <type 'list'>1069walltime:501070walltime type: <type 'str'>1071VALIDATE_JOB1072attrs:{}1073attrs type: <type 'dict'>1074cwd:/​tmp1075cwd type: <type 'str'>1076debug:False1077debug type: <type 'bool'>1078debuglog:False1079debuglog type: <type 'bool'>1080dependencies:False1081dependencies type: <type 'bool'>1082disable_preboot:False1083disable_preboot type: <type 'bool'>1084env:False1085env type: <type 'bool'>1086error:False1087error type: <type 'bool'>1088forcenoval:False1089forcenoval type: <type 'bool'>1090geometry:128x64x32x41091geometry type: <type 'str'>1092held:False1093held type: <type 'bool'>1094inputfile:False1095inputfile type: <type 'bool'>1096interactive:False1097interactive type: <type 'bool'>1098ion_kernel:default1099ion_kernel type: <type 'str'>1100ion_kerneloptions:False1101ion_kerneloptions type: <type 'bool'>1102jobname:False1103jobname type: <type 'bool'>1104kernel:default1105kernel type: <type 'str'>1106kerneloptions:False1107kerneloptions type: <type 'bool'>1108mode:script1109mode type: <type 'str'>1110nodecount:101111nodecount type: <type 'str'>1112notify:False1113notify type: <type 'bool'>1114output:False1115output type: <type 'bool'>1116outputprefix:False1117outputprefix type: <type 'bool'>1118preemptable:False1119preemptable type: <type 'bool'>1120proccount:False1121proccount type: <type 'bool'>1122project:False1123project type: <type 'bool'>1124qsub_host:foo.bar1125qsub_host type: <type 'str'>1126queue:default1127queue type: <type 'str'>1128run_project:False1129run_project type: <type 'bool'>1130time:501131time type: <type 'str'>1132umask:False1133umask type: <type 'bool'>1134user_list:False1135user_list type: <type 'bool'>1136verbose:False1137verbose type: <type 'bool'>1138version:False1139version type: <type 'bool'>1140"""1141 stubout_file = "stub.out"1142 expected_results = ( 1143 0, # Expected return status 1144 cmdout, # Expected command output1145 stubout, # Expected stub functions output1146 cmderr, # Expected command error output 1147 ) 1148 testutils.save_testhook("")1149 results = testutils.run_cmd('qsub.py',args,stubout_file) 1150 result = testutils.validate_results(results,expected_results)1151 testutils.remove_testhook()1152 correct = 11153 assert result == correct, "Result:\n%s" % result1154# ---------------------------------------------------------------------------------1155def test_qsub_no_project_specified():1156 """1157 qsub test run: no_project_specified1158 """1159 args = """-A -t50 -n10 /​bin/​ls"""1160 cmdout = ''1161 cmderr = \1162"""'time' not provided1163"""1164 stubout = ''1165 stubout_file = "stub.out"1166 expected_results = ( 1167 256, # Expected return status 1168 cmdout, # Expected command output1169 stubout, # Expected stub functions output1170 cmderr, # Expected command error output 1171 ) 1172 testutils.save_testhook("")1173 results = testutils.run_cmd('qsub.py',args,stubout_file) 1174 result = testutils.validate_results(results,expected_results)1175 testutils.remove_testhook()1176 correct = 11177 assert result == correct, "Result:\n%s" % result1178# ---------------------------------------------------------------------------------1179def test_qsub_project_specified():1180 """1181 qsub test run: project_specified1182 """1183 args = """-A who -t50 -n10 /​bin/​ls"""1184 cmdout = \1185"""11186"""1187 cmderr = ''1188 stubout = \1189"""1190GET_IMPLEMENTATION1191ADD_JOBS1192args:[]1193args type: <type 'list'>1194attrs:{}1195attrs type: <type 'dict'>1196command:/​bin/​ls1197command type: <type 'str'>1198cwd:/​tmp1199cwd type: <type 'str'>1200ion_kernel:default1201ion_kernel type: <type 'str'>1202jobid:*1203jobid type: <type 'str'>1204kernel:default1205kernel type: <type 'str'>1206mode:c11207mode type: <type 'str'>1208nodes:101209nodes type: <type 'int'>1210outputdir:/​tmp1211outputdir type: <type 'str'>1212path:/​tmp1213path type: <type 'str'>1214procs:5121215procs type: <type 'str'>1216project:who1217project type: <type 'str'>1218queue:default1219queue type: <type 'str'>1220run_project:False1221run_project type: <type 'bool'>1222script_preboot:True1223script_preboot type: <type 'bool'>1224submithost:foo.bar1225submithost type: <type 'str'>1226tag:job1227tag type: <type 'str'>1228ttysession:None1229ttysession type: <type 'NoneType'>1230umask:181231umask type: <type 'int'>1232user:gooduser1233user type: <type 'str'>1234user_list:['gooduser']1235user_list type: <type 'list'>1236walltime:501237walltime type: <type 'str'>1238VALIDATE_JOB1239attrs:{}1240attrs type: <type 'dict'>1241cwd:/​tmp1242cwd type: <type 'str'>1243debug:False1244debug type: <type 'bool'>1245debuglog:False1246debuglog type: <type 'bool'>1247dependencies:False1248dependencies type: <type 'bool'>1249disable_preboot:False1250disable_preboot type: <type 'bool'>1251env:False1252env type: <type 'bool'>1253error:False1254error type: <type 'bool'>1255forcenoval:False1256forcenoval type: <type 'bool'>1257geometry:False1258geometry type: <type 'bool'>1259held:False1260held type: <type 'bool'>1261inputfile:False1262inputfile type: <type 'bool'>1263interactive:False1264interactive type: <type 'bool'>1265ion_kernel:default1266ion_kernel type: <type 'str'>1267ion_kerneloptions:False1268ion_kerneloptions type: <type 'bool'>1269jobname:False1270jobname type: <type 'bool'>1271kernel:default1272kernel type: <type 'str'>1273kerneloptions:False1274kerneloptions type: <type 'bool'>1275mode:False1276mode type: <type 'bool'>1277nodecount:101278nodecount type: <type 'str'>1279notify:False1280notify type: <type 'bool'>1281output:False1282output type: <type 'bool'>1283outputprefix:False1284outputprefix type: <type 'bool'>1285preemptable:False1286preemptable type: <type 'bool'>1287proccount:False1288proccount type: <type 'bool'>1289project:who1290project type: <type 'str'>1291qsub_host:foo.bar1292qsub_host type: <type 'str'>1293queue:default1294queue type: <type 'str'>1295run_project:False1296run_project type: <type 'bool'>1297time:501298time type: <type 'str'>1299umask:False1300umask type: <type 'bool'>1301user_list:False1302user_list type: <type 'bool'>1303verbose:False1304verbose type: <type 'bool'>1305version:False1306version type: <type 'bool'>1307"""1308 stubout_file = "stub.out"1309 expected_results = ( 1310 0, # Expected return status 1311 cmdout, # Expected command output1312 stubout, # Expected stub functions output1313 cmderr, # Expected command error output 1314 ) 1315 testutils.save_testhook("")1316 results = testutils.run_cmd('qsub.py',args,stubout_file) 1317 result = testutils.validate_results(results,expected_results)1318 testutils.remove_testhook()1319 correct = 11320 assert result == correct, "Result:\n%s" % result1321# ---------------------------------------------------------------------------------1322def test_qsub_Check_attrs_1():1323 """1324 qsub test run: Check_attrs_11325 """1326 args = """--attrs xxxx -t50 -n10 /​bin/​ls"""1327 cmdout = \1328"""11329"""1330 cmderr = ''1331 stubout = \1332"""1333GET_IMPLEMENTATION1334ADD_JOBS1335args:[]1336args type: <type 'list'>1337attrs:{'xxxx': 'true'}1338attrs type: <type 'dict'>1339command:/​bin/​ls1340command type: <type 'str'>1341cwd:/​tmp1342cwd type: <type 'str'>1343ion_kernel:default1344ion_kernel type: <type 'str'>1345jobid:*1346jobid type: <type 'str'>1347kernel:default1348kernel type: <type 'str'>1349mode:c11350mode type: <type 'str'>1351nodes:101352nodes type: <type 'int'>1353outputdir:/​tmp1354outputdir type: <type 'str'>1355path:/​tmp1356path type: <type 'str'>1357procs:5121358procs type: <type 'str'>1359queue:default1360queue type: <type 'str'>1361run_project:False1362run_project type: <type 'bool'>1363script_preboot:True1364script_preboot type: <type 'bool'>1365submithost:foo.bar1366submithost type: <type 'str'>1367tag:job1368tag type: <type 'str'>1369ttysession:None1370ttysession type: <type 'NoneType'>1371umask:181372umask type: <type 'int'>1373user:gooduser1374user type: <type 'str'>1375user_list:['gooduser']1376user_list type: <type 'list'>1377walltime:501378walltime type: <type 'str'>1379VALIDATE_JOB1380attrs:{'xxxx': 'true'}1381attrs type: <type 'dict'>1382cwd:/​tmp1383cwd type: <type 'str'>1384debug:False1385debug type: <type 'bool'>1386debuglog:False1387debuglog type: <type 'bool'>1388dependencies:False1389dependencies type: <type 'bool'>1390disable_preboot:False1391disable_preboot type: <type 'bool'>1392env:False1393env type: <type 'bool'>1394error:False1395error type: <type 'bool'>1396forcenoval:False1397forcenoval type: <type 'bool'>1398geometry:False1399geometry type: <type 'bool'>1400held:False1401held type: <type 'bool'>1402inputfile:False1403inputfile type: <type 'bool'>1404interactive:False1405interactive type: <type 'bool'>1406ion_kernel:default1407ion_kernel type: <type 'str'>1408ion_kerneloptions:False1409ion_kerneloptions type: <type 'bool'>1410jobname:False1411jobname type: <type 'bool'>1412kernel:default1413kernel type: <type 'str'>1414kerneloptions:False1415kerneloptions type: <type 'bool'>1416mode:False1417mode type: <type 'bool'>1418nodecount:101419nodecount type: <type 'str'>1420notify:False1421notify type: <type 'bool'>1422output:False1423output type: <type 'bool'>1424outputprefix:False1425outputprefix type: <type 'bool'>1426preemptable:False1427preemptable type: <type 'bool'>1428proccount:False1429proccount type: <type 'bool'>1430project:False1431project type: <type 'bool'>1432qsub_host:foo.bar1433qsub_host type: <type 'str'>1434queue:default1435queue type: <type 'str'>1436run_project:False1437run_project type: <type 'bool'>1438time:501439time type: <type 'str'>1440umask:False1441umask type: <type 'bool'>1442user_list:False1443user_list type: <type 'bool'>1444verbose:False1445verbose type: <type 'bool'>1446version:False1447version type: <type 'bool'>1448"""1449 stubout_file = "stub.out"1450 expected_results = ( 1451 0, # Expected return status 1452 cmdout, # Expected command output1453 stubout, # Expected stub functions output1454 cmderr, # Expected command error output 1455 ) 1456 testutils.save_testhook("")1457 results = testutils.run_cmd('qsub.py',args,stubout_file) 1458 result = testutils.validate_results(results,expected_results)1459 testutils.remove_testhook()1460 correct = 11461 assert result == correct, "Result:\n%s" % result1462# ---------------------------------------------------------------------------------1463def test_qsub_Check_attrs_2():1464 """1465 qsub test run: Check_attrs_21466 """1467 args = """--attrs 1111 -t50 -n10 /​bin/​ls"""1468 cmdout = \1469"""11470"""1471 cmderr = ''1472 stubout = \1473"""1474GET_IMPLEMENTATION1475ADD_JOBS1476args:[]1477args type: <type 'list'>1478attrs:{'1111': 'true'}1479attrs type: <type 'dict'>1480command:/​bin/​ls1481command type: <type 'str'>1482cwd:/​tmp1483cwd type: <type 'str'>1484ion_kernel:default1485ion_kernel type: <type 'str'>1486jobid:*1487jobid type: <type 'str'>1488kernel:default1489kernel type: <type 'str'>1490mode:c11491mode type: <type 'str'>1492nodes:101493nodes type: <type 'int'>1494outputdir:/​tmp1495outputdir type: <type 'str'>1496path:/​tmp1497path type: <type 'str'>1498procs:5121499procs type: <type 'str'>1500queue:default1501queue type: <type 'str'>1502run_project:False1503run_project type: <type 'bool'>1504script_preboot:True1505script_preboot type: <type 'bool'>1506submithost:foo.bar1507submithost type: <type 'str'>1508tag:job1509tag type: <type 'str'>1510ttysession:None1511ttysession type: <type 'NoneType'>1512umask:181513umask type: <type 'int'>1514user:gooduser1515user type: <type 'str'>1516user_list:['gooduser']1517user_list type: <type 'list'>1518walltime:501519walltime type: <type 'str'>1520VALIDATE_JOB1521attrs:{'1111': 'true'}1522attrs type: <type 'dict'>1523cwd:/​tmp1524cwd type: <type 'str'>1525debug:False1526debug type: <type 'bool'>1527debuglog:False1528debuglog type: <type 'bool'>1529dependencies:False1530dependencies type: <type 'bool'>1531disable_preboot:False1532disable_preboot type: <type 'bool'>1533env:False1534env type: <type 'bool'>1535error:False1536error type: <type 'bool'>1537forcenoval:False1538forcenoval type: <type 'bool'>1539geometry:False1540geometry type: <type 'bool'>1541held:False1542held type: <type 'bool'>1543inputfile:False1544inputfile type: <type 'bool'>1545interactive:False1546interactive type: <type 'bool'>1547ion_kernel:default1548ion_kernel type: <type 'str'>1549ion_kerneloptions:False1550ion_kerneloptions type: <type 'bool'>1551jobname:False1552jobname type: <type 'bool'>1553kernel:default1554kernel type: <type 'str'>1555kerneloptions:False1556kerneloptions type: <type 'bool'>1557mode:False1558mode type: <type 'bool'>1559nodecount:101560nodecount type: <type 'str'>1561notify:False1562notify type: <type 'bool'>1563output:False1564output type: <type 'bool'>1565outputprefix:False1566outputprefix type: <type 'bool'>1567preemptable:False1568preemptable type: <type 'bool'>1569proccount:False1570proccount type: <type 'bool'>1571project:False1572project type: <type 'bool'>1573qsub_host:foo.bar1574qsub_host type: <type 'str'>1575queue:default1576queue type: <type 'str'>1577run_project:False1578run_project type: <type 'bool'>1579time:501580time type: <type 'str'>1581umask:False1582umask type: <type 'bool'>1583user_list:False1584user_list type: <type 'bool'>1585verbose:False1586verbose type: <type 'bool'>1587version:False1588version type: <type 'bool'>1589"""1590 stubout_file = "stub.out"1591 expected_results = ( 1592 0, # Expected return status 1593 cmdout, # Expected command output1594 stubout, # Expected stub functions output1595 cmderr, # Expected command error output 1596 ) 1597 testutils.save_testhook("")1598 results = testutils.run_cmd('qsub.py',args,stubout_file) 1599 result = testutils.validate_results(results,expected_results)1600 testutils.remove_testhook()1601 correct = 11602 assert result == correct, "Result:\n%s" % result1603# ---------------------------------------------------------------------------------1604def test_qsub_Check_attrs_3():1605 """1606 qsub test run: Check_attrs_31607 """1608 args = """--attrs xx=:yy -t50 -n10 /​bin/​ls"""1609 cmdout = \1610"""11611"""1612 cmderr = ''1613 stubout = \1614"""1615GET_IMPLEMENTATION1616ADD_JOBS1617args:[]1618args type: <type 'list'>1619attrs:{'yy': 'true', 'xx': ''}1620attrs type: <type 'dict'>1621command:/​bin/​ls1622command type: <type 'str'>1623cwd:/​tmp1624cwd type: <type 'str'>1625ion_kernel:default1626ion_kernel type: <type 'str'>1627jobid:*1628jobid type: <type 'str'>1629kernel:default1630kernel type: <type 'str'>1631mode:c11632mode type: <type 'str'>1633nodes:101634nodes type: <type 'int'>1635outputdir:/​tmp1636outputdir type: <type 'str'>1637path:/​tmp1638path type: <type 'str'>1639procs:5121640procs type: <type 'str'>1641queue:default1642queue type: <type 'str'>1643run_project:False1644run_project type: <type 'bool'>1645script_preboot:True1646script_preboot type: <type 'bool'>1647submithost:foo.bar1648submithost type: <type 'str'>1649tag:job1650tag type: <type 'str'>1651ttysession:None1652ttysession type: <type 'NoneType'>1653umask:181654umask type: <type 'int'>1655user:gooduser1656user type: <type 'str'>1657user_list:['gooduser']1658user_list type: <type 'list'>1659walltime:501660walltime type: <type 'str'>1661VALIDATE_JOB1662attrs:{'yy': 'true', 'xx': ''}1663attrs type: <type 'dict'>1664cwd:/​tmp1665cwd type: <type 'str'>1666debug:False1667debug type: <type 'bool'>1668debuglog:False1669debuglog type: <type 'bool'>1670dependencies:False1671dependencies type: <type 'bool'>1672disable_preboot:False1673disable_preboot type: <type 'bool'>1674env:False1675env type: <type 'bool'>1676error:False1677error type: <type 'bool'>1678forcenoval:False1679forcenoval type: <type 'bool'>1680geometry:False1681geometry type: <type 'bool'>1682held:False1683held type: <type 'bool'>1684inputfile:False1685inputfile type: <type 'bool'>1686interactive:False1687interactive type: <type 'bool'>1688ion_kernel:default1689ion_kernel type: <type 'str'>1690ion_kerneloptions:False1691ion_kerneloptions type: <type 'bool'>1692jobname:False1693jobname type: <type 'bool'>1694kernel:default1695kernel type: <type 'str'>1696kerneloptions:False1697kerneloptions type: <type 'bool'>1698mode:False1699mode type: <type 'bool'>1700nodecount:101701nodecount type: <type 'str'>1702notify:False1703notify type: <type 'bool'>1704output:False1705output type: <type 'bool'>1706outputprefix:False1707outputprefix type: <type 'bool'>1708preemptable:False1709preemptable type: <type 'bool'>1710proccount:False1711proccount type: <type 'bool'>1712project:False1713project type: <type 'bool'>1714qsub_host:foo.bar1715qsub_host type: <type 'str'>1716queue:default1717queue type: <type 'str'>1718run_project:False1719run_project type: <type 'bool'>1720time:501721time type: <type 'str'>1722umask:False1723umask type: <type 'bool'>1724user_list:False1725user_list type: <type 'bool'>1726verbose:False1727verbose type: <type 'bool'>1728version:False1729version type: <type 'bool'>1730"""1731 stubout_file = "stub.out"1732 expected_results = ( 1733 0, # Expected return status 1734 cmdout, # Expected command output1735 stubout, # Expected stub functions output1736 cmderr, # Expected command error output 1737 ) 1738 testutils.save_testhook("")1739 results = testutils.run_cmd('qsub.py',args,stubout_file) 1740 result = testutils.validate_results(results,expected_results)1741 testutils.remove_testhook()1742 correct = 11743 assert result == correct, "Result:\n%s" % result1744# ---------------------------------------------------------------------------------1745def test_qsub_Check_attrs_4():1746 """1747 qsub test run: Check_attrs_41748 """1749 args = """--attrs xx=one:yy=1:zz=1one -t50 -n10 /​bin/​ls"""1750 cmdout = \1751"""11752"""1753 cmderr = ''1754 stubout = \1755"""1756GET_IMPLEMENTATION1757ADD_JOBS1758args:[]1759args type: <type 'list'>1760attrs:{'yy': '1', 'xx': 'one', 'zz': '1one'}1761attrs type: <type 'dict'>1762command:/​bin/​ls1763command type: <type 'str'>1764cwd:/​tmp1765cwd type: <type 'str'>1766ion_kernel:default1767ion_kernel type: <type 'str'>1768jobid:*1769jobid type: <type 'str'>1770kernel:default1771kernel type: <type 'str'>1772mode:c11773mode type: <type 'str'>1774nodes:101775nodes type: <type 'int'>1776outputdir:/​tmp1777outputdir type: <type 'str'>1778path:/​tmp1779path type: <type 'str'>1780procs:5121781procs type: <type 'str'>1782queue:default1783queue type: <type 'str'>1784run_project:False1785run_project type: <type 'bool'>1786script_preboot:True1787script_preboot type: <type 'bool'>1788submithost:foo.bar1789submithost type: <type 'str'>1790tag:job1791tag type: <type 'str'>1792ttysession:None1793ttysession type: <type 'NoneType'>1794umask:181795umask type: <type 'int'>1796user:gooduser1797user type: <type 'str'>1798user_list:['gooduser']1799user_list type: <type 'list'>1800walltime:501801walltime type: <type 'str'>1802VALIDATE_JOB1803attrs:{'yy': '1', 'xx': 'one', 'zz': '1one'}1804attrs type: <type 'dict'>1805cwd:/​tmp1806cwd type: <type 'str'>1807debug:False1808debug type: <type 'bool'>1809debuglog:False1810debuglog type: <type 'bool'>1811dependencies:False1812dependencies type: <type 'bool'>1813disable_preboot:False1814disable_preboot type: <type 'bool'>1815env:False1816env type: <type 'bool'>1817error:False1818error type: <type 'bool'>1819forcenoval:False1820forcenoval type: <type 'bool'>1821geometry:False1822geometry type: <type 'bool'>1823held:False1824held type: <type 'bool'>1825inputfile:False1826inputfile type: <type 'bool'>1827interactive:False1828interactive type: <type 'bool'>1829ion_kernel:default1830ion_kernel type: <type 'str'>1831ion_kerneloptions:False1832ion_kerneloptions type: <type 'bool'>1833jobname:False1834jobname type: <type 'bool'>1835kernel:default1836kernel type: <type 'str'>1837kerneloptions:False1838kerneloptions type: <type 'bool'>1839mode:False1840mode type: <type 'bool'>1841nodecount:101842nodecount type: <type 'str'>1843notify:False1844notify type: <type 'bool'>1845output:False1846output type: <type 'bool'>1847outputprefix:False1848outputprefix type: <type 'bool'>1849preemptable:False1850preemptable type: <type 'bool'>1851proccount:False1852proccount type: <type 'bool'>1853project:False1854project type: <type 'bool'>1855qsub_host:foo.bar1856qsub_host type: <type 'str'>1857queue:default1858queue type: <type 'str'>1859run_project:False1860run_project type: <type 'bool'>1861time:501862time type: <type 'str'>1863umask:False1864umask type: <type 'bool'>1865user_list:False1866user_list type: <type 'bool'>1867verbose:False1868verbose type: <type 'bool'>1869version:False1870version type: <type 'bool'>1871"""1872 stubout_file = "stub.out"1873 expected_results = ( 1874 0, # Expected return status 1875 cmdout, # Expected command output1876 stubout, # Expected stub functions output1877 cmderr, # Expected command error output 1878 ) 1879 testutils.save_testhook("")1880 results = testutils.run_cmd('qsub.py',args,stubout_file) 1881 result = testutils.validate_results(results,expected_results)1882 testutils.remove_testhook()1883 correct = 11884 assert result == correct, "Result:\n%s" % result1885# ---------------------------------------------------------------------------------1886def test_qsub_cwd_option_1():1887 """1888 qsub test run: cwd_option_11889 """1890 args = """--cwd /​tmp/​ -t10 -n 10 -e p /​bin/​ls"""1891 cmdout = \1892"""11893"""1894 cmderr = ''1895 stubout = \1896"""1897GET_IMPLEMENTATION1898ADD_JOBS1899args:[]1900args type: <type 'list'>1901attrs:{}1902attrs type: <type 'dict'>1903command:/​bin/​ls1904command type: <type 'str'>1905cwd:/​tmp/​1906cwd type: <type 'str'>1907errorpath:/​tmp/​/​p1908errorpath type: <type 'str'>1909ion_kernel:default1910ion_kernel type: <type 'str'>1911jobid:*1912jobid type: <type 'str'>1913kernel:default1914kernel type: <type 'str'>1915mode:c11916mode type: <type 'str'>1917nodes:101918nodes type: <type 'int'>1919outputdir:/​tmp/​1920outputdir type: <type 'str'>1921path:/​tmp1922path type: <type 'str'>1923procs:5121924procs type: <type 'str'>1925queue:default1926queue type: <type 'str'>1927run_project:False1928run_project type: <type 'bool'>1929script_preboot:True1930script_preboot type: <type 'bool'>1931submithost:foo.bar1932submithost type: <type 'str'>1933tag:job1934tag type: <type 'str'>1935ttysession:None1936ttysession type: <type 'NoneType'>1937umask:181938umask type: <type 'int'>1939user:gooduser1940user type: <type 'str'>1941user_list:['gooduser']1942user_list type: <type 'list'>1943walltime:101944walltime type: <type 'str'>1945VALIDATE_JOB1946attrs:{}1947attrs type: <type 'dict'>1948cwd:/​tmp/​1949cwd type: <type 'str'>1950debug:False1951debug type: <type 'bool'>1952debuglog:False1953debuglog type: <type 'bool'>1954dependencies:False1955dependencies type: <type 'bool'>1956disable_preboot:False1957disable_preboot type: <type 'bool'>1958env:False1959env type: <type 'bool'>1960error:p1961error type: <type 'str'>1962forcenoval:False1963forcenoval type: <type 'bool'>1964geometry:False1965geometry type: <type 'bool'>1966held:False1967held type: <type 'bool'>1968inputfile:False1969inputfile type: <type 'bool'>1970interactive:False1971interactive type: <type 'bool'>1972ion_kernel:default1973ion_kernel type: <type 'str'>1974ion_kerneloptions:False1975ion_kerneloptions type: <type 'bool'>1976jobname:False1977jobname type: <type 'bool'>1978kernel:default1979kernel type: <type 'str'>1980kerneloptions:False1981kerneloptions type: <type 'bool'>1982mode:False1983mode type: <type 'bool'>1984nodecount:101985nodecount type: <type 'str'>1986notify:False1987notify type: <type 'bool'>1988output:False1989output type: <type 'bool'>1990outputprefix:False1991outputprefix type: <type 'bool'>1992preemptable:False1993preemptable type: <type 'bool'>1994proccount:False1995proccount type: <type 'bool'>1996project:False1997project type: <type 'bool'>1998qsub_host:foo.bar1999qsub_host type: <type 'str'>2000queue:default2001queue type: <type 'str'>2002run_project:False2003run_project type: <type 'bool'>2004time:102005time type: <type 'str'>2006umask:False2007umask type: <type 'bool'>2008user_list:False2009user_list type: <type 'bool'>2010verbose:False2011verbose type: <type 'bool'>2012version:False2013version type: <type 'bool'>2014"""2015 stubout_file = "stub.out"2016 expected_results = ( 2017 0, # Expected return status 2018 cmdout, # Expected command output2019 stubout, # Expected stub functions output2020 cmderr, # Expected command error output 2021 ) 2022 testutils.save_testhook("")2023 results = testutils.run_cmd('qsub.py',args,stubout_file) 2024 result = testutils.validate_results(results,expected_results)2025 testutils.remove_testhook()2026 correct = 12027 assert result == correct, "Result:\n%s" % result2028# ---------------------------------------------------------------------------------2029def test_qsub_cwd_option_2():2030 """2031 qsub test run: cwd_option_22032 """2033 args = """--cwd /​tmp -t10 -n 10 -e p /​bin/​ls"""2034 cmdout = \2035"""12036"""2037 cmderr = ''2038 stubout = \2039"""2040GET_IMPLEMENTATION2041ADD_JOBS2042args:[]2043args type: <type 'list'>2044attrs:{}2045attrs type: <type 'dict'>2046command:/​bin/​ls2047command type: <type 'str'>2048cwd:/​tmp2049cwd type: <type 'str'>2050errorpath:/​tmp/​p2051errorpath type: <type 'str'>2052ion_kernel:default2053ion_kernel type: <type 'str'>2054jobid:*2055jobid type: <type 'str'>2056kernel:default2057kernel type: <type 'str'>2058mode:c12059mode type: <type 'str'>2060nodes:102061nodes type: <type 'int'>2062outputdir:/​tmp2063outputdir type: <type 'str'>2064path:/​tmp2065path type: <type 'str'>2066procs:5122067procs type: <type 'str'>2068queue:default2069queue type: <type 'str'>2070run_project:False2071run_project type: <type 'bool'>2072script_preboot:True2073script_preboot type: <type 'bool'>2074submithost:foo.bar2075submithost type: <type 'str'>2076tag:job2077tag type: <type 'str'>2078ttysession:None2079ttysession type: <type 'NoneType'>2080umask:182081umask type: <type 'int'>2082user:gooduser2083user type: <type 'str'>2084user_list:['gooduser']2085user_list type: <type 'list'>2086walltime:102087walltime type: <type 'str'>2088VALIDATE_JOB2089attrs:{}2090attrs type: <type 'dict'>2091cwd:/​tmp2092cwd type: <type 'str'>2093debug:False2094debug type: <type 'bool'>2095debuglog:False2096debuglog type: <type 'bool'>2097dependencies:False2098dependencies type: <type 'bool'>2099disable_preboot:False2100disable_preboot type: <type 'bool'>2101env:False2102env type: <type 'bool'>2103error:p2104error type: <type 'str'>2105forcenoval:False2106forcenoval type: <type 'bool'>2107geometry:False2108geometry type: <type 'bool'>2109held:False2110held type: <type 'bool'>2111inputfile:False2112inputfile type: <type 'bool'>2113interactive:False2114interactive type: <type 'bool'>2115ion_kernel:default2116ion_kernel type: <type 'str'>2117ion_kerneloptions:False2118ion_kerneloptions type: <type 'bool'>2119jobname:False2120jobname type: <type 'bool'>2121kernel:default2122kernel type: <type 'str'>2123kerneloptions:False2124kerneloptions type: <type 'bool'>2125mode:False2126mode type: <type 'bool'>2127nodecount:102128nodecount type: <type 'str'>2129notify:False2130notify type: <type 'bool'>2131output:False2132output type: <type 'bool'>2133outputprefix:False2134outputprefix type: <type 'bool'>2135preemptable:False2136preemptable type: <type 'bool'>2137proccount:False2138proccount type: <type 'bool'>2139project:False2140project type: <type 'bool'>2141qsub_host:foo.bar2142qsub_host type: <type 'str'>2143queue:default2144queue type: <type 'str'>2145run_project:False2146run_project type: <type 'bool'>2147time:102148time type: <type 'str'>2149umask:False2150umask type: <type 'bool'>2151user_list:False2152user_list type: <type 'bool'>2153verbose:False2154verbose type: <type 'bool'>2155version:False2156version type: <type 'bool'>2157"""2158 stubout_file = "stub.out"2159 expected_results = ( 2160 0, # Expected return status 2161 cmdout, # Expected command output2162 stubout, # Expected stub functions output2163 cmderr, # Expected command error output 2164 ) 2165 testutils.save_testhook("")2166 results = testutils.run_cmd('qsub.py',args,stubout_file) 2167 result = testutils.validate_results(results,expected_results)2168 testutils.remove_testhook()2169 correct = 12170 assert result == correct, "Result:\n%s" % result2171# ---------------------------------------------------------------------------------2172def test_qsub_cwd_option_3():2173 """2174 qsub test run: cwd_option_32175 """2176 args = """--cwd /​x -t10 -n 10 -e p /​bin/​ls"""2177 cmdout = ''2178 cmderr = \2179"""directory /​x/​p does not exist2180"""2181 stubout = ''2182 stubout_file = "stub.out"2183 expected_results = ( 2184 256, # Expected return status 2185 cmdout, # Expected command output2186 stubout, # Expected stub functions output2187 cmderr, # Expected command error output 2188 ) 2189 testutils.save_testhook("")2190 results = testutils.run_cmd('qsub.py',args,stubout_file) 2191 result = testutils.validate_results(results,expected_results)2192 testutils.remove_testhook()2193 correct = 12194 assert result == correct, "Result:\n%s" % result2195# ---------------------------------------------------------------------------------2196def test_qsub_cwd_option_4():2197 """2198 qsub test run: cwd_option_42199 """2200 args = """--cwd /​tmp/​ -t10 -n 10 -e p -o x /​bin/​ls"""2201 cmdout = \2202"""12203"""2204 cmderr = ''2205 stubout = \2206"""2207GET_IMPLEMENTATION2208ADD_JOBS2209args:[]2210args type: <type 'list'>2211attrs:{}2212attrs type: <type 'dict'>2213command:/​bin/​ls2214command type: <type 'str'>2215cwd:/​tmp/​2216cwd type: <type 'str'>2217errorpath:/​tmp/​/​p2218errorpath type: <type 'str'>2219ion_kernel:default2220ion_kernel type: <type 'str'>2221jobid:*2222jobid type: <type 'str'>2223kernel:default2224kernel type: <type 'str'>2225mode:c12226mode type: <type 'str'>2227nodes:102228nodes type: <type 'int'>2229outputdir:/​tmp/​2230outputdir type: <type 'str'>2231outputpath:/​tmp/​/​x2232outputpath type: <type 'str'>2233path:/​tmp2234path type: <type 'str'>2235procs:5122236procs type: <type 'str'>2237queue:default2238queue type: <type 'str'>2239run_project:False2240run_project type: <type 'bool'>2241script_preboot:True2242script_preboot type: <type 'bool'>2243submithost:foo.bar2244submithost type: <type 'str'>2245tag:job2246tag type: <type 'str'>2247ttysession:None2248ttysession type: <type 'NoneType'>2249umask:182250umask type: <type 'int'>2251user:gooduser2252user type: <type 'str'>2253user_list:['gooduser']2254user_list type: <type 'list'>2255walltime:102256walltime type: <type 'str'>2257VALIDATE_JOB2258attrs:{}2259attrs type: <type 'dict'>2260cwd:/​tmp/​2261cwd type: <type 'str'>2262debug:False2263debug type: <type 'bool'>2264debuglog:False2265debuglog type: <type 'bool'>2266dependencies:False2267dependencies type: <type 'bool'>2268disable_preboot:False2269disable_preboot type: <type 'bool'>2270env:False2271env type: <type 'bool'>2272error:p2273error type: <type 'str'>2274forcenoval:False2275forcenoval type: <type 'bool'>2276geometry:False2277geometry type: <type 'bool'>2278held:False2279held type: <type 'bool'>2280inputfile:False2281inputfile type: <type 'bool'>2282interactive:False2283interactive type: <type 'bool'>2284ion_kernel:default2285ion_kernel type: <type 'str'>2286ion_kerneloptions:False2287ion_kerneloptions type: <type 'bool'>2288jobname:False2289jobname type: <type 'bool'>2290kernel:default2291kernel type: <type 'str'>2292kerneloptions:False2293kerneloptions type: <type 'bool'>2294mode:False2295mode type: <type 'bool'>2296nodecount:102297nodecount type: <type 'str'>2298notify:False2299notify type: <type 'bool'>2300output:x2301output type: <type 'str'>2302outputprefix:False2303outputprefix type: <type 'bool'>2304preemptable:False2305preemptable type: <type 'bool'>2306proccount:False2307proccount type: <type 'bool'>2308project:False2309project type: <type 'bool'>2310qsub_host:foo.bar2311qsub_host type: <type 'str'>2312queue:default2313queue type: <type 'str'>2314run_project:False2315run_project type: <type 'bool'>2316time:102317time type: <type 'str'>2318umask:False2319umask type: <type 'bool'>2320user_list:False2321user_list type: <type 'bool'>2322verbose:False2323verbose type: <type 'bool'>2324version:False2325version type: <type 'bool'>2326"""2327 stubout_file = "stub.out"2328 expected_results = ( 2329 0, # Expected return status 2330 cmdout, # Expected command output2331 stubout, # Expected stub functions output2332 cmderr, # Expected command error output 2333 ) 2334 testutils.save_testhook("")2335 results = testutils.run_cmd('qsub.py',args,stubout_file) 2336 result = testutils.validate_results(results,expected_results)2337 testutils.remove_testhook()2338 correct = 12339 assert result == correct, "Result:\n%s" % result2340# ---------------------------------------------------------------------------------2341def test_qsub_cwd_option_5():2342 """2343 qsub test run: cwd_option_52344 """2345 args = """--cwd /​tmp -t10 -n 10 -e p -o x /​bin/​ls"""2346 cmdout = \2347"""12348"""2349 cmderr = ''2350 stubout = \2351"""2352GET_IMPLEMENTATION2353ADD_JOBS2354args:[]2355args type: <type 'list'>2356attrs:{}2357attrs type: <type 'dict'>2358command:/​bin/​ls2359command type: <type 'str'>2360cwd:/​tmp2361cwd type: <type 'str'>2362errorpath:/​tmp/​p2363errorpath type: <type 'str'>2364ion_kernel:default2365ion_kernel type: <type 'str'>2366jobid:*2367jobid type: <type 'str'>2368kernel:default2369kernel type: <type 'str'>2370mode:c12371mode type: <type 'str'>2372nodes:102373nodes type: <type 'int'>2374outputdir:/​tmp2375outputdir type: <type 'str'>2376outputpath:/​tmp/​x2377outputpath type: <type 'str'>2378path:/​tmp2379path type: <type 'str'>2380procs:5122381procs type: <type 'str'>2382queue:default2383queue type: <type 'str'>2384run_project:False2385run_project type: <type 'bool'>2386script_preboot:True2387script_preboot type: <type 'bool'>2388submithost:foo.bar2389submithost type: <type 'str'>2390tag:job2391tag type: <type 'str'>2392ttysession:None2393ttysession type: <type 'NoneType'>2394umask:182395umask type: <type 'int'>2396user:gooduser2397user type: <type 'str'>2398user_list:['gooduser']2399user_list type: <type 'list'>2400walltime:102401walltime type: <type 'str'>2402VALIDATE_JOB2403attrs:{}2404attrs type: <type 'dict'>2405cwd:/​tmp2406cwd type: <type 'str'>2407debug:False2408debug type: <type 'bool'>2409debuglog:False2410debuglog type: <type 'bool'>2411dependencies:False2412dependencies type: <type 'bool'>2413disable_preboot:False2414disable_preboot type: <type 'bool'>2415env:False2416env type: <type 'bool'>2417error:p2418error type: <type 'str'>2419forcenoval:False2420forcenoval type: <type 'bool'>2421geometry:False2422geometry type: <type 'bool'>2423held:False2424held type: <type 'bool'>2425inputfile:False2426inputfile type: <type 'bool'>2427interactive:False2428interactive type: <type 'bool'>2429ion_kernel:default2430ion_kernel type: <type 'str'>2431ion_kerneloptions:False2432ion_kerneloptions type: <type 'bool'>2433jobname:False2434jobname type: <type 'bool'>2435kernel:default2436kernel type: <type 'str'>2437kerneloptions:False2438kerneloptions type: <type 'bool'>2439mode:False2440mode type: <type 'bool'>2441nodecount:102442nodecount type: <type 'str'>2443notify:False2444notify type: <type 'bool'>2445output:x2446output type: <type 'str'>2447outputprefix:False2448outputprefix type: <type 'bool'>2449preemptable:False2450preemptable type: <type 'bool'>2451proccount:False2452proccount type: <type 'bool'>2453project:False2454project type: <type 'bool'>2455qsub_host:foo.bar2456qsub_host type: <type 'str'>2457queue:default2458queue type: <type 'str'>2459run_project:False2460run_project type: <type 'bool'>2461time:102462time type: <type 'str'>2463umask:False2464umask type: <type 'bool'>2465user_list:False2466user_list type: <type 'bool'>2467verbose:False2468verbose type: <type 'bool'>2469version:False2470version type: <type 'bool'>2471"""2472 stubout_file = "stub.out"2473 expected_results = ( 2474 0, # Expected return status 2475 cmdout, # Expected command output2476 stubout, # Expected stub functions output2477 cmderr, # Expected command error output 2478 ) 2479 testutils.save_testhook("")2480 results = testutils.run_cmd('qsub.py',args,stubout_file) 2481 result = testutils.validate_results(results,expected_results)2482 testutils.remove_testhook()2483 correct = 12484 assert result == correct, "Result:\n%s" % result2485# ---------------------------------------------------------------------------------2486def test_qsub_debuglog_option():2487 """2488 qsub test run: debuglog_option2489 """2490 args = """-t10 -n 10 -e p -o x --debuglog y /​bin/​ls"""2491 cmdout = \2492"""12493"""2494 cmderr = ''2495 stubout = \2496"""2497GET_IMPLEMENTATION2498ADD_JOBS2499args:[]2500args type: <type 'list'>2501attrs:{}2502attrs type: <type 'dict'>2503cobalt_log_file:/​tmp/​y2504cobalt_log_file type: <type 'str'>2505command:/​bin/​ls2506command type: <type 'str'>2507cwd:/​tmp2508cwd type: <type 'str'>2509errorpath:/​tmp/​p2510errorpath type: <type 'str'>2511ion_kernel:default2512ion_kernel type: <type 'str'>2513jobid:*2514jobid type: <type 'str'>2515kernel:default2516kernel type: <type 'str'>2517mode:c12518mode type: <type 'str'>2519nodes:102520nodes type: <type 'int'>2521outputdir:/​tmp2522outputdir type: <type 'str'>2523outputpath:/​tmp/​x2524outputpath type: <type 'str'>2525path:/​tmp2526path type: <type 'str'>2527procs:5122528procs type: <type 'str'>2529queue:default2530queue type: <type 'str'>2531run_project:False2532run_project type: <type 'bool'>2533script_preboot:True2534script_preboot type: <type 'bool'>2535submithost:foo.bar2536submithost type: <type 'str'>2537tag:job2538tag type: <type 'str'>2539ttysession:None2540ttysession type: <type 'NoneType'>2541umask:182542umask type: <type 'int'>2543user:gooduser2544user type: <type 'str'>2545user_list:['gooduser']2546user_list type: <type 'list'>2547walltime:102548walltime type: <type 'str'>2549VALIDATE_JOB2550attrs:{}2551attrs type: <type 'dict'>2552cwd:/​tmp2553cwd type: <type 'str'>2554debug:False2555debug type: <type 'bool'>2556debuglog:y2557debuglog type: <type 'str'>2558dependencies:False2559dependencies type: <type 'bool'>2560disable_preboot:False2561disable_preboot type: <type 'bool'>2562env:False2563env type: <type 'bool'>2564error:p2565error type: <type 'str'>2566forcenoval:False2567forcenoval type: <type 'bool'>2568geometry:False2569geometry type: <type 'bool'>2570held:False2571held type: <type 'bool'>2572inputfile:False2573inputfile type: <type 'bool'>2574interactive:False2575interactive type: <type 'bool'>2576ion_kernel:default2577ion_kernel type: <type 'str'>2578ion_kerneloptions:False2579ion_kerneloptions type: <type 'bool'>2580jobname:False2581jobname type: <type 'bool'>2582kernel:default2583kernel type: <type 'str'>2584kerneloptions:False2585kerneloptions type: <type 'bool'>2586mode:False2587mode type: <type 'bool'>2588nodecount:102589nodecount type: <type 'str'>2590notify:False2591notify type: <type 'bool'>2592output:x2593output type: <type 'str'>2594outputprefix:False2595outputprefix type: <type 'bool'>2596preemptable:False2597preemptable type: <type 'bool'>2598proccount:False2599proccount type: <type 'bool'>2600project:False2601project type: <type 'bool'>2602qsub_host:foo.bar2603qsub_host type: <type 'str'>2604queue:default2605queue type: <type 'str'>2606run_project:False2607run_project type: <type 'bool'>2608time:102609time type: <type 'str'>2610umask:False2611umask type: <type 'bool'>2612user_list:False2613user_list type: <type 'bool'>2614verbose:False2615verbose type: <type 'bool'>2616version:False2617version type: <type 'bool'>2618"""2619 stubout_file = "stub.out"2620 expected_results = ( 2621 0, # Expected return status 2622 cmdout, # Expected command output2623 stubout, # Expected stub functions output2624 cmderr, # Expected command error output 2625 ) 2626 testutils.save_testhook("")2627 results = testutils.run_cmd('qsub.py',args,stubout_file) 2628 result = testutils.validate_results(results,expected_results)2629 testutils.remove_testhook()2630 correct = 12631 assert result == correct, "Result:\n%s" % result2632# ---------------------------------------------------------------------------------2633def test_qsub_inputfile_option_1():2634 """2635 qsub test run: inputfile_option_12636 """2637 args = """-i none -t10 -n 10 /​bin/​ls"""2638 cmdout = ''2639 cmderr = \2640"""file /​tmp/​none not found, or is not a file2641"""2642 stubout = ''2643 stubout_file = "stub.out"2644 expected_results = ( 2645 256, # Expected return status 2646 cmdout, # Expected command output2647 stubout, # Expected stub functions output2648 cmderr, # Expected command error output 2649 ) 2650 testutils.save_testhook("")2651 results = testutils.run_cmd('qsub.py',args,stubout_file) 2652 result = testutils.validate_results(results,expected_results)2653 testutils.remove_testhook()2654 correct = 12655 assert result == correct, "Result:\n%s" % result2656# ---------------------------------------------------------------------------------2657def test_qsub_inputfile_option_2():2658 """2659 qsub test run: inputfile_option_22660 """2661 args = """-i y -t10 -n 10 /​bin/​ls"""2662 cmdout = \2663"""12664"""2665 cmderr = ''2666 stubout = \2667"""2668GET_IMPLEMENTATION2669ADD_JOBS2670args:[]2671args type: <type 'list'>2672attrs:{}2673attrs type: <type 'dict'>2674command:/​bin/​ls2675command type: <type 'str'>2676cwd:/​tmp2677cwd type: <type 'str'>2678inputfile:/​tmp/​y2679inputfile type: <type 'str'>2680ion_kernel:default2681ion_kernel type: <type 'str'>2682jobid:*2683jobid type: <type 'str'>2684kernel:default2685kernel type: <type 'str'>2686mode:c12687mode type: <type 'str'>2688nodes:102689nodes type: <type 'int'>2690outputdir:/​tmp2691outputdir type: <type 'str'>2692path:/​tmp2693path type: <type 'str'>2694procs:5122695procs type: <type 'str'>2696queue:default2697queue type: <type 'str'>2698run_project:False2699run_project type: <type 'bool'>2700script_preboot:True2701script_preboot type: <type 'bool'>2702submithost:foo.bar2703submithost type: <type 'str'>2704tag:job2705tag type: <type 'str'>2706ttysession:None2707ttysession type: <type 'NoneType'>2708umask:182709umask type: <type 'int'>2710user:gooduser2711user type: <type 'str'>2712user_list:['gooduser']2713user_list type: <type 'list'>2714walltime:102715walltime type: <type 'str'>2716VALIDATE_JOB2717attrs:{}2718attrs type: <type 'dict'>2719cwd:/​tmp2720cwd type: <type 'str'>2721debug:False2722debug type: <type 'bool'>2723debuglog:False2724debuglog type: <type 'bool'>2725dependencies:False2726dependencies type: <type 'bool'>2727disable_preboot:False2728disable_preboot type: <type 'bool'>2729env:False2730env type: <type 'bool'>2731error:False2732error type: <type 'bool'>2733forcenoval:False2734forcenoval type: <type 'bool'>2735geometry:False2736geometry type: <type 'bool'>2737held:False2738held type: <type 'bool'>2739inputfile:y2740inputfile type: <type 'str'>2741interactive:False2742interactive type: <type 'bool'>2743ion_kernel:default2744ion_kernel type: <type 'str'>2745ion_kerneloptions:False2746ion_kerneloptions type: <type 'bool'>2747jobname:False2748jobname type: <type 'bool'>2749kernel:default2750kernel type: <type 'str'>2751kerneloptions:False2752kerneloptions type: <type 'bool'>2753mode:False2754mode type: <type 'bool'>2755nodecount:102756nodecount type: <type 'str'>2757notify:False2758notify type: <type 'bool'>2759output:False2760output type: <type 'bool'>2761outputprefix:False2762outputprefix type: <type 'bool'>2763preemptable:False2764preemptable type: <type 'bool'>2765proccount:False2766proccount type: <type 'bool'>2767project:False2768project type: <type 'bool'>2769qsub_host:foo.bar2770qsub_host type: <type 'str'>2771queue:default2772queue type: <type 'str'>2773run_project:False2774run_project type: <type 'bool'>2775time:102776time type: <type 'str'>2777umask:False2778umask type: <type 'bool'>2779user_list:False2780user_list type: <type 'bool'>2781verbose:False2782verbose type: <type 'bool'>2783version:False2784version type: <type 'bool'>2785"""2786 stubout_file = "stub.out"2787 expected_results = ( 2788 0, # Expected return status 2789 cmdout, # Expected command output2790 stubout, # Expected stub functions output2791 cmderr, # Expected command error output 2792 ) 2793 testutils.save_testhook("")2794 results = testutils.run_cmd('qsub.py',args,stubout_file) 2795 result = testutils.validate_results(results,expected_results)2796 testutils.remove_testhook()2797 correct = 12798 assert result == correct, "Result:\n%s" % result2799# ---------------------------------------------------------------------------------2800def test_qsub_email_option():2801 """2802 qsub test run: email_option2803 """2804 args = """-M g -t10 -n10 /​bin/​ls"""2805 cmdout = \2806"""12807"""2808 cmderr = ''2809 stubout = \2810"""2811GET_IMPLEMENTATION2812ADD_JOBS2813args:[]2814args type: <type 'list'>2815attrs:{}2816attrs type: <type 'dict'>2817command:/​bin/​ls2818command type: <type 'str'>2819cwd:/​tmp2820cwd type: <type 'str'>2821ion_kernel:default2822ion_kernel type: <type 'str'>2823jobid:*2824jobid type: <type 'str'>2825kernel:default2826kernel type: <type 'str'>2827mode:c12828mode type: <type 'str'>2829nodes:102830nodes type: <type 'int'>2831notify:g2832notify type: <type 'str'>2833outputdir:/​tmp2834outputdir type: <type 'str'>2835path:/​tmp2836path type: <type 'str'>2837procs:5122838procs type: <type 'str'>2839queue:default2840queue type: <type 'str'>2841run_project:False2842run_project type: <type 'bool'>2843script_preboot:True2844script_preboot type: <type 'bool'>2845submithost:foo.bar2846submithost type: <type 'str'>2847tag:job2848tag type: <type 'str'>2849ttysession:None2850ttysession type: <type 'NoneType'>2851umask:182852umask type: <type 'int'>2853user:gooduser2854user type: <type 'str'>2855user_list:['gooduser']2856user_list type: <type 'list'>2857walltime:102858walltime type: <type 'str'>2859VALIDATE_JOB2860attrs:{}2861attrs type: <type 'dict'>2862cwd:/​tmp2863cwd type: <type 'str'>2864debug:False2865debug type: <type 'bool'>2866debuglog:False2867debuglog type: <type 'bool'>2868dependencies:False2869dependencies type: <type 'bool'>2870disable_preboot:False2871disable_preboot type: <type 'bool'>2872env:False2873env type: <type 'bool'>2874error:False2875error type: <type 'bool'>2876forcenoval:False2877forcenoval type: <type 'bool'>2878geometry:False2879geometry type: <type 'bool'>2880held:False2881held type: <type 'bool'>2882inputfile:False2883inputfile type: <type 'bool'>2884interactive:False2885interactive type: <type 'bool'>2886ion_kernel:default2887ion_kernel type: <type 'str'>2888ion_kerneloptions:False2889ion_kerneloptions type: <type 'bool'>2890jobname:False2891jobname type: <type 'bool'>2892kernel:default2893kernel type: <type 'str'>2894kerneloptions:False2895kerneloptions type: <type 'bool'>2896mode:False2897mode type: <type 'bool'>2898nodecount:102899nodecount type: <type 'str'>2900notify:g2901notify type: <type 'str'>2902output:False2903output type: <type 'bool'>2904outputprefix:False2905outputprefix type: <type 'bool'>2906preemptable:False2907preemptable type: <type 'bool'>2908proccount:False2909proccount type: <type 'bool'>2910project:False2911project type: <type 'bool'>2912qsub_host:foo.bar2913qsub_host type: <type 'str'>2914queue:default2915queue type: <type 'str'>2916run_project:False2917run_project type: <type 'bool'>2918time:102919time type: <type 'str'>2920umask:False2921umask type: <type 'bool'>2922user_list:False2923user_list type: <type 'bool'>2924verbose:False2925verbose type: <type 'bool'>2926version:False2927version type: <type 'bool'>2928"""2929 stubout_file = "stub.out"2930 expected_results = ( 2931 0, # Expected return status 2932 cmdout, # Expected command output2933 stubout, # Expected stub functions output2934 cmderr, # Expected command error output 2935 ) 2936 testutils.save_testhook("")2937 results = testutils.run_cmd('qsub.py',args,stubout_file) 2938 result = testutils.validate_results(results,expected_results)2939 testutils.remove_testhook()2940 correct = 12941 assert result == correct, "Result:\n%s" % result2942# ---------------------------------------------------------------------------------2943def test_qsub_outputprefix():2944 """2945 qsub test run: outputprefix2946 """2947 args = """-O tmp -t10 -n10 /​bin/​ls"""2948 cmdout = \2949"""12950"""2951 cmderr = ''2952 stubout = \2953"""2954GET_IMPLEMENTATION2955ADD_JOBS2956args:[]2957args type: <type 'list'>2958attrs:{}2959attrs type: <type 'dict'>2960cobalt_log_file:/​tmp/​tmp.cobaltlog2961cobalt_log_file type: <type 'str'>2962command:/​bin/​ls2963command type: <type 'str'>2964cwd:/​tmp2965cwd type: <type 'str'>2966errorpath:/​tmp/​tmp.error2967errorpath type: <type 'str'>2968ion_kernel:default2969ion_kernel type: <type 'str'>2970jobid:*2971jobid type: <type 'str'>2972kernel:default2973kernel type: <type 'str'>2974mode:c12975mode type: <type 'str'>2976nodes:102977nodes type: <type 'int'>2978outputdir:/​tmp2979outputdir type: <type 'str'>2980outputpath:/​tmp/​tmp.output2981outputpath type: <type 'str'>2982path:/​tmp2983path type: <type 'str'>2984procs:5122985procs type: <type 'str'>2986queue:default2987queue type: <type 'str'>2988run_project:False2989run_project type: <type 'bool'>2990script_preboot:True2991script_preboot type: <type 'bool'>2992submithost:foo.bar2993submithost type: <type 'str'>2994tag:job2995tag type: <type 'str'>2996ttysession:None2997ttysession type: <type 'NoneType'>2998umask:182999umask type: <type 'int'>3000user:gooduser3001user type: <type 'str'>3002user_list:['gooduser']3003user_list type: <type 'list'>3004walltime:103005walltime type: <type 'str'>3006VALIDATE_JOB3007attrs:{}3008attrs type: <type 'dict'>3009cwd:/​tmp3010cwd type: <type 'str'>3011debug:False3012debug type: <type 'bool'>3013debuglog:False3014debuglog type: <type 'bool'>3015dependencies:False3016dependencies type: <type 'bool'>3017disable_preboot:False3018disable_preboot type: <type 'bool'>3019env:False3020env type: <type 'bool'>3021error:False3022error type: <type 'bool'>3023forcenoval:False3024forcenoval type: <type 'bool'>3025geometry:False3026geometry type: <type 'bool'>3027held:False3028held type: <type 'bool'>3029inputfile:False3030inputfile type: <type 'bool'>3031interactive:False3032interactive type: <type 'bool'>3033ion_kernel:default3034ion_kernel type: <type 'str'>3035ion_kerneloptions:False3036ion_kerneloptions type: <type 'bool'>3037jobname:False3038jobname type: <type 'bool'>3039kernel:default3040kernel type: <type 'str'>3041kerneloptions:False3042kerneloptions type: <type 'bool'>3043mode:False3044mode type: <type 'bool'>3045nodecount:103046nodecount type: <type 'str'>3047notify:False3048notify type: <type 'bool'>3049output:False3050output type: <type 'bool'>3051outputprefix:tmp3052outputprefix type: <type 'str'>3053preemptable:False3054preemptable type: <type 'bool'>3055proccount:False3056proccount type: <type 'bool'>3057project:False3058project type: <type 'bool'>3059qsub_host:foo.bar3060qsub_host type: <type 'str'>3061queue:default3062queue type: <type 'str'>3063run_project:False3064run_project type: <type 'bool'>3065time:103066time type: <type 'str'>3067umask:False3068umask type: <type 'bool'>3069user_list:False3070user_list type: <type 'bool'>3071verbose:False3072verbose type: <type 'bool'>3073version:False3074version type: <type 'bool'>3075"""3076 stubout_file = "stub.out"3077 expected_results = ( 3078 0, # Expected return status 3079 cmdout, # Expected command output3080 stubout, # Expected stub functions output3081 cmderr, # Expected command error output 3082 ) 3083 testutils.save_testhook("")3084 results = testutils.run_cmd('qsub.py',args,stubout_file) 3085 result = testutils.validate_results(results,expected_results)3086 testutils.remove_testhook()3087 correct = 13088 assert result == correct, "Result:\n%s" % result3089# ---------------------------------------------------------------------------------3090def test_qsub_invalid_user():3091 """3092 qsub test run: invalid_user3093 """3094 args = """--run_users naughtyuser -t10 -n10 /​bin/​ls"""3095 cmdout = ''3096 cmderr = \3097"""user naughtyuser does not exist.3098"""3099 stubout = ''3100 stubout_file = "stub.out"3101 expected_results = ( 3102 256, # Expected return status 3103 cmdout, # Expected command output3104 stubout, # Expected stub functions output3105 cmderr, # Expected command error output 3106 ) 3107 testutils.save_testhook("")3108 results = testutils.run_cmd('qsub.py',args,stubout_file) 3109 result = testutils.validate_results(results,expected_results)3110 testutils.remove_testhook()3111 correct = 13112 assert result == correct, "Result:\n%s" % result3113# ---------------------------------------------------------------------------------3114def test_qsub_mode_option_1():3115 """3116 qsub test run: mode_option_13117 """3118 args = """-t10 -n512 --proccount 1023 --mode vn /​bin/​ls"""3119 cmdout = \3120"""13121"""3122 cmderr = ''3123 stubout = \3124"""3125GET_IMPLEMENTATION3126ADD_JOBS3127args:[]3128args type: <type 'list'>3129attrs:{}3130attrs type: <type 'dict'>3131command:/​bin/​ls3132command type: <type 'str'>3133cwd:/​tmp3134cwd type: <type 'str'>3135ion_kernel:default3136ion_kernel type: <type 'str'>3137jobid:*3138jobid type: <type 'str'>3139kernel:default3140kernel type: <type 'str'>3141mode:vn3142mode type: <type 'str'>3143nodes:5123144nodes type: <type 'int'>3145outputdir:/​tmp3146outputdir type: <type 'str'>3147path:/​tmp3148path type: <type 'str'>3149procs:10233150procs type: <type 'str'>3151queue:default3152queue type: <type 'str'>3153run_project:False3154run_project type: <type 'bool'>3155script_preboot:True3156script_preboot type: <type 'bool'>3157submithost:foo.bar3158submithost type: <type 'str'>3159tag:job3160tag type: <type 'str'>3161ttysession:None3162ttysession type: <type 'NoneType'>3163umask:183164umask type: <type 'int'>3165user:gooduser3166user type: <type 'str'>3167user_list:['gooduser']3168user_list type: <type 'list'>3169walltime:103170walltime type: <type 'str'>3171VALIDATE_JOB3172attrs:{}3173attrs type: <type 'dict'>3174cwd:/​tmp3175cwd type: <type 'str'>3176debug:False3177debug type: <type 'bool'>3178debuglog:False3179debuglog type: <type 'bool'>3180dependencies:False3181dependencies type: <type 'bool'>3182disable_preboot:False3183disable_preboot type: <type 'bool'>3184env:False3185env type: <type 'bool'>3186error:False3187error type: <type 'bool'>3188forcenoval:False3189forcenoval type: <type 'bool'>3190geometry:False3191geometry type: <type 'bool'>3192held:False3193held type: <type 'bool'>3194inputfile:False3195inputfile type: <type 'bool'>3196interactive:False3197interactive type: <type 'bool'>3198ion_kernel:default3199ion_kernel type: <type 'str'>3200ion_kerneloptions:False3201ion_kerneloptions type: <type 'bool'>3202jobname:False3203jobname type: <type 'bool'>3204kernel:default3205kernel type: <type 'str'>3206kerneloptions:False3207kerneloptions type: <type 'bool'>3208mode:vn3209mode type: <type 'str'>3210nodecount:5123211nodecount type: <type 'str'>3212notify:False3213notify type: <type 'bool'>3214output:False3215output type: <type 'bool'>3216outputprefix:False3217outputprefix type: <type 'bool'>3218preemptable:False3219preemptable type: <type 'bool'>3220proccount:10233221proccount type: <type 'str'>3222project:False3223project type: <type 'bool'>3224qsub_host:foo.bar3225qsub_host type: <type 'str'>3226queue:default3227queue type: <type 'str'>3228run_project:False3229run_project type: <type 'bool'>3230time:103231time type: <type 'str'>3232umask:False3233umask type: <type 'bool'>3234user_list:False3235user_list type: <type 'bool'>3236verbose:False3237verbose type: <type 'bool'>3238version:False3239version type: <type 'bool'>3240"""3241 stubout_file = "stub.out"3242 expected_results = ( 3243 0, # Expected return status 3244 cmdout, # Expected command output3245 stubout, # Expected stub functions output3246 cmderr, # Expected command error output 3247 ) 3248 testutils.save_testhook("")3249 results = testutils.run_cmd('qsub.py',args,stubout_file) 3250 result = testutils.validate_results(results,expected_results)3251 testutils.remove_testhook()3252 correct = 13253 assert result == correct, "Result:\n%s" % result3254# ---------------------------------------------------------------------------------3255def test_qsub_mode_option_2():3256 """3257 qsub test run: mode_option_23258 """3259 args = """-t10 -n512 --proccount 1023 --mode vn /​bin/​ls"""3260 cmdout = \3261"""13262"""3263 cmderr = ''3264 stubout = \3265"""3266GET_IMPLEMENTATION3267ADD_JOBS3268args:[]3269args type: <type 'list'>3270attrs:{}3271attrs type: <type 'dict'>3272command:/​bin/​ls3273command type: <type 'str'>3274cwd:/​tmp3275cwd type: <type 'str'>3276ion_kernel:default3277ion_kernel type: <type 'str'>3278jobid:*3279jobid type: <type 'str'>3280kernel:default3281kernel type: <type 'str'>3282mode:vn3283mode type: <type 'str'>3284nodes:5123285nodes type: <type 'int'>3286outputdir:/​tmp3287outputdir type: <type 'str'>3288path:/​tmp3289path type: <type 'str'>3290procs:10233291procs type: <type 'str'>3292queue:default3293queue type: <type 'str'>3294run_project:False3295run_project type: <type 'bool'>3296script_preboot:True3297script_preboot type: <type 'bool'>3298submithost:foo.bar3299submithost type: <type 'str'>3300tag:job3301tag type: <type 'str'>3302ttysession:None3303ttysession type: <type 'NoneType'>3304umask:183305umask type: <type 'int'>3306user:gooduser3307user type: <type 'str'>3308user_list:['gooduser']3309user_list type: <type 'list'>3310walltime:103311walltime type: <type 'str'>3312VALIDATE_JOB3313attrs:{}3314attrs type: <type 'dict'>3315cwd:/​tmp3316cwd type: <type 'str'>3317debug:False3318debug type: <type 'bool'>3319debuglog:False3320debuglog type: <type 'bool'>3321dependencies:False3322dependencies type: <type 'bool'>3323disable_preboot:False3324disable_preboot type: <type 'bool'>3325env:False3326env type: <type 'bool'>3327error:False3328error type: <type 'bool'>3329forcenoval:False3330forcenoval type: <type 'bool'>3331geometry:False3332geometry type: <type 'bool'>3333held:False3334held type: <type 'bool'>3335inputfile:False3336inputfile type: <type 'bool'>3337interactive:False3338interactive type: <type 'bool'>3339ion_kernel:default3340ion_kernel type: <type 'str'>3341ion_kerneloptions:False3342ion_kerneloptions type: <type 'bool'>3343jobname:False3344jobname type: <type 'bool'>3345kernel:default3346kernel type: <type 'str'>3347kerneloptions:False3348kerneloptions type: <type 'bool'>3349mode:vn3350mode type: <type 'str'>3351nodecount:5123352nodecount type: <type 'str'>3353notify:False3354notify type: <type 'bool'>3355output:False3356output type: <type 'bool'>3357outputprefix:False3358outputprefix type: <type 'bool'>3359preemptable:False3360preemptable type: <type 'bool'>3361proccount:10233362proccount type: <type 'str'>3363project:False3364project type: <type 'bool'>3365qsub_host:foo.bar3366qsub_host type: <type 'str'>3367queue:default3368queue type: <type 'str'>3369run_project:False3370run_project type: <type 'bool'>3371time:103372time type: <type 'str'>3373umask:False3374umask type: <type 'bool'>3375user_list:False3376user_list type: <type 'bool'>3377verbose:False3378verbose type: <type 'bool'>3379version:False3380version type: <type 'bool'>3381"""3382 stubout_file = "stub.out"3383 expected_results = ( 3384 0, # Expected return status 3385 cmdout, # Expected command output3386 stubout, # Expected stub functions output3387 cmderr, # Expected command error output 3388 ) 3389 testutils.save_testhook("")3390 results = testutils.run_cmd('qsub.py',args,stubout_file) 3391 result = testutils.validate_results(results,expected_results)3392 testutils.remove_testhook()3393 correct = 13394 assert result == correct, "Result:\n%s" % result3395# ---------------------------------------------------------------------------------3396def test_qsub_mode_option_3():3397 """3398 qsub test run: mode_option_33399 """3400 args = """--mode script -t50 -n10 --geometry 40x40x50x50 /​bin/​ls"""3401 cmdout = \3402"""13403"""3404 cmderr = ''3405 stubout = \3406"""3407GET_IMPLEMENTATION3408ADD_JOBS3409args:[]3410args type: <type 'list'>3411attrs:{}3412attrs type: <type 'dict'>3413command:/​bin/​ls3414command type: <type 'str'>3415cwd:/​tmp3416cwd type: <type 'str'>3417geometry:[40, 40, 50, 50, 2]3418geometry type: <type 'list'>3419ion_kernel:default3420ion_kernel type: <type 'str'>3421jobid:*3422jobid type: <type 'str'>3423kernel:default3424kernel type: <type 'str'>3425mode:script3426mode type: <type 'str'>3427nodes:103428nodes type: <type 'int'>3429outputdir:/​tmp3430outputdir type: <type 'str'>3431path:/​tmp3432path type: <type 'str'>3433procs:5123434procs type: <type 'str'>3435queue:default3436queue type: <type 'str'>3437run_project:False3438run_project type: <type 'bool'>3439script_preboot:True3440script_preboot type: <type 'bool'>3441submithost:foo.bar3442submithost type: <type 'str'>3443tag:job3444tag type: <type 'str'>3445ttysession:None3446ttysession type: <type 'NoneType'>3447umask:183448umask type: <type 'int'>3449user:gooduser3450user type: <type 'str'>3451user_list:['gooduser']3452user_list type: <type 'list'>3453walltime:503454walltime type: <type 'str'>3455VALIDATE_JOB3456attrs:{}3457attrs type: <type 'dict'>3458cwd:/​tmp3459cwd type: <type 'str'>3460debug:False3461debug type: <type 'bool'>3462debuglog:False3463debuglog type: <type 'bool'>3464dependencies:False3465dependencies type: <type 'bool'>3466disable_preboot:False3467disable_preboot type: <type 'bool'>3468env:False3469env type: <type 'bool'>3470error:False3471error type: <type 'bool'>3472forcenoval:False3473forcenoval type: <type 'bool'>3474geometry:40x40x50x503475geometry type: <type 'str'>3476held:False3477held type: <type 'bool'>3478inputfile:False3479inputfile type: <type 'bool'>3480interactive:False3481interactive type: <type 'bool'>3482ion_kernel:default3483ion_kernel type: <type 'str'>3484ion_kerneloptions:False3485ion_kerneloptions type: <type 'bool'>3486jobname:False3487jobname type: <type 'bool'>3488kernel:default3489kernel type: <type 'str'>3490kerneloptions:False3491kerneloptions type: <type 'bool'>3492mode:script3493mode type: <type 'str'>3494nodecount:103495nodecount type: <type 'str'>3496notify:False3497notify type: <type 'bool'>3498output:False3499output type: <type 'bool'>3500outputprefix:False3501outputprefix type: <type 'bool'>3502preemptable:False3503preemptable type: <type 'bool'>3504proccount:False3505proccount type: <type 'bool'>3506project:False3507project type: <type 'bool'>3508qsub_host:foo.bar3509qsub_host type: <type 'str'>3510queue:default3511queue type: <type 'str'>3512run_project:False3513run_project type: <type 'bool'>3514time:503515time type: <type 'str'>3516umask:False3517umask type: <type 'bool'>3518user_list:False3519user_list type: <type 'bool'>3520verbose:False3521verbose type: <type 'bool'>3522version:False3523version type: <type 'bool'>3524"""3525 stubout_file = "stub.out"3526 expected_results = ( 3527 0, # Expected return status 3528 cmdout, # Expected command output3529 stubout, # Expected stub functions output3530 cmderr, # Expected command error output 3531 ) 3532 testutils.save_testhook("")3533 results = testutils.run_cmd('qsub.py',args,stubout_file) 3534 result = testutils.validate_results(results,expected_results)3535 testutils.remove_testhook()3536 correct = 13537 assert result == correct, "Result:\n%s" % result3538# ---------------------------------------------------------------------------------3539def test_qsub_mode_option_4():3540 """3541 qsub test run: mode_option_43542 """3543 args = """-A Acceptance -q testing -n 49152 -t 60 --mode script /​bin/​ls"""3544 cmdout = ''3545 cmderr = \3546"""node count out of realistic range3547"""3548 stubout = ''3549 stubout_file = "stub.out"3550 expected_results = ( 3551 256, # Expected return status 3552 cmdout, # Expected command output3553 stubout, # Expected stub functions output3554 cmderr, # Expected command error output 3555 ) 3556 testutils.save_testhook("")3557 results = testutils.run_cmd('qsub.py',args,stubout_file) 3558 result = testutils.validate_results(results,expected_results)3559 testutils.remove_testhook()3560 correct = 13561 assert result == correct, "Result:\n%s" % result3562# ---------------------------------------------------------------------------------3563def test_qsub_preboot_option():3564 """3565 qsub test run: preboot_option3566 """3567 args = """--disable_preboot -t10 -n512 --proccount 1023 --mode vn /​bin/​ls"""3568 cmdout = \3569"""13570"""3571 cmderr = ''3572 stubout = \3573"""3574GET_IMPLEMENTATION3575ADD_JOBS3576args:[]3577args type: <type 'list'>3578attrs:{}3579attrs type: <type 'dict'>3580command:/​bin/​ls3581command type: <type 'str'>3582cwd:/​tmp3583cwd type: <type 'str'>3584ion_kernel:default3585ion_kernel type: <type 'str'>3586jobid:*3587jobid type: <type 'str'>3588kernel:default3589kernel type: <type 'str'>3590mode:vn3591mode type: <type 'str'>3592nodes:5123593nodes type: <type 'int'>3594outputdir:/​tmp3595outputdir type: <type 'str'>3596path:/​tmp3597path type: <type 'str'>3598procs:10233599procs type: <type 'str'>3600queue:default3601queue type: <type 'str'>3602run_project:False3603run_project type: <type 'bool'>3604script_preboot:False3605script_preboot type: <type 'bool'>3606submithost:foo.bar3607submithost type: <type 'str'>3608tag:job3609tag type: <type 'str'>3610ttysession:None3611ttysession type: <type 'NoneType'>3612umask:183613umask type: <type 'int'>3614user:gooduser3615user type: <type 'str'>3616user_list:['gooduser']3617user_list type: <type 'list'>3618walltime:103619walltime type: <type 'str'>3620VALIDATE_JOB3621attrs:{}3622attrs type: <type 'dict'>3623cwd:/​tmp3624cwd type: <type 'str'>3625debug:False3626debug type: <type 'bool'>3627debuglog:False3628debuglog type: <type 'bool'>3629dependencies:False3630dependencies type: <type 'bool'>3631disable_preboot:True3632disable_preboot type: <type 'bool'>3633env:False3634env type: <type 'bool'>3635error:False3636error type: <type 'bool'>3637forcenoval:False3638forcenoval type: <type 'bool'>3639geometry:False3640geometry type: <type 'bool'>3641held:False3642held type: <type 'bool'>3643inputfile:False3644inputfile type: <type 'bool'>3645interactive:False3646interactive type: <type 'bool'>3647ion_kernel:default3648ion_kernel type: <type 'str'>3649ion_kerneloptions:False3650ion_kerneloptions type: <type 'bool'>3651jobname:False3652jobname type: <type 'bool'>3653kernel:default3654kernel type: <type 'str'>3655kerneloptions:False3656kerneloptions type: <type 'bool'>3657mode:vn3658mode type: <type 'str'>3659nodecount:5123660nodecount type: <type 'str'>3661notify:False3662notify type: <type 'bool'>3663output:False3664output type: <type 'bool'>3665outputprefix:False3666outputprefix type: <type 'bool'>3667preemptable:False3668preemptable type: <type 'bool'>3669proccount:10233670proccount type: <type 'str'>3671project:False3672project type: <type 'bool'>3673qsub_host:foo.bar3674qsub_host type: <type 'str'>3675queue:default3676queue type: <type 'str'>3677run_project:False3678run_project type: <type 'bool'>3679time:103680time type: <type 'str'>3681umask:False3682umask type: <type 'bool'>3683user_list:False3684user_list type: <type 'bool'>3685verbose:False3686verbose type: <type 'bool'>3687version:False3688version type: <type 'bool'>3689"""3690 stubout_file = "stub.out"3691 expected_results = ( 3692 0, # Expected return status 3693 cmdout, # Expected command output3694 stubout, # Expected stub functions output3695 cmderr, # Expected command error output 3696 ) 3697 testutils.save_testhook("")3698 results = testutils.run_cmd('qsub.py',args,stubout_file) 3699 result = testutils.validate_results(results,expected_results)3700 testutils.remove_testhook()3701 correct = 13702 assert result == correct, "Result:\n%s" % result3703# ---------------------------------------------------------------------------------3704def test_qsub_env_option_1():3705 """3706 qsub test run: env_option_13707 """3708 args = """--env var1=val1,var2=val2 -t50 -n10 /​bin/​ls"""3709 cmdout = \3710"""13711"""3712 cmderr = ''3713 stubout = \3714"""3715GET_IMPLEMENTATION3716ADD_JOBS3717args:[]3718args type: <type 'list'>3719attrs:{}3720attrs type: <type 'dict'>3721command:/​bin/​ls3722command type: <type 'str'>3723cwd:/​tmp3724cwd type: <type 'str'>3725envs:{'var1': 'val1,var2=val2'}3726envs type: <type 'dict'>3727ion_kernel:default3728ion_kernel type: <type 'str'>3729jobid:*3730jobid type: <type 'str'>3731kernel:default3732kernel type: <type 'str'>3733mode:c13734mode type: <type 'str'>3735nodes:103736nodes type: <type 'int'>3737outputdir:/​tmp3738outputdir type: <type 'str'>3739path:/​tmp3740path type: <type 'str'>3741procs:5123742procs type: <type 'str'>3743queue:default3744queue type: <type 'str'>3745run_project:False3746run_project type: <type 'bool'>3747script_preboot:True3748script_preboot type: <type 'bool'>3749submithost:foo.bar3750submithost type: <type 'str'>3751tag:job3752tag type: <type 'str'>3753ttysession:None3754ttysession type: <type 'NoneType'>3755umask:183756umask type: <type 'int'>3757user:gooduser3758user type: <type 'str'>3759user_list:['gooduser']3760user_list type: <type 'list'>3761walltime:503762walltime type: <type 'str'>3763VALIDATE_JOB3764attrs:{}3765attrs type: <type 'dict'>3766cwd:/​tmp3767cwd type: <type 'str'>3768debug:False3769debug type: <type 'bool'>3770debuglog:False3771debuglog type: <type 'bool'>3772dependencies:False3773dependencies type: <type 'bool'>3774disable_preboot:False3775disable_preboot type: <type 'bool'>3776env:var1=val1,var2=val23777env type: <type 'str'>3778error:False3779error type: <type 'bool'>3780forcenoval:False3781forcenoval type: <type 'bool'>3782geometry:False3783geometry type: <type 'bool'>3784held:False3785held type: <type 'bool'>3786inputfile:False3787inputfile type: <type 'bool'>3788interactive:False3789interactive type: <type 'bool'>3790ion_kernel:default3791ion_kernel type: <type 'str'>3792ion_kerneloptions:False3793ion_kerneloptions type: <type 'bool'>3794jobname:False3795jobname type: <type 'bool'>3796kernel:default3797kernel type: <type 'str'>3798kerneloptions:False3799kerneloptions type: <type 'bool'>3800mode:False3801mode type: <type 'bool'>3802nodecount:103803nodecount type: <type 'str'>3804notify:False3805notify type: <type 'bool'>3806output:False3807output type: <type 'bool'>3808outputprefix:False3809outputprefix type: <type 'bool'>3810preemptable:False3811preemptable type: <type 'bool'>3812proccount:False3813proccount type: <type 'bool'>3814project:False3815project type: <type 'bool'>3816qsub_host:foo.bar3817qsub_host type: <type 'str'>3818queue:default3819queue type: <type 'str'>3820run_project:False3821run_project type: <type 'bool'>3822time:503823time type: <type 'str'>3824umask:False3825umask type: <type 'bool'>3826user_list:False3827user_list type: <type 'bool'>3828verbose:False3829verbose type: <type 'bool'>3830version:False3831version type: <type 'bool'>3832"""3833 stubout_file = "stub.out"3834 expected_results = ( 3835 0, # Expected return status 3836 cmdout, # Expected command output3837 stubout, # Expected stub functions output3838 cmderr, # Expected command error output 3839 ) 3840 testutils.save_testhook("")3841 results = testutils.run_cmd('qsub.py',args,stubout_file) 3842 result = testutils.validate_results(results,expected_results)3843 testutils.remove_testhook()3844 correct = 13845 assert result == correct, "Result:\n%s" % result3846# ---------------------------------------------------------------------------------3847def test_qsub_env_option_2():3848 """3849 qsub test run: env_option_23850 """3851 args = """--env var1=val1:var2=val2 -t50 -n10 /​bin/​ls"""3852 cmdout = \3853"""13854"""3855 cmderr = ''3856 stubout = \3857"""3858GET_IMPLEMENTATION3859ADD_JOBS3860args:[]3861args type: <type 'list'>3862attrs:{}3863attrs type: <type 'dict'>3864command:/​bin/​ls3865command type: <type 'str'>3866cwd:/​tmp3867cwd type: <type 'str'>3868envs:{'var1': 'val1', 'var2': 'val2'}3869envs type: <type 'dict'>3870ion_kernel:default3871ion_kernel type: <type 'str'>3872jobid:*3873jobid type: <type 'str'>3874kernel:default3875kernel type: <type 'str'>3876mode:c13877mode type: <type 'str'>3878nodes:103879nodes type: <type 'int'>3880outputdir:/​tmp3881outputdir type: <type 'str'>3882path:/​tmp3883path type: <type 'str'>3884procs:5123885procs type: <type 'str'>3886queue:default3887queue type: <type 'str'>3888run_project:False3889run_project type: <type 'bool'>3890script_preboot:True3891script_preboot type: <type 'bool'>3892submithost:foo.bar3893submithost type: <type 'str'>3894tag:job3895tag type: <type 'str'>3896ttysession:None3897ttysession type: <type 'NoneType'>3898umask:183899umask type: <type 'int'>3900user:gooduser3901user type: <type 'str'>3902user_list:['gooduser']3903user_list type: <type 'list'>3904walltime:503905walltime type: <type 'str'>3906VALIDATE_JOB3907attrs:{}3908attrs type: <type 'dict'>3909cwd:/​tmp3910cwd type: <type 'str'>3911debug:False3912debug type: <type 'bool'>3913debuglog:False3914debuglog type: <type 'bool'>3915dependencies:False3916dependencies type: <type 'bool'>3917disable_preboot:False3918disable_preboot type: <type 'bool'>3919env:var1=val1:var2=val23920env type: <type 'str'>3921error:False3922error type: <type 'bool'>3923forcenoval:False3924forcenoval type: <type 'bool'>3925geometry:False3926geometry type: <type 'bool'>3927held:False3928held type: <type 'bool'>3929inputfile:False3930inputfile type: <type 'bool'>3931interactive:False3932interactive type: <type 'bool'>3933ion_kernel:default3934ion_kernel type: <type 'str'>3935ion_kerneloptions:False3936ion_kerneloptions type: <type 'bool'>3937jobname:False3938jobname type: <type 'bool'>3939kernel:default3940kernel type: <type 'str'>3941kerneloptions:False3942kerneloptions type: <type 'bool'>3943mode:False3944mode type: <type 'bool'>3945nodecount:103946nodecount type: <type 'str'>3947notify:False3948notify type: <type 'bool'>3949output:False3950output type: <type 'bool'>3951outputprefix:False3952outputprefix type: <type 'bool'>3953preemptable:False3954preemptable type: <type 'bool'>3955proccount:False3956proccount type: <type 'bool'>3957project:False3958project type: <type 'bool'>3959qsub_host:foo.bar3960qsub_host type: <type 'str'>3961queue:default3962queue type: <type 'str'>3963run_project:False3964run_project type: <type 'bool'>3965time:503966time type: <type 'str'>3967umask:False3968umask type: <type 'bool'>3969user_list:False3970user_list type: <type 'bool'>3971verbose:False3972verbose type: <type 'bool'>3973version:False3974version type: <type 'bool'>3975"""3976 stubout_file = "stub.out"3977 expected_results = ( 3978 0, # Expected return status 3979 cmdout, # Expected command output3980 stubout, # Expected stub functions output3981 cmderr, # Expected command error output 3982 ) 3983 testutils.save_testhook("")3984 results = testutils.run_cmd('qsub.py',args,stubout_file) 3985 result = testutils.validate_results(results,expected_results)3986 testutils.remove_testhook()3987 correct = 13988 assert result == correct, "Result:\n%s" % result3989# ---------------------------------------------------------------------------------3990def test_qsub_env_option_3():3991 """3992 qsub test run: env_option_33993 """3994 args = """--env "var1=val1:var2=svar1\=sval1\:svar2\=sval2:var3=val3" -t50 -n10 -d /​bin/​ls"""3995 cmdout = \3996"""13997"""3998 cmderr = \3999"""4000qsub.py --env var1=val1:var2=svar1\=sval1\:svar2\=sval2:var3=val3 -t50 -n10 -d /​bin/​ls4001component: "system.validate_job", defer: False4002 validate_job(4003 {'kernel': 'default', 'verbose': False, 'held': False, 'notify': False, 'ion_kerneloptions': False, 'project': False, 'preemptable': False, 'outputprefix': False, 'umask': False, 'version': False, 'env': 'var1=val1:var2=svar1\\=sval1\\:svar2\\=sval2:var3=val3', 'cwd': '/​tmp', 'run_project': False, 'forcenoval': False, 'kerneloptions': False, 'time': '50', 'jobname': False, 'debug': True, 'dependencies': False, 'debuglog': False, 'qsub_host': 'foo.bar', 'ion_kernel': 'default', 'proccount': False, 'disable_preboot': False, 'geometry': False, 'queue': 'default', 'mode': False, 'error': False, 'nodecount': '10', 'output': False, 'inputfile': False, 'attrs': {}, 'user_list': False, 'interactive': False},4004 )4005component: "system.get_implementation", defer: False4006 get_implementation(4007 )4008fd 1 not associated with a terminal device4009component: "queue-manager.add_jobs", defer: False4010 add_jobs(4011 [{'kernel': 'default', 'tag': 'job', 'attrs': {}, 'outputdir': '/​tmp', 'envs': {'var1': 'val1', 'var3': 'val3', 'var2': 'svar1=sval1:svar2=sval2'}, 'umask': 18, 'submithost': 'foo.bar', 'nodes': 10, 'cwd': '/​tmp', 'run_project': False, 'ttysession': None, 'args': [], 'user': 'gooduser', 'path': '/​tmp', 'ion_kernel': 'default', 'procs': '512', 'walltime': '50', 'jobid': '*', 'queue': 'default', 'script_preboot': True, 'command': '/​bin/​ls', 'mode': 'c1', 'user_list': ['gooduser']}],4012 )4013Environment Vars: {'var1': 'val1', 'var3': 'val3', 'var2': 'svar1=sval1:svar2=sval2'}4014"""4015 stubout = \4016"""4017GET_IMPLEMENTATION4018ADD_JOBS4019args:[]4020args type: <type 'list'>4021attrs:{}4022attrs type: <type 'dict'>4023command:/​bin/​ls4024command type: <type 'str'>4025cwd:/​tmp4026cwd type: <type 'str'>4027envs:{'var1': 'val1', 'var3': 'val3', 'var2': 'svar1=sval1:svar2=sval2'}4028envs type: <type 'dict'>4029ion_kernel:default4030ion_kernel type: <type 'str'>4031jobid:*4032jobid type: <type 'str'>4033kernel:default4034kernel type: <type 'str'>4035mode:c14036mode type: <type 'str'>4037nodes:104038nodes type: <type 'int'>4039outputdir:/​tmp4040outputdir type: <type 'str'>4041path:/​tmp4042path type: <type 'str'>4043procs:5124044procs type: <type 'str'>4045queue:default4046queue type: <type 'str'>4047run_project:False4048run_project type: <type 'bool'>4049script_preboot:True4050script_preboot type: <type 'bool'>4051submithost:foo.bar4052submithost type: <type 'str'>4053tag:job4054tag type: <type 'str'>4055ttysession:None4056ttysession type: <type 'NoneType'>4057umask:184058umask type: <type 'int'>4059user:gooduser4060user type: <type 'str'>4061user_list:['gooduser']4062user_list type: <type 'list'>4063walltime:504064walltime type: <type 'str'>4065VALIDATE_JOB4066attrs:{}4067attrs type: <type 'dict'>4068cwd:/​tmp4069cwd type: <type 'str'>4070debug:True4071debug type: <type 'bool'>4072debuglog:False4073debuglog type: <type 'bool'>4074dependencies:False4075dependencies type: <type 'bool'>4076disable_preboot:False4077disable_preboot type: <type 'bool'>4078env:var1=val1:var2=svar1\=sval1\:svar2\=sval2:var3=val34079env type: <type 'str'>4080error:False4081error type: <type 'bool'>4082forcenoval:False4083forcenoval type: <type 'bool'>4084geometry:False4085geometry type: <type 'bool'>4086held:False4087held type: <type 'bool'>4088inputfile:False4089inputfile type: <type 'bool'>4090interactive:False4091interactive type: <type 'bool'>4092ion_kernel:default4093ion_kernel type: <type 'str'>4094ion_kerneloptions:False4095ion_kerneloptions type: <type 'bool'>4096jobname:False4097jobname type: <type 'bool'>4098kernel:default4099kernel type: <type 'str'>4100kerneloptions:False4101kerneloptions type: <type 'bool'>4102mode:False4103mode type: <type 'bool'>4104nodecount:104105nodecount type: <type 'str'>4106notify:False4107notify type: <type 'bool'>4108output:False4109output type: <type 'bool'>4110outputprefix:False4111outputprefix type: <type 'bool'>4112preemptable:False4113preemptable type: <type 'bool'>4114proccount:False4115proccount type: <type 'bool'>4116project:False4117project type: <type 'bool'>4118qsub_host:foo.bar4119qsub_host type: <type 'str'>4120queue:default4121queue type: <type 'str'>4122run_project:False4123run_project type: <type 'bool'>4124time:504125time type: <type 'str'>4126umask:False4127umask type: <type 'bool'>4128user_list:False4129user_list type: <type 'bool'>4130verbose:False4131verbose type: <type 'bool'>4132version:False4133version type: <type 'bool'>4134"""4135 stubout_file = "stub.out"4136 expected_results = ( 4137 0, # Expected return status 4138 cmdout, # Expected command output4139 stubout, # Expected stub functions output4140 cmderr, # Expected command error output 4141 ) 4142 testutils.save_testhook("")4143 results = testutils.run_cmd('qsub.py',args,stubout_file) 4144 result = testutils.validate_results(results,expected_results)4145 testutils.remove_testhook()4146 correct = 14147 assert result == correct, "Result:\n%s" % result4148# ---------------------------------------------------------------------------------4149def test_qsub_env_option_4():4150 """4151 qsub test run: env_option_44152 """4153 args = """--env var1=val1 --env "var2=svar1\=sval1\:svar2\=sval2" --env var3=val3 -t50 -n10 /​bin/​ls"""4154 cmdout = \4155"""14156"""4157 cmderr = ''4158 stubout = \4159"""4160GET_IMPLEMENTATION4161ADD_JOBS4162args:[]4163args type: <type 'list'>4164attrs:{}4165attrs type: <type 'dict'>4166command:/​bin/​ls4167command type: <type 'str'>4168cwd:/​tmp4169cwd type: <type 'str'>4170envs:{'var1': 'val1', 'var3': 'val3', 'var2': 'svar1=sval1:svar2=sval2'}4171envs type: <type 'dict'>4172ion_kernel:default4173ion_kernel type: <type 'str'>4174jobid:*4175jobid type: <type 'str'>4176kernel:default4177kernel type: <type 'str'>4178mode:c14179mode type: <type 'str'>4180nodes:104181nodes type: <type 'int'>4182outputdir:/​tmp4183outputdir type: <type 'str'>4184path:/​tmp4185path type: <type 'str'>4186procs:5124187procs type: <type 'str'>4188queue:default4189queue type: <type 'str'>4190run_project:False4191run_project type: <type 'bool'>4192script_preboot:True4193script_preboot type: <type 'bool'>4194submithost:foo.bar4195submithost type: <type 'str'>4196tag:job4197tag type: <type 'str'>4198ttysession:None4199ttysession type: <type 'NoneType'>4200umask:184201umask type: <type 'int'>4202user:gooduser4203user type: <type 'str'>4204user_list:['gooduser']4205user_list type: <type 'list'>4206walltime:504207walltime type: <type 'str'>4208VALIDATE_JOB4209attrs:{}4210attrs type: <type 'dict'>4211cwd:/​tmp4212cwd type: <type 'str'>4213debug:False4214debug type: <type 'bool'>4215debuglog:False4216debuglog type: <type 'bool'>4217dependencies:False4218dependencies type: <type 'bool'>4219disable_preboot:False4220disable_preboot type: <type 'bool'>4221env:var1=val1:var2=svar1\=sval1\:svar2\=sval2:var3=val34222env type: <type 'str'>4223error:False4224error type: <type 'bool'>4225forcenoval:False4226forcenoval type: <type 'bool'>4227geometry:False4228geometry type: <type 'bool'>4229held:False4230held type: <type 'bool'>4231inputfile:False4232inputfile type: <type 'bool'>4233interactive:False4234interactive type: <type 'bool'>4235ion_kernel:default4236ion_kernel type: <type 'str'>4237ion_kerneloptions:False4238ion_kerneloptions type: <type 'bool'>4239jobname:False4240jobname type: <type 'bool'>4241kernel:default4242kernel type: <type 'str'>4243kerneloptions:False4244kerneloptions type: <type 'bool'>4245mode:False4246mode type: <type 'bool'>4247nodecount:104248nodecount type: <type 'str'>4249notify:False4250notify type: <type 'bool'>4251output:False4252output type: <type 'bool'>4253outputprefix:False4254outputprefix type: <type 'bool'>4255preemptable:False4256preemptable type: <type 'bool'>4257proccount:False4258proccount type: <type 'bool'>4259project:False4260project type: <type 'bool'>4261qsub_host:foo.bar4262qsub_host type: <type 'str'>4263queue:default4264queue type: <type 'str'>4265run_project:False4266run_project type: <type 'bool'>4267time:504268time type: <type 'str'>4269umask:False4270umask type: <type 'bool'>4271user_list:False4272user_list type: <type 'bool'>4273verbose:False4274verbose type: <type 'bool'>4275version:False4276version type: <type 'bool'>4277"""4278 stubout_file = "stub.out"4279 expected_results = ( 4280 0, # Expected return status 4281 cmdout, # Expected command output4282 stubout, # Expected stub functions output4283 cmderr, # Expected command error output 4284 ) 4285 testutils.save_testhook("")4286 results = testutils.run_cmd('qsub.py',args,stubout_file) 4287 result = testutils.validate_results(results,expected_results)4288 testutils.remove_testhook()4289 correct = 14290 assert result == correct, "Result:\n%s" % result4291# ---------------------------------------------------------------------------------4292def test_qsub_script_1():4293 """4294 qsub test run: script_14295 """4296 args = """cobalt_script1.sh"""4297 cmdout = \4298"""14299"""4300 cmderr = ''4301 stubout = \4302"""4303GET_IMPLEMENTATION4304ADD_JOBS4305args:[]4306args type: <type 'list'>4307attrs:{}4308attrs type: <type 'dict'>4309command:/​tmp/​cobalt_script1.sh4310command type: <type 'str'>4311cwd:/​tmp4312cwd type: <type 'str'>4313envs:{'a': '1', 'c': '3', 'b': '2'}4314envs type: <type 'dict'>4315ion_kernel:default4316ion_kernel type: <type 'str'>4317jobid:*4318jobid type: <type 'str'>4319kernel:default4320kernel type: <type 'str'>4321mode:script4322mode type: <type 'str'>4323nodes:1004324nodes type: <type 'int'>4325outputdir:/​tmp4326outputdir type: <type 'str'>4327path:/​tmp4328path type: <type 'str'>4329procs:5124330procs type: <type 'str'>4331queue:default4332queue type: <type 'str'>4333run_project:False4334run_project type: <type 'bool'>4335script_preboot:True4336script_preboot type: <type 'bool'>4337submithost:foo.bar4338submithost type: <type 'str'>4339tag:job4340tag type: <type 'str'>4341ttysession:None4342ttysession type: <type 'NoneType'>4343umask:184344umask type: <type 'int'>4345user:gooduser4346user type: <type 'str'>4347user_list:['gooduser']4348user_list type: <type 'list'>4349walltime:754350walltime type: <type 'str'>4351VALIDATE_JOB4352attrs:{}4353attrs type: <type 'dict'>4354cwd:/​tmp4355cwd type: <type 'str'>4356debug:False4357debug type: <type 'bool'>4358debuglog:False4359debuglog type: <type 'bool'>4360dependencies:False4361dependencies type: <type 'bool'>4362disable_preboot:False4363disable_preboot type: <type 'bool'>4364env:a=1:c=3:b=24365env type: <type 'str'>4366error:False4367error type: <type 'bool'>4368forcenoval:False4369forcenoval type: <type 'bool'>4370geometry:False4371geometry type: <type 'bool'>4372held:False4373held type: <type 'bool'>4374inputfile:False4375inputfile type: <type 'bool'>4376interactive:False4377interactive type: <type 'bool'>4378ion_kernel:default4379ion_kernel type: <type 'str'>4380ion_kerneloptions:False4381ion_kerneloptions type: <type 'bool'>4382jobname:False4383jobname type: <type 'bool'>4384kernel:default4385kernel type: <type 'str'>4386kerneloptions:False4387kerneloptions type: <type 'bool'>4388mode:script4389mode type: <type 'str'>4390nodecount:1004391nodecount type: <type 'str'>4392notify:False4393notify type: <type 'bool'>4394output:False4395output type: <type 'bool'>4396outputprefix:False4397outputprefix type: <type 'bool'>4398preemptable:False4399preemptable type: <type 'bool'>4400proccount:False4401proccount type: <type 'bool'>4402project:False4403project type: <type 'bool'>4404qsub_host:foo.bar4405qsub_host type: <type 'str'>4406queue:default4407queue type: <type 'str'>4408run_project:False4409run_project type: <type 'bool'>4410time:754411time type: <type 'str'>4412umask:False4413umask type: <type 'bool'>4414user_list:False4415user_list type: <type 'bool'>4416verbose:False4417verbose type: <type 'bool'>4418version:False4419version type: <type 'bool'>4420"""4421 stubout_file = "stub.out"4422 expected_results = ( 4423 0, # Expected return status 4424 cmdout, # Expected command output4425 stubout, # Expected stub functions output4426 cmderr, # Expected command error output 4427 ) 4428 testutils.save_testhook("")4429 results = testutils.run_cmd('qsub.py',args,stubout_file) 4430 result = testutils.validate_results(results,expected_results)4431 testutils.remove_testhook()4432 correct = 14433 assert result == correct, "Result:\n%s" % result4434# ---------------------------------------------------------------------------------4435def test_qsub_script_2():4436 """4437 qsub test run: script_24438 """4439 args = """-t 50 cobalt_script1.sh"""4440 cmdout = \4441"""14442"""4443 cmderr = ''4444 stubout = \4445"""4446GET_IMPLEMENTATION4447ADD_JOBS4448args:[]4449args type: <type 'list'>4450attrs:{}4451attrs type: <type 'dict'>4452command:/​tmp/​cobalt_script1.sh4453command type: <type 'str'>4454cwd:/​tmp4455cwd type: <type 'str'>4456envs:{'a': '1', 'c': '3', 'b': '2'}4457envs type: <type 'dict'>4458ion_kernel:default4459ion_kernel type: <type 'str'>4460jobid:*4461jobid type: <type 'str'>4462kernel:default4463kernel type: <type 'str'>4464mode:script4465mode type: <type 'str'>4466nodes:1004467nodes type: <type 'int'>4468outputdir:/​tmp4469outputdir type: <type 'str'>4470path:/​tmp4471path type: <type 'str'>4472procs:5124473procs type: <type 'str'>4474queue:default4475queue type: <type 'str'>4476run_project:False4477run_project type: <type 'bool'>4478script_preboot:True4479script_preboot type: <type 'bool'>4480submithost:foo.bar4481submithost type: <type 'str'>4482tag:job4483tag type: <type 'str'>4484ttysession:None4485ttysession type: <type 'NoneType'>4486umask:184487umask type: <type 'int'>4488user:gooduser4489user type: <type 'str'>4490user_list:['gooduser']4491user_list type: <type 'list'>4492walltime:504493walltime type: <type 'str'>4494VALIDATE_JOB4495attrs:{}4496attrs type: <type 'dict'>4497cwd:/​tmp4498cwd type: <type 'str'>4499debug:False4500debug type: <type 'bool'>4501debuglog:False4502debuglog type: <type 'bool'>4503dependencies:False4504dependencies type: <type 'bool'>4505disable_preboot:False4506disable_preboot type: <type 'bool'>4507env:a=1:c=3:b=24508env type: <type 'str'>4509error:False4510error type: <type 'bool'>4511forcenoval:False4512forcenoval type: <type 'bool'>4513geometry:False4514geometry type: <type 'bool'>4515held:False4516held type: <type 'bool'>4517inputfile:False4518inputfile type: <type 'bool'>4519interactive:False4520interactive type: <type 'bool'>4521ion_kernel:default4522ion_kernel type: <type 'str'>4523ion_kerneloptions:False4524ion_kerneloptions type: <type 'bool'>4525jobname:False4526jobname type: <type 'bool'>4527kernel:default4528kernel type: <type 'str'>4529kerneloptions:False4530kerneloptions type: <type 'bool'>4531mode:script4532mode type: <type 'str'>4533nodecount:1004534nodecount type: <type 'str'>4535notify:False4536notify type: <type 'bool'>4537output:False4538output type: <type 'bool'>4539outputprefix:False4540outputprefix type: <type 'bool'>4541preemptable:False4542preemptable type: <type 'bool'>4543proccount:False4544proccount type: <type 'bool'>4545project:False4546project type: <type 'bool'>4547qsub_host:foo.bar4548qsub_host type: <type 'str'>4549queue:default4550queue type: <type 'str'>4551run_project:False4552run_project type: <type 'bool'>4553time:504554time type: <type 'str'>4555umask:False4556umask type: <type 'bool'>4557user_list:False4558user_list type: <type 'bool'>4559verbose:False4560verbose type: <type 'bool'>4561version:False4562version type: <type 'bool'>4563"""4564 stubout_file = "stub.out"4565 expected_results = ( 4566 0, # Expected return status 4567 cmdout, # Expected command output4568 stubout, # Expected stub functions output4569 cmderr, # Expected command error output 4570 ) 4571 testutils.save_testhook("")4572 results = testutils.run_cmd('qsub.py',args,stubout_file) 4573 result = testutils.validate_results(results,expected_results)4574 testutils.remove_testhook()4575 correct = 14576 assert result == correct, "Result:\n%s" % result4577# ---------------------------------------------------------------------------------4578def test_qsub_script_3():4579 """4580 qsub test run: script_34581 """4582 args = """--mode vn cobalt_script1.sh"""4583 cmdout = ''4584 cmderr = \4585"""Mode already set to 'script' and trying to set it again to 'vn'4586"""4587 stubout = ''4588 stubout_file = "stub.out"4589 expected_results = ( 4590 256, # Expected return status 4591 cmdout, # Expected command output4592 stubout, # Expected stub functions output4593 cmderr, # Expected command error output 4594 ) 4595 testutils.save_testhook("")4596 results = testutils.run_cmd('qsub.py',args,stubout_file) 4597 result = testutils.validate_results(results,expected_results)4598 testutils.remove_testhook()4599 correct = 14600 assert result == correct, "Result:\n%s" % result4601# ---------------------------------------------------------------------------------4602def test_qsub_script_4():4603 """4604 qsub test run: script_44605 """4606 args = """-d cobalt_script2.sh"""4607 cmdout = ''4608 cmderr = \4609"""4610qsub.py -d cobalt_script2.sh4611Mode already set to 'script' and trying to set it again to 'vn'4612"""4613 stubout = ''4614 stubout_file = "stub.out"4615 expected_results = ( 4616 256, # Expected return status 4617 cmdout, # Expected command output4618 stubout, # Expected stub functions output4619 cmderr, # Expected command error output 4620 ) 4621 testutils.save_testhook("")4622 results = testutils.run_cmd('qsub.py',args,stubout_file) 4623 result = testutils.validate_results(results,expected_results)4624 testutils.remove_testhook()4625 correct = 14626 assert result == correct, "Result:\n%s" % result4627# ---------------------------------------------------------------------------------4628def test_qsub_walltime_0():4629 """4630 qsub test run: walltime_04631 """4632 args = """-t0 -n 10 /​bin/​ls"""4633 cmdout = \4634"""14635"""4636 cmderr = ''4637 stubout = \4638"""4639GET_IMPLEMENTATION4640ADD_JOBS4641args:[]4642args type: <type 'list'>4643attrs:{}4644attrs type: <type 'dict'>4645command:/​bin/​ls4646command type: <type 'str'>4647cwd:/​tmp4648cwd type: <type 'str'>4649ion_kernel:default4650ion_kernel type: <type 'str'>4651jobid:*4652jobid type: <type 'str'>4653kernel:default4654kernel type: <type 'str'>4655mode:c14656mode type: <type 'str'>4657nodes:104658nodes type: <type 'int'>4659outputdir:/​tmp4660outputdir type: <type 'str'>4661path:/​tmp4662path type: <type 'str'>4663procs:5124664procs type: <type 'str'>4665queue:default4666queue type: <type 'str'>4667run_project:False4668run_project type: <type 'bool'>4669script_preboot:True4670script_preboot type: <type 'bool'>4671submithost:foo.bar4672submithost type: <type 'str'>4673tag:job4674tag type: <type 'str'>4675ttysession:None4676ttysession type: <type 'NoneType'>4677umask:184678umask type: <type 'int'>4679user:gooduser4680user type: <type 'str'>4681user_list:['gooduser']4682user_list type: <type 'list'>4683walltime:04684walltime type: <type 'str'>4685VALIDATE_JOB4686attrs:{}4687attrs type: <type 'dict'>4688cwd:/​tmp4689cwd type: <type 'str'>4690debug:False4691debug type: <type 'bool'>4692debuglog:False4693debuglog type: <type 'bool'>4694dependencies:False4695dependencies type: <type 'bool'>4696disable_preboot:False4697disable_preboot type: <type 'bool'>4698env:False4699env type: <type 'bool'>4700error:False4701error type: <type 'bool'>4702forcenoval:False4703forcenoval type: <type 'bool'>4704geometry:False4705geometry type: <type 'bool'>4706held:False4707held type: <type 'bool'>4708inputfile:False4709inputfile type: <type 'bool'>4710interactive:False4711interactive type: <type 'bool'>4712ion_kernel:default4713ion_kernel type: <type 'str'>4714ion_kerneloptions:False4715ion_kerneloptions type: <type 'bool'>4716jobname:False4717jobname type: <type 'bool'>4718kernel:default4719kernel type: <type 'str'>4720kerneloptions:False4721kerneloptions type: <type 'bool'>4722mode:False4723mode type: <type 'bool'>4724nodecount:104725nodecount type: <type 'str'>4726notify:False4727notify type: <type 'bool'>4728output:False4729output type: <type 'bool'>4730outputprefix:False4731outputprefix type: <type 'bool'>4732preemptable:False4733preemptable type: <type 'bool'>4734proccount:False4735proccount type: <type 'bool'>4736project:False4737project type: <type 'bool'>4738qsub_host:foo.bar4739qsub_host type: <type 'str'>4740queue:default4741queue type: <type 'str'>4742run_project:False4743run_project type: <type 'bool'>4744time:04745time type: <type 'str'>4746umask:False4747umask type: <type 'bool'>4748user_list:False4749user_list type: <type 'bool'>4750verbose:False4751verbose type: <type 'bool'>4752version:False4753version type: <type 'bool'>4754"""4755 stubout_file = "stub.out"4756 expected_results = ( 4757 0, # Expected return status 4758 cmdout, # Expected command output4759 stubout, # Expected stub functions output4760 cmderr, # Expected command error output 4761 ) 4762 testutils.save_testhook("")4763 results = testutils.run_cmd('qsub.py',args,stubout_file) 4764 result = testutils.validate_results(results,expected_results)4765 testutils.remove_testhook()4766 correct = 14767 assert result == correct, "Result:\n%s" % result4768# ---------------------------------------------------------------------------------4769def test_qsub_interactive_1():4770 """4771 qsub test run: interactive_14772 """4773 args = """-I -t50 -n 1 /​bin/​ls"""4774 cmdout = ''4775 cmderr = \4776"""An executable may not be specified if using the interactive option.4777"""4778 stubout = ''4779 stubout_file = "stub.out"4780 expected_results = ( 4781 256, # Expected return status 4782 cmdout, # Expected command output4783 stubout, # Expected stub functions output4784 cmderr, # Expected command error output 4785 ) 4786 testutils.save_testhook("")4787 results = testutils.run_cmd('qsub.py',args,stubout_file) 4788 result = testutils.validate_results(results,expected_results)4789 testutils.remove_testhook()4790 correct = 14791 assert result == correct, "Result:\n%s" % result4792# ---------------------------------------------------------------------------------4793def test_qsub_interactive_2():4794 """4795 qsub test run: interactive_24796 """4797 args = """-I -t50 -n 1 -i inputfile"""4798 cmdout = ''4799 cmderr = \4800"""Cannot specify an input file for interactive jobs.4801"""4802 stubout = ''4803 stubout_file = "stub.out"4804 expected_results = ( 4805 256, # Expected return status 4806 cmdout, # Expected command output4807 stubout, # Expected stub functions output4808 cmderr, # Expected command error output 4809 ) 4810 testutils.save_testhook("")4811 results = testutils.run_cmd('qsub.py',args,stubout_file) 4812 result = testutils.validate_results(results,expected_results)4813 testutils.remove_testhook()4814 correct = 14815 assert result == correct, "Result:\n%s" % result4816# ---------------------------------------------------------------------------------4817def test_qsub_interactive_4():4818 """4819 qsub test run: interactive_44820 """4821 args = """-I -t50 -n 1 --mode script"""4822 cmdout = ''4823 cmderr = \4824"""Mode already set to 'interactive' and trying to set it again to 'script'4825"""4826 stubout = ''4827 stubout_file = "stub.out"4828 expected_results = ( 4829 256, # Expected return status 4830 cmdout, # Expected command output4831 stubout, # Expected stub functions output4832 cmderr, # Expected command error output 4833 ) 4834 testutils.save_testhook("")4835 results = testutils.run_cmd('qsub.py',args,stubout_file) 4836 result = testutils.validate_results(results,expected_results)4837 testutils.remove_testhook()4838 correct = 14839 assert result == correct, "Result:\n%s" % result4840# ---------------------------------------------------------------------------------4841def test_qsub_interactive_5():4842 """4843 qsub test run: interactive_54844 """4845 args = """-I -t50 -n 1"""4846 cmdout = \4847"""Wait for job 1 to start...4848Opening interactive session to /​4849Deleting interactive job 14850"""4851 cmderr = ''4852 stubout = \4853"""4854GET_IMPLEMENTATION4855ADD_JOBS4856attrs:{}4857attrs type: <type 'dict'>4858cwd:/​tmp4859cwd type: <type 'str'>4860ion_kernel:default4861ion_kernel type: <type 'str'>4862jobid:*4863jobid type: <type 'str'>4864kernel:default4865kernel type: <type 'str'>4866mode:interactive4867mode type: <type 'str'>4868nodes:14869nodes type: <type 'int'>4870outputdir:/​tmp4871outputdir type: <type 'str'>4872path:/​tmp4873path type: <type 'str'>4874procs:5124875procs type: <type 'str'>4876queue:default4877queue type: <type 'str'>4878run_project:False4879run_project type: <type 'bool'>4880script_preboot:True4881script_preboot type: <type 'bool'>4882submithost:foo.bar4883submithost type: <type 'str'>4884tag:job4885tag type: <type 'str'>4886ttysession:None4887ttysession type: <type 'NoneType'>4888umask:184889umask type: <type 'int'>4890user:gooduser4891user type: <type 'str'>4892user_list:['gooduser']4893user_list type: <type 'list'>4894walltime:504895walltime type: <type 'str'>4896VALIDATE_JOB4897attrs:{}4898attrs type: <type 'dict'>4899cwd:/​tmp4900cwd type: <type 'str'>4901debug:False4902debug type: <type 'bool'>4903debuglog:False4904debuglog type: <type 'bool'>4905dependencies:False4906dependencies type: <type 'bool'>4907disable_preboot:False4908disable_preboot type: <type 'bool'>4909env:False4910env type: <type 'bool'>4911error:False4912error type: <type 'bool'>4913forcenoval:False4914forcenoval type: <type 'bool'>4915geometry:False4916geometry type: <type 'bool'>4917held:False4918held type: <type 'bool'>4919inputfile:False4920inputfile type: <type 'bool'>4921interactive:False4922interactive type: <type 'bool'>4923ion_kernel:default4924ion_kernel type: <type 'str'>4925ion_kerneloptions:False4926ion_kerneloptions type: <type 'bool'>4927jobname:False4928jobname type: <type 'bool'>4929kernel:default4930kernel type: <type 'str'>4931kerneloptions:False4932kerneloptions type: <type 'bool'>4933mode:interactive4934mode type: <type 'str'>4935nodecount:14936nodecount type: <type 'str'>4937notify:False4938notify type: <type 'bool'>4939output:False4940output type: <type 'bool'>4941outputprefix:False4942outputprefix type: <type 'bool'>4943preemptable:False4944preemptable type: <type 'bool'>4945proccount:False4946proccount type: <type 'bool'>4947project:False4948project type: <type 'bool'>4949qsub_host:foo.bar4950qsub_host type: <type 'str'>4951queue:default4952queue type: <type 'str'>4953run_project:False4954run_project type: <type 'bool'>4955time:504956time type: <type 'str'>4957umask:False4958umask type: <type 'bool'>4959user_list:False4960user_list type: <type 'bool'>4961verbose:False4962verbose type: <type 'bool'>4963version:False4964version type: <type 'bool'>4965GET_IMPLEMENTATION4966GET_JOBS4967jobid:14968jobid type: <type 'int'>4969location:*4970location type: <type 'str'>4971resid:*4972resid type: <type 'str'>4973state:*4974state type: <type 'str'>4975tag:job4976tag type: <type 'str'>4977DEL_JOBS4978force:False4979whoami:gooduser4980jobid:14981jobid type: <type 'int'>4982tag:job4983tag type: <type 'str'>4984user:gooduser4985user type: <type 'str'>4986"""4987 stubout_file = "stub.out"4988 expected_results = ( 4989 0, # Expected return status 4990 cmdout, # Expected command output4991 stubout, # Expected stub functions output4992 cmderr, # Expected command error output 4993 ) 4994 testutils.save_testhook("JOB_RUNNING")4995 results = testutils.run_cmd('qsub.py',args,stubout_file) 4996 result = testutils.validate_results(results,expected_results)4997 testutils.remove_testhook()4998 correct = 14999 assert result == correct, "Result:\n%s" % result5000# ---------------------------------------------------------------------------------5001def test_qsub_interactive_6():5002 """5003 qsub test run: interactive_65004 """5005 args = """--interactive -t50 -n 1"""5006 cmdout = \5007"""Wait for job 1 to start...5008Opening interactive session to /​5009Deleting interactive job 15010"""5011 cmderr = ''5012 stubout = \5013"""5014GET_IMPLEMENTATION5015ADD_JOBS5016attrs:{}5017attrs type: <type 'dict'>5018cwd:/​tmp5019cwd type: <type 'str'>5020ion_kernel:default5021ion_kernel type: <type 'str'>5022jobid:*5023jobid type: <type 'str'>5024kernel:default5025kernel type: <type 'str'>5026mode:interactive5027mode type: <type 'str'>5028nodes:15029nodes type: <type 'int'>5030outputdir:/​tmp5031outputdir type: <type 'str'>5032path:/​tmp5033path type: <type 'str'>5034procs:5125035procs type: <type 'str'>5036queue:default5037queue type: <type 'str'>5038run_project:False5039run_project type: <type 'bool'>5040script_preboot:True5041script_preboot type: <type 'bool'>5042submithost:foo.bar5043submithost type: <type 'str'>5044tag:job5045tag type: <type 'str'>5046ttysession:None5047ttysession type: <type 'NoneType'>5048umask:185049umask type: <type 'int'>5050user:gooduser5051user type: <type 'str'>5052user_list:['gooduser']5053user_list type: <type 'list'>5054walltime:505055walltime type: <type 'str'>5056VALIDATE_JOB5057attrs:{}5058attrs type: <type 'dict'>5059cwd:/​tmp5060cwd type: <type 'str'>5061debug:False5062debug type: <type 'bool'>5063debuglog:False5064debuglog type: <type 'bool'>5065dependencies:False5066dependencies type: <type 'bool'>5067disable_preboot:False5068disable_preboot type: <type 'bool'>5069env:False5070env type: <type 'bool'>5071error:False5072error type: <type 'bool'>5073forcenoval:False5074forcenoval type: <type 'bool'>5075geometry:False5076geometry type: <type 'bool'>5077held:False5078held type: <type 'bool'>5079inputfile:False5080inputfile type: <type 'bool'>5081interactive:False5082interactive type: <type 'bool'>5083ion_kernel:default5084ion_kernel type: <type 'str'>5085ion_kerneloptions:False5086ion_kerneloptions type: <type 'bool'>5087jobname:False5088jobname type: <type 'bool'>5089kernel:default5090kernel type: <type 'str'>5091kerneloptions:False5092kerneloptions type: <type 'bool'>5093mode:interactive5094mode type: <type 'str'>5095nodecount:15096nodecount type: <type 'str'>5097notify:False5098notify type: <type 'bool'>5099output:False5100output type: <type 'bool'>5101outputprefix:False5102outputprefix type: <type 'bool'>5103preemptable:False5104preemptable type: <type 'bool'>5105proccount:False5106proccount type: <type 'bool'>5107project:False5108project type: <type 'bool'>5109qsub_host:foo.bar5110qsub_host type: <type 'str'>5111queue:default5112queue type: <type 'str'>5113run_project:False5114run_project type: <type 'bool'>5115time:505116time type: <type 'str'>5117umask:False5118umask type: <type 'bool'>5119user_list:False5120user_list type: <type 'bool'>5121verbose:False5122verbose type: <type 'bool'>5123version:False5124version type: <type 'bool'>5125GET_IMPLEMENTATION5126GET_JOBS5127jobid:15128jobid type: <type 'int'>5129location:*5130location type: <type 'str'>5131resid:*5132resid type: <type 'str'>5133state:*5134state type: <type 'str'>5135tag:job5136tag type: <type 'str'>5137DEL_JOBS5138force:False5139whoami:gooduser5140jobid:15141jobid type: <type 'int'>5142tag:job5143tag type: <type 'str'>5144user:gooduser5145user type: <type 'str'>5146"""5147 stubout_file = "stub.out"5148 expected_results = ( 5149 0, # Expected return status 5150 cmdout, # Expected command output5151 stubout, # Expected stub functions output5152 cmderr, # Expected command error output 5153 ) 5154 testutils.save_testhook("JOB_RUNNING")5155 results = testutils.run_cmd('qsub.py',args,stubout_file) 5156 result = testutils.validate_results(results,expected_results)5157 testutils.remove_testhook()5158 correct = 15159 assert result == correct, "Result:\n%s" % result5160# ---------------------------------------------------------------------------------5161def test_qsub_interactive_7():5162 """5163 qsub test run: interactive_75164 """5165 args = """--mode interactive -t50 -n 1"""5166 cmdout = \5167"""Wait for job 1 to start...5168Opening interactive session to /​5169Deleting interactive job 15170"""5171 cmderr = ''5172 stubout = \5173"""5174GET_IMPLEMENTATION5175ADD_JOBS5176attrs:{}5177attrs type: <type 'dict'>5178cwd:/​tmp5179cwd type: <type 'str'>5180ion_kernel:default5181ion_kernel type: <type 'str'>5182jobid:*5183jobid type: <type 'str'>5184kernel:default5185kernel type: <type 'str'>5186mode:interactive5187mode type: <type 'str'>5188nodes:15189nodes type: <type 'int'>5190outputdir:/​tmp5191outputdir type: <type 'str'>5192path:/​tmp5193path type: <type 'str'>5194procs:5125195procs type: <type 'str'>5196queue:default5197queue type: <type 'str'>5198run_project:False5199run_project type: <type 'bool'>5200script_preboot:True5201script_preboot type: <type 'bool'>5202submithost:foo.bar5203submithost type: <type 'str'>5204tag:job5205tag type: <type 'str'>5206ttysession:None5207ttysession type: <type 'NoneType'>5208umask:185209umask type: <type 'int'>5210user:gooduser5211user type: <type 'str'>5212user_list:['gooduser']5213user_list type: <type 'list'>5214walltime:505215walltime type: <type 'str'>5216VALIDATE_JOB5217attrs:{}5218attrs type: <type 'dict'>5219cwd:/​tmp5220cwd type: <type 'str'>5221debug:False5222debug type: <type 'bool'>5223debuglog:False5224debuglog type: <type 'bool'>5225dependencies:False5226dependencies type: <type 'bool'>5227disable_preboot:False5228disable_preboot type: <type 'bool'>5229env:False5230env type: <type 'bool'>5231error:False5232error type: <type 'bool'>5233forcenoval:False5234forcenoval type: <type 'bool'>5235geometry:False5236geometry type: <type 'bool'>5237held:False5238held type: <type 'bool'>5239inputfile:False5240inputfile type: <type 'bool'>5241interactive:False5242interactive type: <type 'bool'>5243ion_kernel:default5244ion_kernel type: <type 'str'>5245ion_kerneloptions:False5246ion_kerneloptions type: <type 'bool'>5247jobname:False5248jobname type: <type 'bool'>5249kernel:default5250kernel type: <type 'str'>5251kerneloptions:False5252kerneloptions type: <type 'bool'>5253mode:interactive5254mode type: <type 'str'>5255nodecount:15256nodecount type: <type 'str'>5257notify:False5258notify type: <type 'bool'>5259output:False5260output type: <type 'bool'>5261outputprefix:False5262outputprefix type: <type 'bool'>5263preemptable:False5264preemptable type: <type 'bool'>5265proccount:False5266proccount type: <type 'bool'>5267project:False5268project type: <type 'bool'>5269qsub_host:foo.bar5270qsub_host type: <type 'str'>5271queue:default5272queue type: <type 'str'>5273run_project:False5274run_project type: <type 'bool'>5275time:505276time type: <type 'str'>5277umask:False5278umask type: <type 'bool'>5279user_list:False5280user_list type: <type 'bool'>5281verbose:False5282verbose type: <type 'bool'>5283version:False5284version type: <type 'bool'>5285GET_IMPLEMENTATION5286GET_JOBS5287jobid:15288jobid type: <type 'int'>5289location:*5290location type: <type 'str'>5291resid:*5292resid type: <type 'str'>5293state:*5294state type: <type 'str'>5295tag:job5296tag type: <type 'str'>5297DEL_JOBS5298force:False5299whoami:gooduser5300jobid:15301jobid type: <type 'int'>5302tag:job5303tag type: <type 'str'>5304user:gooduser5305user type: <type 'str'>5306"""5307 stubout_file = "stub.out"5308 expected_results = ( 5309 0, # Expected return status 5310 cmdout, # Expected command output5311 stubout, # Expected stub functions output5312 cmderr, # Expected command error output 5313 ) 5314 testutils.save_testhook("JOB_RUNNING")5315 results = testutils.run_cmd('qsub.py',args,stubout_file) 5316 result = testutils.validate_results(results,expected_results)5317 testutils.remove_testhook()5318 correct = 1...

run_project.py

Source:run_project.py

1#!/​usr/​bin/​env python2''' primary entry point for the NLP pipeline '''3import collections4import itertools5import logging6import operator7import re8import os9import string10import sys11import hashlib12import copy13import nltk14import nose15import unittest16import numpy as np17from sklearn.feature_extraction.text import TfidfVectorizer18from sklearn.cluster import MiniBatchKMeans19from sklearn.cross_validation import train_test_split20from sklearn.linear_model import LogisticRegression21#sys.path.append('C:\Users\NDT567\Small_Data\source')22from settings import INPUT_FILE_PATH_SAMPLE as INPUT_FILE_PATH 23import settings24# numpy options25np.set_printoptions(linewidth=150)26# what is a random seed? http:/​/​en.wikipedia.org/​wiki/​Random_seed27# why is it a good idea? http:/​/​stats.stackexchange.com/​questions/​121225/​references-and-best-practices-for-setting-seeds-in-pseudo-random-number-generati28np.random.seed(42)29# logging configuration30logging.basicConfig(format='%(asctime)s - %(levelname)s: %(message)s', level=logging.INFO)31# module variables32UserSpeechFragment = collections.namedtuple('UserSpeechFragment', ['role', 'fragment_text']) #A named tuple containing who said what text33AGENT, CALLER = 'agent', 'caller' #The names given to the agent and the caller in the transcripts34def quiz_wiki_url_for_first_speech_recognition():35 ''' on the Speech Recognition Wikipedia page, look in the 'History'36 section, and find the first organization to investigate speech 37 recognition. Return that organization's wiki url.38 test with:39 nosetests -v run_project:test_quiz_wiki_url_for_first_speech_recognition '''40 return 'http:/​/​en.wikipedia.org/​wiki/​Bell_Labs'41def test_quiz_wiki_url_for_first_speech_recognition():42 assert '35a23e07d7b4ec4d76eb96b46857a05f115f07920b7dd81239c96931' == hashlib.sha224(quiz_wiki_url_for_first_speech_recognition()).hexdigest()43def get_lines_from_transcript_file(file_path=INPUT_FILE_PATH):44 ''' using the "with" statement, the "open" function,45 the INPUT_FILE_PATH variable defined above, and the "yield" keyword,46 return a generator that yields each line of the transcripts file.47 Make sure to .lower() each line - it'll be easier to process48 the text with no capital letters.49 Arguments:50 file_path: A string containing the filename of the file51 to read in. Defaults to the INPUT_FILE_PATH variable.52 Returns:53 A generator which returns the next line of the file each54 time its .next() function is called.55 56 Read the Green-Checked answer to this StackOverflow question:57 http:/​/​stackoverflow.com/​questions/​11555468/​how-should-i-read-a-file-line-by-line-in-python58 More info on using the with statement:59 http:/​/​effbot.org/​zone/​python-with-statement.htm60 More info on yield statements:61 http:/​/​stackoverflow.com/​questions/​231767/​what-does-the-yield-keyword-do-in-python62 test with:63 nosetests -v run_project:test_get_lines_from_transcript_file '''64 with open(file_path) as fp:65 for line in fp:66 yield line.lower()67 68class test_get_lines_from_transcript_file(object):69 def setup(self):70 self.test_filename = 'test_file.nogit'71 test_file = open(self.test_filename, 'wb')72 test_file.write('This is line 1.\nThis is line 2.\n')73 test_file.close()74 self.line_generator = get_lines_from_transcript_file(file_path=self.test_filename)75 def teardown(self):76 self.line_generator.close()77 os.remove(self.test_filename)78 def test_get_lines_from_transcript_file_line0(self):79 line0 = self.line_generator.next()80 assert line0 == 'this is line 1.\n'81 def test_get_lines_from_transcript_file_line1(self):82 line0 = self.line_generator.next()83 line1 = self.line_generator.next()84 assert line1 == 'this is line 2.\n'85def line_to_feature_dictionary(line):86 ''' Each call record has several fields, containing information87 about the call as well as the transcript. Edit this function88 to split a pipe (|) delimited call record up into its components,89 removing the leading and trailing whitespace around each field.90 Arguments:91 line: A single call record, with sections delimited92 by the pipe (|) character.93 Returns:94 call_dict: A dictionary of the call components,95 with the keys listed in the 'columns' list.96 Hint: use line.split('|'), and check out the string.strip()97 command. To make the output dictionary, you can start98 with an empty dictionary ({}), then iterate through99 the columns list adding key:value pairs to it.100 All remaining prior information regarding the data schema is available here:101 https:/​/​github.kdc.capitalone.com/​BigDataAcademy/​Small_Data/​issues/​24102 test with:103 nosetests -v run_project:test_line_to_feature_dictionary '''104 columns = ['audio_file_name',#Name of the audio file transcript105 'date',#Date of call106 'transcript',#The call text107 'hvs_or_core',#Where the call was routed108 'zero_or_one0',#dummy field?109 'zero_or_one1',#dummy field?110 'customer_net_promoter_score_response',#NPS score111 'blank_field']#dummy field due to trailing |112 d={}113 for i in range (len(line.split('|'))):114 d[columns[i]]=string.strip(line.split('|')[i])115 return d116 117def test_line_to_feature_dictionary():118 # dummy input data119 audio_file_name = 'cap1_core_2014-07-27-02-05-42_2950008_21_6040556861322767314_1_46.wav'120 date = '27jul14'121 transcript = 'speaker1 (agent): thank you for calling capital one.'122 hvs_or_core = 'hvs'123 zero_or_one0 = '0'124 zero_or_one1 = '1'125 customer_net_promoter_score_response = ''126 blank_field = ''127 input_row = [audio_file_name, date, transcript, hvs_or_core,128 zero_or_one0, zero_or_one1, customer_net_promoter_score_response, blank_field]129 input_line = ' |'.join(input_row)130 expected_output = {'audio_file_name': audio_file_name,131 'date': date,132 'transcript': transcript,133 'hvs_or_core': hvs_or_core,134 'zero_or_one0': zero_or_one0,135 'zero_or_one1': zero_or_one1,136 'customer_net_promoter_score_response': customer_net_promoter_score_response,137 'blank_field': blank_field}138 feature_dict = line_to_feature_dictionary(input_line)139 assert sorted(feature_dict.items()) == sorted(expected_output.items())140 141def split_transcript_by_speaker(transcript):142 '''143 The transcript indicates that the speaker has changed144 with the text 'speakerN (caller/​agent):'. This is great,145 because we want to split the wall of text that is the146 transcript into lists of the pieces of conversation,147 split both by caller vs. agent as well as into coherent thoughts148 (i.e. blocks of uninterrupted speaking).149 We'll do the latter splitting here, which boils down to150 parsing through a single string, finding any instances of 'speakerN',151 and splitting the string (while removing the 'speakerN' text152 since it's not actually a part of the conversation). Since we'll153 want to split based on who the speaker is later on,154 we'll leave the (caller/​agent) part in the transcript for now.155 Arguments:156 transcript: A string that is a single call center transcript.157 Returns:158 A list of strings, one string for each time the speaker changes.159 To be safe, any empty strings ('' or None) in the list should160 be filtered out of the list.161 Hint: You could write a custom parser for the text to find instances162 of'speaker1' and 'speaker2', which is a totally acceptable way to163 solve this problem. However, if that sounds like a lot of work,164 look into using regular expressions - see165 https:/​/​docs.python.org/​2/​howto/​regex.html for details on how166 to make a regular expression as well as how to use it to167 split text.168 Test with:169 nosetests -v run_project:test_split_transcript_by_speaker170 '''171 172 a=[item for item in re.split('speaker. ', transcript)]173 return filter(None, a)174def test_split_transcript_by_speaker():175 input_transcript = 'speaker1 (caller): hello. speaker2 (agent): thank you for calling capital one.'176 expected_output = ['(caller): hello. ', '(agent): thank you for calling capital one.']177 assert expected_output == split_transcript_by_speaker(input_transcript)178def transcript_fragment_to_speaker_text_tuple(transcript_fragment):179 '''180 With the transcript broken up by speaker, the next thing we181 want to do is identify who the speaker is. This function takes182 in a single entry of the list returned by split_transcript_by_speaker183 (a 'transcript fragment') and figures out a) who the speaker was184 and b) what they said.185 Arguments:186 transcript_fragment: a string containing text spoken by a single187 party of the conversation (either the agent or the caller),188 with an identification of the party given in parentheses before189 the text. For example: '(caller): I am the person calling in. '190 Returns:191 identified_fragment: An instance of the UserSpeechFragment192 named tuple (defined at the top of this file, in the module193 variables section).194 Hint 1: You may not have encountered named tuples before. Don't195 worry! They're not too complex. You can think of them as196 regular tuples, but instead of accessing their values197 by a numerical index:198 test_tuple = ('hi',"I'm",'a','tuple')199 print test_tuple[2] -> prints 'a'200 You can instead access their values by string:201 test_namedtuple = UserSpeechFragment(role='caller',fragment_text='hello.')202 print test_namedtuple.role -> prints 'caller'203 Note that named tuples retain all the functionality of regular tuples:204 print test_namedtuple[1] -> prints 'hello.'205 For more details about named tuples, check out these links:206 http:/​/​pymotw.com/​2/​collections/​namedtuple.html207 http:/​/​sys-exit.blogspot.in/​2013/​12/​python-named-tuples.html208 Hint 2: Like for split_transcript_by_speaker, while you could write209 a full text parser for this specific purpose yourself, a regular210 expression is really the proper tool for this job. The needed211 regular expression is much more complicated than in212 split_transcript_by_speaker, however, so we'll give it to you:213 regex_string = r'[(](\w+)[)]:\s(.*)'214 However, we're not going to tell you exactly how to use it215 (that would spoil the fun!). Take a close look at the Python216 regular expression documentation217 (https:/​/​docs.python.org/​2/​howto/​regex.html), especially the218 section on 'Grouping'.219 220 Test with:221 nosetests -v run_project:test_transcript_fragment_to_speaker_text_tuple222 '''223 UserSpeechFragment = collections.namedtuple('UserSpeechFragment', ['role', 'fragment_text']) 224 aa=[] 225 for l in transcript_fragment:226 #aa=filter(None,re.split(r'[(](\w+)[)]:\s(.*)',l)) 227 if len(filter(None,re.split(r'[(](\w+)[)]:\s(.*)',l)))>1:228 aa=aa+ [UserSpeechFragment(role=filter(None,re.split(r'[(](\w+)[)]:\s(.*)',l))[0], fragment_text=filter(None,re.split(r'[(](\w+)[)]:\s(.*)',l))[1].strip())]229 return aa230 231def test_transcript_fragment_to_speaker_text_tuple():232 transcript_fragment = '({0:s}): hello. '.format(CALLER)233 expected_output = (CALLER, 'hello.')234 assert expected_output == transcript_fragment_to_speaker_text_tuple(transcript_fragment)235def consolidate_speaker_runs(assigned_transcripts):236 '''237 Take a list of transcript fragments (each one a UserSpeechFragment)238 and merge any occurances where the text transcription software has split239 up a single block of speech into multiple blocks. For instance,240 if the transcript reads 'speaker1 (agent): Thank you for. speaker1 (agent): Calling241 Capital One. speaker2 (caller): You're welcome.' then our processing pipeline will produce a list of UserSpeechFragments like this:242 [UserSpeechFragment(role='agent',fragment_text='thank you for.'),243 UserSpeechFragment(role='agent',fragment_text='calling capital one.'),244 UserSpeechFragment(role='caller',fragment_text='you're welcome.')245 ]246 When we'd really like:247 [UserSpeechFragment(role='agent',fragment_text='thank you for. calling capital one.'),248 UserSpeechFragment(role='caller',fragment_text='you're welcome.')249 ]250 (The messed up punctuation is annoying, but not something to worry about.)251 Arguments:252 assigned_transcripts: a list of UserSpeechFragments to be consolidated253 Returns:254 consolidated_transcripts: a list of UserSpeechFragments, identical to255 the input list *except* in that multiple consecutive fragments256 with the same role have been merged into a single fragment.257 258 Test with:259 nosetests -v run_project:test_consolidate_speaker_runs260 '''261 UserSpeechFragment = collections.namedtuple('UserSpeechFragment', ['role', 'fragment_text']) 262 dd=[]263 i=0264 while i <len(assigned_transcripts):265 d=assigned_transcripts[i].fragment_text266 while i <>len(assigned_transcripts)-1 and assigned_transcripts[i].role==assigned_transcripts[i+1].role:267 d=d+' ' + assigned_transcripts[i+1].fragment_text268 i=i+1 269 dd=dd+[UserSpeechFragment(assigned_transcripts[i].role, d)]270 i=i+1271 return dd272def test_consolidate_speaker_runs():273 input_collection = [UserSpeechFragment(CALLER, 'hello'),274 UserSpeechFragment(CALLER, 'world'),275 UserSpeechFragment(AGENT, 'thank you for calling capital one.'),276 UserSpeechFragment(CALLER, "You're welcome.")]277 input_collection_copy = copy.deepcopy(input_collection)278 expected_output = [UserSpeechFragment(CALLER, 'hello world'),279 UserSpeechFragment(AGENT, 'thank you for calling capital one.'),280 UserSpeechFragment(CALLER, "You're welcome.")]281 assert expected_output == consolidate_speaker_runs(input_collection)282 assert input_collection == input_collection_copy #see below if you fail283 # https:/​/​github.kdc.capitalone.com/​pages/​BigDataAcademy/​Python_Foundations_I/​modules/​Datatypes.html#mutability284def transcript_to_dialogue_collection(transcript):285 '''286 This function acts as a wrapper around the 'split_transcript_by_speaker',287 'transcript_fragment_to_speaker_text_tuple', and288 'consolidate_speaker_runs' functions. Starting from a transcript,289 do the following:290 1. Split the transcript into a list of strings, one for each time the291 transcription software identifies a speaker.292 2. For each string in the list, convert the raw string into a293 UserSpeechFragment named tuple.294 3. Remove any list elements that are None (i.e. places where295 the conversion to UserSpeechFragment failed).296 4. Consolidate the filtered list of UserSpeechFragments for297 instances where the same speaker spoke twice (or more) times298 in a row299 5. Return this consolidated list of UserSpeechFragments.300 Arguments:301 transcript: A string representing a full call transcript (the302 same as you would input to 'transcript_fragment_to_speaker_text_tuple')303 Returns:304 transcript_text_collection: A list of UserSpeechFragments, generated305 as listed in the steps above.306 307 Test with:308 nosetests -v run_project:test_transcript_to_dialogue_collection309 '''310 transcript_fragment=split_transcript_by_speaker(transcript)311 assigned_transcripts=transcript_fragment_to_speaker_text_tuple(transcript_fragment)312 return consolidate_speaker_runs(assigned_transcripts)313def test_transcript_to_dialogue_collection():314 transcript = 'speaker1 ({0:s}): Hello. speaker1 ({0:s}): World. speaker2 ({1:s}): Thank you for calling Capital One.'.format(CALLER,AGENT)315 expected_output = [(CALLER, 'Hello. World.'), (AGENT, 'Thank you for calling Capital One.')]316 assert expected_output == transcript_to_dialogue_collection(transcript)317def dialogue_collection_pretty_print(dialogue_collection):318 '''319 This is just a helper function to make things print nicer.320 Don't worry about it :)321 '''322 lines = map(lambda t: '\t{}\t: {}'.format(*t), dialogue_collection)323 return '\n'.join(lines)324# a print_debugging_function325def print_out_munging_results():326 ''' take a small sample, the first few records in the transcript file327 perform the row-ification and dialogue collection transformation328 then print each fragment in each dialogue329 330 run with:331 python -c 'import run_project;run_project.print_out_munging_results()'332 '''333 lines = get_lines_from_transcript_file()334 lines = itertools.islice(lines, 10**3)335 for line in lines:336 row = line_to_feature_dictionary(line)337 dialogue_collection = transcript_to_dialogue_collection(row['transcript'])338 for fragment in dialogue_collection:339 print '\t', fragment340 print341def quiz_printed_value_of_logging_dot_ERROR():342 '''343 Open the python interpreter and import344 the logging module. Then run print logging.ERROR.345 Make this function return what the interpreter prints346 to the screen.347 test with:348 nosetests -v run_project:test_quiz_printed_value_of_logging_dot_ERROR349 '''350 return 40351def test_quiz_printed_value_of_logging_dot_ERROR():352 assert 'af91525e3568e15041ccfa1a7e58f2a8eac837b3c89f400e3f8aea0a' == hashlib.sha224(str(quiz_printed_value_of_logging_dot_ERROR())).hexdigest()353def dialogue_collection_to_string(dialogue_collection, role_filter=None):354 '''355 Convert a list of UserSpeechFragments back into a single string,356 with the fragments separated by spaces. The default behavior of357 this function is designed to take the fragmented transcript and358 just return the whole thing as a single string - almost like undoing359 all the work that we did in fragmenting the data, **EXCEPT THAT THE360 TEXT RELATING TO SPEAKER IDENTIFICATION IS GONE**. This is pretty361 important, since that text will only serve to make any kind of analysis362 we do worse - it's not part of the actual conversation, after all.363 But wait, there's more! When the role_filter argument is not set364 to None, this function will actually return a single string for365 *just* the person defined by the role_filter argument. So, for366 example, if you set role_filter='caller', you'll get back a367 single string which only has the text from the caller merged together.368 Arguments:369 dialogue_collection: A list of UserSpeechFragments, such as that370 returned by consolidate_speaker_runs.371 role_filter (default=None): if not None, tells the function to372 only return text spoken by the person in that role (e.g.373 'caller' or 'agent').374 Returns:375 collection_string: A single string representing the merged376 UserSpeechFragments, with each fragment separated by a space (' ').377 Hint: Google "join list of strings python" for help on merging the378 fragments together.379 380 Test with:381 nosetests -v run_project:test_dialogue_collection_to_string382 '''383 text=''384 if role_filter==None:385 for i in range(len(dialogue_collection)):386 text=text + dialogue_collection[i].fragment_text + ' '387 else:388 for i in range(len(dialogue_collection)):389 if dialogue_collection[i].role==role_filter:390 text=text + dialogue_collection[i].fragment_text + ' '391 return text.strip()392 393def get_agent_transcript(dialogue_collection):394 ''' syntactic sugar on top of dialogue_collection_to_string '''395 return dialogue_collection_to_string(dialogue_collection, role_filter=AGENT)396def get_caller_transcript(dialogue_collection):397 ''' syntactic sugar on top of dialogue_collection_to_string '''398 return dialogue_collection_to_string(dialogue_collection, role_filter=CALLER)399 400class test_dialogue_collection_to_string(object):401 def setup(self):402 self.fragment_collection = [UserSpeechFragment(AGENT, 'hello'),403 UserSpeechFragment(CALLER, 'hi'),404 UserSpeechFragment(AGENT, 'how is it going?'),405 UserSpeechFragment(CALLER, 'great!')]406 def is_string(self,obj):407 try:408 obj + ''409 return True410 except TypeError:411 return False412 def test_dialogue_collection_to_string_norole(self):413 expected_output = 'hello hi how is it going? great!'414 received_output = '[NOT A STRING]'415 if self.is_string(dialogue_collection_to_string(self.fragment_collection)):416 received_output = dialogue_collection_to_string(self.fragment_collection)417 assert expected_output == dialogue_collection_to_string(self.fragment_collection),\418 "Expected '{0:s}' from [{1:s}]; Received '{2:s}'".format(expected_output,419 ','.join(["UserSpeechFragment('"+f.role+"', '"+f.fragment_text+"')" for f in self.fragment_collection]),420 received_output)421 def test_dialogue_collection_to_string_agent(self):422 expected_output = 'hello how is it going?'423 received_output = '[NOT A STRING]'424 if self.is_string(get_agent_transcript(self.fragment_collection)):425 received_output = get_agent_transcript(self.fragment_collection)426 assert expected_output == get_agent_transcript(self.fragment_collection),\427 "Expected '{0:s}' from [{1:s}]; Received '{2:s}'".format(expected_output,428 ','.join(["UserSpeechFragment('"+f.role+"', '"+f.fragment_text+"')" for f in self.fragment_collection]),429 received_output)430 def test_dialogue_collection_to_string_caller(self):431 expected_output = 'hi great!'432 received_output = '[NOT A STRING]'433 if self.is_string(get_caller_transcript(self.fragment_collection)):434 received_output = get_caller_transcript(self.fragment_collection)435 assert expected_output == get_caller_transcript(self.fragment_collection),\436 "Expected '{0:s}' from [{1:s}]; Received '{2:s}'".format(expected_output,437 ','.join(["UserSpeechFragment('"+f.role+"', '"+f.fragment_text+"')" for f in self.fragment_collection]),438 received_output)439 440# a print_debugging_function441def print_out_filtration_results():442 ''' take a small sample, the first few records in the transcript file443 perform the row-ification and dialogue collection transformation444 then print each fragment in each dialogue445 446 run with:447 python -c 'import run_project;run_project.print_out_filtration_results()'448 '''449 lines = get_lines_from_transcript_file()450 lines = itertools.islice(lines, 10**3)451 for line in lines:452 row = line_to_feature_dictionary(line)453 dialogue_collection = transcript_to_dialogue_collection(row['transcript'])454 print '\t', dialogue_collection_to_string(dialogue_collection)455 print '\t', get_agent_transcript(dialogue_collection)456 print '\t', get_caller_transcript(dialogue_collection)457 print458def create_stopwords_set():459 ''' pulls stopwords from NLTK, a given name dataset and a last name dataset '''460 def get_words_from_file(file_path):461 ''' assuming 1 word per line, this function retrieves462 a list of words from a a file in utf-8 encoding '''463 with open(file_path) as fin:464 return [unicode(word.lower().strip(), 'utf-8') for word in fin]465 nltk_stopwords = nltk.corpus.stopwords.words('english')466 firstnames = get_words_from_file(settings.GIVEN_NAMES_FILE_PATH)467 lastnames = get_words_from_file(settings.FAMILY_NAMES_FILE_PATH)468 return set(nltk_stopwords + firstnames + lastnames)469class StopWords(object):470 ''' don't worry about how this works for now.471 if you must know, stopwords is a static/​class variable472 that is lazy instantiated. Retrieve the stopwords by calling473 StopWords.get_stopwords() '''474 stopwords = None475 @classmethod476 def get_stopwords(cls):477 if not cls.stopwords:478 cls.stopwords = create_stopwords_set()479 return cls.stopwords480def remove_stopwords_from_token_collection(tokens):481 '''482 If given a list of tokens, filter out any that are stopwords.483 Generally you can think of a token as an individual word, although484 if you get at all deeper into Natural Language Processing you'll485 find that usually the tokens are actually pieces of words with things486 like suffixes stripped off.487 Arguments:488 tokens: A list of individual tokens (as strings) to have stopwords489 filtered out.490 Returns:491 filtered_tokens: A list of the input tokens, minus the stopwords.492 Hint: You'll need to first get the list of stopwords, which you can do493 by calling StopWords.get_stopwords() and assigning the output of494 that to a variable. Then for each token in your list you'll need to495 check and see if it's in that list of stopwords.496 497 Test with:498 nosetests -v run_project:test_remove_stopwords_from_token_collection499 '''500 501def test_remove_stopwords_from_token_collection():502 input_tokens = 'devin is the coolest python programmer on the block'.split(' ')503 expected_output = ['coolest', 'python', 'programmer']504 assert expected_output == remove_stopwords_from_token_collection(input_tokens)505def remove_non_alphabetic_characters(dirty_string):506 '''507 Parse through a string and remove any characters that508 are not alphabetic (i.e. not a-z).509 Arguments:510 dirty_string: A string which may (or may not!) have non-alphabetic511 characters.512 Returns:513 clean_string: A version of dirty_string with all non-alphabetic514 characters removed. If all of dirty_string's characters were515 non-alphabetic, return an empty string ('').516 Hint: You can parse a string character-by-character, just as you would a517 list or array. You can also append characters to a string with the518 plus sign (+).519 520 Test with:521 nosetests -v run_project:test_remove_non_alphabetic_characters522 '''523 524def test_remove_non_alphabetic_characters():525 input_string = 'abc 123 easy as hello, world 1 ! \xef\xbb\xbf \t \n'526 expected_output = 'abceasyashelloworld'527 assert expected_output == remove_non_alphabetic_characters(input_string)528def remove_short_tokens_from_collection(tokens, min_length=4):529 '''530 Go through a list of tokens and filter out any short ones.531 Arguments:532 tokens: A list of strings, where each string is a token.533 min_length (4): A keyword argument allowing the minimum534 length to filter on to be changed.535 536 Returns:537 filtered_tokens: A list of all the strings in 'tokens' with538 >= min_length characters.539 Hint: If you're clever, you can do this in as little as one line :).540 541 Test with:542 nosetests -v run_project:test_remove_short_tokens_from_collection543 '''544def test_remove_short_tokens_from_collection():545 input_tokens = 'andrew is the coolest python programmer on the block'.split(' ')546 expected_output = ['andrew', 'coolest', 'python', 'programmer']547 assert expected_output == remove_short_tokens_from_collection(input_tokens, min_length=6)548def tokenize_and_clean_transcript(transcript, min_length=4):549 '''550 This function acts as a wrapper to several functions:551 1. Because we want to treat "capital one" as a single thing,552 but the pipeline splits everything up into individual words,553 we replace any instances of "capital one" with a single word.554 2. We then use the nltk function 'word_tokenize' to split the transcript555 into a list of individual words (the tokens).556 3. We remove any punctuation, numbers, and any other oddness by applying557 the 'remove_non_alphabetic_characters' function to each token. For more558 details about the 'map' function used here, see:559 https:/​/​docs.python.org/​2/​howto/​functional.html#built-in-functions.560 4. Remove any stopwords from the list of tokens with the561 'remove_stopwords_from_token_collection' function.562 5. Finally, remove any remaining short tokens with the563 'remove_short_tokens_from_collection_function'.564 Test, just to make sure all the pieces are working, with:565 nosetests -v run_project:test_tokenize_and_clean_transcript566 '''567 # custom entity parsing568 transcript = transcript.replace('capital one', 'capitalone')569 # tokenize570 transcript_tokenized = nltk.word_tokenize(transcript)571 # clean up punctuation and remove empty tokens572 no_punk = map(remove_non_alphabetic_characters, transcript_tokenized)573 #Remove stopwords:574 no_stopwords = remove_stopwords_from_token_collection(no_punk)575 576 # remove short words and empty tokens577 no_shorts = remove_short_tokens_from_collection(no_stopwords, min_length=min_length)578 return no_shorts579def test_tokenize_and_clean_transcript():580 input_string = 'Macy is the coolest, most fabulous pooch on the playground'.lower()581 expected_output = ['coolest', 'fabulous', 'playground']582 assert expected_output == tokenize_and_clean_transcript(input_string, min_length=3)583# a print_debugging_function584def print_out_cleaning_results():585 ''' sanity checking for the text processing pipeline586 run with:587 python -c 'import run_project;run_project.print_out_cleaning_results()' > ~/​Desktop/​results.txt588 '''589 lines = get_lines_from_transcript_file()590 lines = itertools.islice(lines, 10**2)591 for line in lines:592 row = line_to_feature_dictionary(line)593 dialogue_collection = transcript_to_dialogue_collection(row['transcript'])594 transcript = dialogue_collection_to_string(dialogue_collection)595 vectorizable_tokens = tokenize_and_clean_transcript(transcript)596 print transcript597 print tokens_clean598 print vectorizable_tokens599 print600# a print_debugging_function601def print_out_cleaning_results():602 ''' run with:603 python -c 'import run_project;run_project.print_out_cleaning_results()' > ~/​Desktop/​results.txt604 '''605 lines = get_lines_from_transcript_file(INPUT_FILE_PATH)606 lines = itertools.islice(lines, 10**3)607 for line in lines:608 row = line_to_feature_dictionary(line)609 dialogue_collection = transcript_to_dialogue_collection(row['transcript'])610 transcript = dialogue_collection_to_string(dialogue_collection)611 vectorizable_tokens = tokenize_and_clean_transcript(transcript)612 print dialogue_collection_pretty_print(dialogue_collection)613 print 614def get_transcripts_collection():615 '''616 Here's the payout for all your hard work - the one-stop-shop617 text cleaning function! As a reward for your efforts we've618 gone ahead and filled it in for you :). Here's what it does:619 1. Load the transcript file.620 2. Iterate through the file:621 2a. Parse the line into a dictionary of features.622 2b. Convert the transcript part of the line into a collection of623 UserSpeechFragment tuples for each uninterruped string of words624 by a single speaker.625 2c. Merge the list of UserSpeechFragments back into a single string.626 2d. Convert that string into a list of individual tokens (words), remove627 punctuation, strip out stopwords, and delete small words.628 2e. Rejoin the cleaned tokens back into a single string, add to a list.629 Note that the function returns both the fully processed transcripts as well630 as the raw ones - this will be valuable later, when we want to check the results631 of our modeling (the processed transcripts are even more unintelligible than632 the originals, if you can believe it). 633 '''634 transcript_collection = []635 raw_transcript_collection = []636 lines = get_lines_from_transcript_file(INPUT_FILE_PATH)637 for line in lines:638 row = line_to_feature_dictionary(line)639 dialogue_collection = transcript_to_dialogue_collection(row['transcript'])640 raw_transcript_collection.append(row['transcript'])641 transcript = dialogue_collection_to_string(dialogue_collection)642 vectorizable_tokens = tokenize_and_clean_transcript(transcript)643 transcript_collection.append(' '.join(vectorizable_tokens))644 return raw_transcript_collection, transcript_collection645def vectorize_transcript_collection(transcript_collection):646 '''647 To convert our transcripts into features, we use a process648 called vectorization, which is the standard way to convert649 raw text into useful features for machine learning algorithms650 (see http:/​/​scikit-learn.org/​stable/​modules/​feature_extraction.html#the-bag-of-words-representation651 for some more info about vectorization and its application in652 text analysis).653 Specifically we'll be using the TfidfVectorizor, which will weight654 the transcripts by how uncommon the words they use are - the less a word655 appears the more likely it is to be important for classifying/​understanding656 a transcript. More information on tf-idf weighting can be found at the657 above link as well as at http:/​/​en.wikipedia.org/​wiki/​Tf%E2%80%93idf).658 Initialize the TfidfVectorizer with the following parameters:659 sublinear_tf=True660 max_df=0.5661 max_features=100662 stop_words='english'663 Then train the vectorizer on the transcript_collection via the '.fit' method664 of the vectorizer. For details about the TfidfVectorizer, see the documentation:665 http:/​/​scikit-learn.org/​stable/​modules/​generated/​sklearn.feature_extraction.text.TfidfVectorizer.html666 Arguments:667 transcript_collection: The list of call transcript strings you want668 to vectorize.669 Returns:670 vectorizer: The output of the fit operation671 on the transcript_collection.672 Hint: As you'll see at the top of this file,673 the TfidfVectorizer has already been imported674 for you. From this point initializing an instance of the675 vectorizer and running 'fit' should be one line676 of code each.677 678 Test with:679 nosetests -v run_project:test_vectorize_transcript_collection680 '''681class test_vectorize_transcript_collection(object):682 @classmethod683 def setUpClass(self):684 input_collection = ['hello my name is andrew',685 'hello my name is devin',686 'hi my name is chris',687 'hi im lia']688 self.vectorizer = vectorize_transcript_collection(input_collection)689 self.correct_params = {'analyzer': u'word',690 'binary': False,691 'charset': None,692 'charset_error': None,693 'decode_error': u'strict',694 'dtype': np.int64,695 'encoding': u'utf-8',696 'input': u'content',697 'lowercase': True,698 'max_df': 0.5,699 'max_features': 100,700 'min_df': 1,701 'ngram_range': (1, 1),702 'norm': u'l2',703 'preprocessor': None,704 'smooth_idf': True,705 'stop_words': 'english',706 'strip_accents': None,707 'sublinear_tf': True,708 'token_pattern': u'(?u)\\b\\w\\w+\\b',709 'tokenizer': None,710 'use_idf': True,711 'vocabulary': None}712 self.output_params = {'analyzer': u'word',713 'binary': False,714 'charset': None,715 'charset_error': None,716 'decode_error': u'strict',717 'dtype': np.int64,718 'encoding': u'utf-8',719 'input': u'content',720 'lowercase': True,721 'max_df': 0.5,722 'max_features': 100,723 'min_df': 1,724 'ngram_range': (1, 1),725 'norm': u'l2',726 'preprocessor': None,727 'smooth_idf': True,728 'stop_words': 'english',729 'strip_accents': None,730 'sublinear_tf': True,731 'token_pattern': u'(?u)\\b\\w\\w+\\b',732 'tokenizer': None,733 'use_idf': True,734 'vocabulary': None}735 if hasattr(self.vectorizer,'get_params'):736 self.output_params = self.vectorizer.get_params()737 def test_vectorizer_fits(self):738 assert hasattr(self.vectorizer,'vocabulary_') == True, 'vectorize_transcript_collection does not return '\739 'a model that has been fit to data.'740 def test_vectorizer_params_exist(self):741 assert hasattr(self.vectorizer,'get_params') == True, "vectorize_transcript_collection doesn't "\742 'initialize the TfidfVectorizer.'743 def test_vectorizer_params(self):744 for key in self.correct_params.keys():745 yield self.check_correct_vectorizer_params, key746 def check_correct_vectorizer_params(self, key):747 assert self.correct_params[key] == self.output_params[key],'Incorrect value for '\748 '{0} parameter. Expected {1}. Received {2}.'.format(key, self.correct_params[key],749 self.output_params[key])750def run_k_means(text_features):751 '''752 Initialize and fit a K-Means clustering algorithm to the753 vectorized transcripts. This will take two steps:754 1. Initialize the K-Means algorithm. We'll use the755 scikit-learn MiniBatchKMeans algorithm (imported756 at the top of this file) rather than a regular757 K-Means algorithm because it is faster without758 sacrificing much accuracy (for a description and comparison,759 see http:/​/​scikit-learn.org/​stable/​modules/​clustering.html#mini-batch-kmeans).760 Use the following parameters in the initialization:761 init='k-means++'762 n_clusters=10763 n_init=10764 init_size=1000765 batch_size=1000766 verbose=1767 2. Use the .fit function of your initialized MiniBatchKMeans to cluster the data.768 Arguments:769 text_features: The vectorized transcripts, from 'vectorize_transcript_collection'770 Returns:771 fit_kmeans: The MiniBatchKMeans object, initialized with the parameters above and772 fit to the text_features.773 Hint: Scikit-Learn uses classes extensively to provide persistence in its models.774 This is really useful, since not only do you want to train your model but you775 of course want to test it out, and then ultimately may end up running it over776 and over on different data. It does make things a little complicated to learn,777 however. If you're having difficulty, take a look at the tutorial778 (http:/​/​scikit-learn.org/​stable/​tutorial/​basic/​tutorial.html) and the examples779 (http:/​/​scikit-learn.org/​stable/​auto_examples/​index.html).780 781 Test with:782 nosetests -v run_project:test_run_k_means783 784 '''785class test_run_k_means(object):786 @classmethod787 def setUpClass(self):788 data = np.array([0.78528828, 0.6191303, 0.78528828, 0.6191303 , 0.78528828,789 0.6191303, 0.61761437, 0.61761437, 0.48693426, 0.4869, 0.4869, 0.4869,790 0.25, 0.34, 0.5])#15791 row = np.array([0, 0, 1, 1, 2, 2, 3, 3, 3, 3, 4, 4, 8, 9, 10])792 col = np.array([0, 3, 2, 3, 1, 4, 6, 5, 4, 1, 4, 0, 6, 3, 4])793 from scipy.sparse import csr_matrix794 self.vectorized_data = csr_matrix((data,(row,col)))795 self.fit_kmeans = run_k_means(self.vectorized_data)796 self.correct_params = {'batch_size':1000,797 'compute_labels':True,798 'init':'k-means++',799 'init_size':1000,800 'max_iter':100,801 'max_no_improvement':10,802 'n_clusters':10,803 'n_init':10,804 'random_state':None,805 'reassignment_ratio':0.01,806 'tol':0.0,807 'verbose':1808 }809 self.output_params = {'batch_size':1000,810 'compute_labels':True,811 'init':'k-means++',812 'init_size':1000,813 'max_iter':100,814 'max_no_improvement':10,815 'n_clusters':10,816 'n_init':10,817 'random_state':None,818 'reassignment_ratio':0.01,819 'tol':0.0,820 'verbose':1821 }822 if hasattr(self.fit_kmeans,'get_params'):823 self.output_params = self.fit_kmeans.get_params()824 def test_kmeans_fits(self):825 assert hasattr(self.fit_kmeans,'labels_') == True, 'run_k_means does not return '\826 'a model that has been fit to data.'827 def test_kmeans_params_exist(self):828 assert hasattr(self.fit_kmeans,'get_params') == True, "run_k_means doesn't "\829 'initialize the MiniBatchKMeans model.'830 def test_kmeans_params(self):831 for key in self.correct_params.keys():832 yield self.check_correct_kmeans_params, key833 def check_correct_kmeans_params(self, key):834 assert self.correct_params[key] == self.output_params[key],'Incorrect value for '\835 '{0} parameter. Expected {1}. Received {2}.'.format(key, self.correct_params[key],836 self.output_params[key])837def split_data_into_samples(transcript_collection, cluster_labels, test_frac=0.2):838 '''839 We don't want to train a predictive model on our entire labeled840 sample - if we do that we won't be able to accurately gauge841 its performance. So this function splits the data and labels842 into two groups: the training set (the data the model will be843 fit on) and the test set (the data we'll use to estimate how844 good the model is). The important thing is to make this split happen845 RANDOMLY in the data - if your data has any kind of ordering to it,846 you'll bias your model if you do any kind of non-random sampling.847 Arguments:848 transcript_collection: A list of strings, one for each transcript.849 cluster_labels: A list of labels for each of those transcripts, from k_means.labels_850 test_frac (0.2): What fraction of the data should be used for testing851 (20% is a good default).852 Returns:853 train_transcripts, test_transcripts, train_labels, test_labels: A tuple854 of lists, giving (in order) the training transcripts, the testing transcripts,855 the training labels, and the testing labels.856 Hint: You can do this in a few different ways, but scikit-learn has857 a function called 'train_test_split' (imported at the top of this file)858 that's designed to do exactly this.859 Test with:860 nosetests -v run_project:test_split_data_into_samples861 862 '''863class test_split_data_into_samples(object):864 def setup(self):865 self.transcripts = ['a','b','c','d','e','f','g','h','i','j']866 self.labels = [0,0,1,0,0,1,2,1,2,0]867 def test_split_data_into_samples_output_format(self):868 output = split_data_into_samples(self.transcripts,self.labels)869 assert hasattr(output,'__iter__') and len(output) == 4, "output format of"\870 " split_data_into_samples is wrong, needs to be a 4-element tuple"871 def test_split_data_into_samples_random(self):872 train_t1, test_t1, train_l1, test_l1 = split_data_into_samples(self.transcripts,self.labels)873 train_t2, test_t2, train_l2, test_l2 = split_data_into_samples(self.transcripts,self.labels)874 assert np.array_equal(train_t1,train_t2) == False\875 and np.array_equal(train_l1,train_l2) == False,\876 "The data are not being sufficiently randomized."877 def test_split_data_into_samples_same_size(self):878 train_t1, test_t1, train_l1, test_l1 = split_data_into_samples(self.transcripts,self.labels)879 train_t2, test_t2, train_l2, test_l2 = split_data_into_samples(self.transcripts,self.labels)880 assert len(train_t1) == len(train_t2)\881 and len(train_l1) == len(train_l2),\882 "The data are not being split into consistent training and testing subsamples."883 def test_split_data_into_samples_proper_split(self):884 train_t1, test_t1, train_l1, test_l1 = split_data_into_samples(self.transcripts,self.labels,885 test_frac=0.5)886 assert len(train_t1) == len(test_t1),\887 "The data are not being split appropriately for test_frac=0.5."888def train_logistic_regression(vectorized_training_transcripts,training_labels):889 '''890 Initialize and train a multiclass logistic regression classifier on the891 training data, using scikit-learn's LogisticRegression class with default892 parameters.893 Arguments:894 vectorized_training_transcripts: The fully processed and vectorized call895 center transcripts that make up the training set.896 training_labels: The k-means clustering labels assigned to the training897 set transcripts.898 Returns:899 fit_logistic: An instance of scikit-learn's LogisticRegression class,900 initialized with default parameters and fit to the training data.901 Hint: You can look up documentation and examples for the LogisticRegression902 class by Googling 'scikit-learn logistic regression'. You'll notice,903 however, that using it is virtually identical to how we used904 scikit-learn's k-means clustering. This type of scheme is applied905 consistently across scikit-learn's entire suite of predictive906 models, which makes it really easy to swap between models when907 you're doing exploratory analysis.908 Test with:909 nosetests -v run_project:test_train_logistic_regression910 '''911 912class test_train_logistic_regression(object):913 @classmethod914 def setup(self):915 self.transcripts = np.random.rand(20,20)916 self.labels = np.random.randint(0,2,20)917 self.fit_logreg = train_logistic_regression(self.transcripts,self.labels)918 #print dir(self.fit_logreg)919 self.correct_params = {'C': 1.0,920 'class_weight': None,921 'dual': False,922 'fit_intercept': True,923 'intercept_scaling': 1,924 'penalty': 'l2',925 'random_state': None,926 'tol': 0.0001}927 self.output_params = {'C': 1.0,928 'class_weight': None,929 'dual': False,930 'fit_intercept': True,931 'intercept_scaling': 1,932 'penalty': 'l2',933 'random_state': None,934 'tol': 0.0001}935 def test_train_logistic_regression_fits(self):936 assert hasattr(self.fit_logreg,'coef_') == True, 'train_logistic_regression '\937 'does not return a model that has been fit to data.'938 def test_train_logistic_regression_params_exist(self):939 assert hasattr(self.fit_logreg,'get_params') == True, "train_logistic_regression doesn't "\940 'initialize the LogisticRegression model.'941 def test_train_logistic_regression_params(self):942 for key in self.correct_params.keys():943 yield self.check_correct_logreg_params, key944 def check_correct_logreg_params(self, key):945 assert self.correct_params[key] == self.output_params[key],'Incorrect value for '\946 '{0} parameter. Expected {1}. Received {2}.'.format(key, self.correct_params[key],947 self.output_params[key])948 949def print_cluster_examples(transcripts,labels,num_examples=10,file_extension='.txt'):950 '''951 Print a small number of examples of transcripts from each cluster to files,952 for by-eye inspection.953 Arguments:954 transcripts: A list of strings, one for each transcript.955 labels: The labels assigned to the transcripts by the clustering algorithm.956 num_examples (10): The number of transcripts to write for each cluster.957 file_extension (.txt): The extension to append to the file. 958 '''959 unique_labels = np.unique(labels)960 np_transcripts = np.array(transcripts)961 for i,label in enumerate(unique_labels):962 transcripts_with_label = np_transcripts[labels == label]963 output_filename = 'cluster_{0}_examples{1:s}'.format(label,file_extension)964 np.savetxt(output_filename,transcripts_with_label[:num_examples],fmt='%s')965 966def vectorize_and_model(raw_transcripts, transcript_collection):967 ''' run with:968 python -c 'import run_project;run_project.vectorize_and_model()'969 '''970 text_vectorizer = vectorize_transcript_collection(transcript_collection)971 text_features = text_vectorizer.transform(transcript_collection)972 # matrix logging973 logging.info('X - n_samples: {0}, n_features: {1}'.format(*text_features.shape))974 #Use clustering to make labels for the transcripts975 k_means = run_k_means(text_features)976 k_means_labels = k_means.labels_977 #print cluster information to the log and save sample transcripts978 logging.info('clusters and cluster frequency: {}'.format(collections.Counter(k_means_labels)))979 print_cluster_examples(raw_transcripts, k_means_labels, num_examples=5, file_extension='_temp.nogit')980 # we started with unlabeled data, and now we've labeled it using an981 # unsupervised machine learning algorithm. Let's see if we can reconstruct 982 # those classifications with a supervised algorithm.983 # but we don't want to train and test on the same data, so let's construct984 # a supervised modeling pipeline with a train/​test split:985 # http:/​/​scikit-learn.org/​stable/​modules/​pipeline.html986 # http:/​/​scikit-learn.org/​stable/​tutorial/​text_analytics/​working_with_text_data.html#building-a-pipeline987 # http:/​/​stats.stackexchange.com/​questions/​95797/​how-to-split-the-dataset-for-cross-validation-learning-curve-and-final-evaluat988 #Split the data into training and test sets:989 transcript_train, transcript_test, label_train, label_test = split_data_into_samples(transcript_collection,k_means_labels)990 #Vectorize the train and test data:991 vectorized_transcript_train = text_vectorizer.transform(transcript_train)992 vectorized_transcript_test = text_vectorizer.transform(transcript_test)993 #Initialize the Logistic Regression:994 logreg = LogisticRegression()995 logreg.fit(vectorized_transcript_train,label_train)996 predicted = logreg.predict(vectorized_transcript_test)997 accuracy = np.mean(predicted == label_test)998 print "Done: accuracy = {0:.2f} on {1:d}"\999 " testing transcripts ({2:d} transcripts used for training)".format(accuracy,len(label_test),len(label_train))1000 1001 # text_clf_trained = text_clf_pipeline.fit(X_train, y_train)1002 # predicted = text_clf_trained.predict(X_test)1003 1004 # y_test_category_match_ratio = np.mean(predicted == y_test)1005 # logging.info('all_data_category_match_ratio: {}'.format(y_test_category_match_ratio))1006 # CONGRATULATIONS!1007 # You just built a real natural language processing pipeline1008 # that can automatically categorize new call center transcripts into auto-discovered topic1009 # buckets. Pretty gosh-darn cool if you ask me.1010 # Where do you go from here?1011 # interesting question... try modifying the transcript fetching function to1012 # pull only agent or caller transcripts and re-running the pipeline1013 #1014 # There are lots of perspectives you could take at this point:1015 # business would want to to understand the clustered data and what the clusters mean1016 # statisticians would want to validate the model and try different clustering and classification algorithms1017 # developers would want to know how the algorithms scale and if stream processing is possible1018 # MRO would want validation and monitoring1019 #1020 # If you don't see your use case or you have strong feelings about a particular topic,1021 # let me know and I'll do my best to implement your request - let me know here:1022 # https:/​/​github.kdc.capitalone.com/​BigDataAcademy/​Small_Data/​issues?state=open1023 #1024def main():1025 ''' '''1026 print "Starting"1027 raw_transcripts, transcript_collection = get_transcripts_collection()1028 print "Finished Collecting Transcripts"1029 vectorize_and_model(raw_transcripts, transcript_collection)1030 1031if __name__ == '__main__':...

scheduler.py

Source:scheduler.py

1import paramiko2import sys3import re4import os5import json6import shutil7from jinja2 import Template8from datetime import datetime9import pymysql10sys.stdout.flush()11serverInfo = {12 'IP': {13 'user': '',14 'port': '',15 'disk': '',16 'load1': '',17 'load5': '',18 'memory': '',19 },20 'IP': {21 'user': '',22 'port': '',23 'disk': '',24 'load1': '',25 'load5': '',26 'memory': '',27 }28}29PROXYSERVER = 'IP'30DIR_PATH = os.path.split(os.path.realpath(__file__))[0]31BUILD_CONF = '%s/​build.conf' % DIR_PATH32NS_CONF = '%s/​namespace.conf' % DIR_PATH33NG_TEMPLATE = '%s/​template.ups' % DIR_PATH34NG_UP_path = '/​usr/​local/​orange/​conf/​upstreams'35environment_tab = 'facilities_environment'36applicant_tab = 'facilities_applicant'37project_tab = 'facilities_project'38TIME = datetime.now()39MYSQL_SERVER = 'IP'40# 初始化数据链接41db = pymysql.connect(host=MYSQL_SERVER, user='user', password='passwd', db='cmdb', charset='utf8',42 autocommit=True)43# 执行sql以json格式返回数据44cur = db.cursor(pymysql.cursors.DictCursor)45def select_data(table, name, option):46 """通过传入不同的表以及字段值获取对应的id并返回 返回值为id号"""47 if option == 'container':48 field = 'container_name'49 else:50 field = 'name'51 try:52 sql = "select id from %s where %s = '%s'" % (table, field, name)53 cur.execute(sql)54 data = cur.fetchall()55 print(data)56 print('数据id获取成功')57 return data[0]['id']58 except Exception as e:59 print('数据id获取失败 %s\n' % name, e)60 exit()61def check_sql_data():62 """判断容器名是否已经存在 存在返回0 不存在返回1"""63 sql = "select * from %s where container_name = '%s'" % (environment_tab, container_name)64 cur.execute(sql)65 data = cur.fetchall()66 if data:67 return 068 else:69 return 170def relation_data():71 """关联数据库数据"""72 project_id = select_data(project_tab, project, 'name')73 applicant_id = select_data(applicant_tab, build_user, 'name')74 env_id = select_data(environment_tab, container_name, 'container')75 env_applicant_sql = "insert into facilities_environment_applicant " \76 "(environment_id,applicant_id) values ('%s','%s')" % (env_id, applicant_id)77 env_project_sql = "insert into facilities_environment_project" \78 "(environment_id,project_id) values ('%s','%s')" % (env_id, project_id)79 try:80 cur.execute(env_applicant_sql)81 cur.execute(env_project_sql)82 print('数据关联成功')83 except Exception as e:84 print('数据关联失败\n', e)85 exit()86def data_insert(port):87 """数据插入与更新 如果有相同数据 那么更新即可 如果没有即插入数据"""88 data_status = check_sql_data()89 update_sql = "update %s set container_port = %s where container_name = '%s'" % \90 (environment_tab, port, container_name)91 environment_sql = "insert into %s " \92 "(env_flag,env_tag,container_name,container_port,node_ip,create_time) " \93 "values('%s','%s','%s%s%s',%s,'%s','%s')" % (environment_tab, env, tag, env, tag, project,94 port, return_ip(), datetime.now())95 if data_status == 0:96 print(data_status)97 print(update_sql)98 try:99 cur.execute(update_sql)100 print('更新数据库信息成功')101 except Exception as e:102 print('数据库信息更新失败\n%s' % e)103 exit()104 else:105 try:106 cur.execute(environment_sql)107 print('数据库插入完成')108 relation_data() # 调用数据关联函数 关联对应表数据109 except Exception as e:110 print('数据库插入失败\n%s' % e)111 exit()112def get_port(host, port, user):113 """获取应用容器端口"""114 get_port_cmd = "docker ps | awk '$NF == \"%s%s%s\" {print $(NF-1)}' | awk -F \"->\" '{print $1}' | awk -F \":\" " \115 "'{print $NF}'" % (env, tag, project)116 stout, err = connection_server(host, port, user, get_port_cmd)117 return stout.read().decode('utf-8')118def build_configuration_file():119 server_port = serverInfo[return_ip()]['port']120 server_user = serverInfo[return_ip()]['user']121 """创建nginx配置文件"""122 with open(NG_TEMPLATE, 'r') as f:123 new_conf = '%s/​%s%s/​%s%s%s.ups' % (NG_UP_path, env, tag, env, tag, project)124 upstream_name = '%s%s%s' % (env, tag, project)125 container_port = ''126 if get_port(return_ip(), server_port, server_user):127 container_port = get_port(return_ip(), server_port, server_user)128 else:129 print("容器端口获取失败")130 exit()131 with open(new_conf, 'w') as f1:132 template_data = f.read()133 upstream_data = {'env': env, 'branch': branch, 'project': project, 'user': build_user,134 'namespace': namespace, 'createtime': TIME, 'tag': tag,135 'upstraemname': upstream_name, 'host': return_ip(), 'port': container_port}136 template = Template(template_data)137 new_data = template.render(upstream_data)138 f1.write(new_data)139 data_insert(container_port) # 传入contqiner端口 并生成数据信息插入到数据库140def modify_filename(up_path):141 """批量修改公共upstream的配置文件名及upstream名称"""142 for filename in os.listdir(up_path):143 new_filename = filename.replace(filename[4:15], '%s' % tag)144 os.rename('%s/​%s' % (up_path, filename), '%s/​%s' % (up_path, new_filename))145 temp_file = '/​tmp/​temp.ups'146 old_file = '%s/​%s' % (up_path, new_filename)147 with open(old_file, 'r') as f:148 with open(temp_file, 'w') as f1:149 for i in f.readlines():150 if '%s-common' % env in i:151 i = i.replace('%s-common' % env, tag)152 f1.write(i)153 os.remove(old_file)154 os.rename(temp_file, old_file)155 with open(old_file, 'r') as f2:156 with open(temp_file, 'w') as f3:157 for i in f2.readlines():158 if 'USER' in i:159 i = i.replace(i, '#USER: %s\n' % build_user)160 f3.write(i)161 os.remove(old_file)162 os.rename(temp_file, old_file)163def judge_up_conf():164 """判断环境下的tag配置文件是否已存在 并生成不同环境下tag的配置文件及common配置rename"""165 if os.path.exists('%s/​%s%s' % (NG_UP_path, env, tag)):166 build_configuration_file()167 else:168 new_path = '%s/​%s%s' % (NG_UP_path, env, tag)169 if env == "beta":170 shutil.copytree('%s/​betabeta-common' % NG_UP_path, new_path)171 elif env == "stage":172 shutil.copytree('%s/​stagestage-common' % NG_UP_path, new_path)173 modify_filename(new_path)174 build_configuration_file()175 os.system('orange reload')176 print("代理配置文件已生成")177def create_ng_conf():178 """生成upstream配置文件"""179 if project != "wacc-core" and project != "wacc-query":180 if env == "beta" and tag != '':181 judge_up_conf()182 elif env == "stage" and tag != '':183 judge_up_conf()184 else:185 print("%s 无需代理配置" % project)186 exit()187def connection_server(host, port, user, cmd):188 client = paramiko.SSHClient()189 """登录服务器封装 执行命令 自动保存认证策略到本地know_hosts文件"""190 client.set_missing_host_key_policy(paramiko.AutoAddPolicy())191 private_key = paramiko.RSAKey.from_private_key_file('/​root/​.ssh/​id_rsa')192 client.connect(hostname=host, port=port, username=user, pkey=private_key)193 stin, stout, sterr = client.exec_command(cmd)194 return stout, sterr195def load_check(host, port, user):196 """获取服务器1分钟5分钟负载情况"""197 cmd = 'uptime'198 stout, sterr = connection_server(host, port, user, cmd)199 if sterr.read().decode('utf-8'):200 print('painc: get load average error')201 else:202 load = stout.read().decode('utf-8').split(':')[-1].split(',')203 serverInfo[host]['load1'] = load[0]204 serverInfo[host]['load5'] = load[1]205def mem_check(host, port, user):206 """获取内存余量"""207 cmd = "free -g | awk '/​^Mem/​ NR > 1 {print $NF}'"208 stout, sterr = connection_server(host, port, user, cmd)209 if sterr.read().decode('utf-8'):210 print('painc: get memoryinfo error')211 else:212 meminfo = stout.read().decode('utf-8').rstrip('\n')213 serverInfo[host]['memory'] = meminfo214def disk_check(host, port, user):215 """获取服务器磁盘可选总量 只获取home和含有data的数据盘剩余可用"""216 cmd = "df -h | awk 'NR > 1 {print $NF}'"217 stout, sterr = connection_server(host, port, user, cmd)218 if sterr.read().decode('utf-8'):219 print('painc: get diskinfo error')220 else:221 disk = stout.read().decode('utf-8').split('\n')222 datadisk = []223 for i in disk:224 if "home" in i or "data" in i:225 datadisk.append(i)226 for i in datadisk:227 getavail = "df -h %s | awk 'NR > 1 {print $4}' | sed -e 's/​[a-zA-Z]/​/​'" % i228 stout, sterr = connection_server(host, port, user, getavail)229 datadisk[datadisk.index(i)] = stout.read().decode('utf-8').rstrip('\n')230 total = 0231 for i in datadisk:232 total = total + int(i)233 serverInfo[host]['disk'] = total234def save_data(dic):235 """将获取到的服务器相关信息存放到字典中"""236 for i, v in dic.items():237 load_check(i, dic[i]['port'], dic[i]['user'])238 mem_check(i, dic[i]['port'], dic[i]['user'])239def check_tag(tag_check):240 """根据正则匹配检测输入的tag是否符合规则 字母不限制 结尾以4个数字结尾"""241 rule = re.compile('^[a-zA-z]+?[0-9]{4}$')242 if rule.match(tag_check) is not None:243 print("tag检测完毕 开始检测环境")244 else:245 print("tag输入格式不正确 请按照提示规则重新发布")246 exit()247def check_env(proxyserver, project_check):248 """发布前环境检查"""249 if project_check == "wacc-core" or project_check == "wacc-query":250 print("正在更新发布")251 else:252 cmd = "[ -f /​usr/​local/​orange/​conf/​upstreams/​%s%s/​%s%s%s.ups ] && echo 0 || echo 1" % (253 env, tag, env, tag, project_check)254 stout, err = connection_server(proxyserver, 1662, 'root', cmd)255 status = stout.read().decode('utf-8').rstrip('\n')256 if int(status) == 1:257 print('准备创建新环境 ')258 else:259 cmd = "awk 'NR==4 {print $2}' /​usr/​local/​orange/​conf/​upstreams/​%s%s/​%s%s%s.ups" % (260 env, tag, env, tag, project_check)261 user, sterr = connection_server(proxyserver, 1662, 'root', cmd)262 old_user = user.read().decode('utf-8')263 if old_user == '':264 print('painc: 获取信息失败 ')265 exit()266 else:267 if old_user.strip('\n') == build_user:268 print("环境已存在 正在更新发布")269 else:270 print("环境已存在")271 exit()272def get_config(parent, subpro, file):273 """解析每个项目的发布配置"""274 data_dic = json.load(open(file))275 return data_dic[parent][subpro]276def get_ns_conf(user):277 """获取公共或者个人的ACM配置名称空间地址"""278 if namespace == "stage-common" or namespace == "beta-common":279 return get_config(namespace, 'namespace', NS_CONF)280 else:281 return get_config('%s-%s' % (env, user), 'namespace', NS_CONF)282def conf_check(file, strings):283 """检查文件中是否含有某字符串 当前用来检查dockerfile中是否含有某配置"""284 with open(file, 'r+') as f:285 if re.search(strings, f.read()) is not None:286 return True287 else:288 return False289def conf_add(file, new_str):290 """Dockerfile添加配置"""291 line = []292 new_file = '/​data/​tmp/​add_temp'293 with open(file, 'r+') as f:294 with open(new_file, 'w') as f1:295 for i in f.readlines():296 line.append(i)297 line.insert(2, new_str)298 for i in line:299 f1.write(i)300 os.remove(file)301 os.rename(new_file, file)302def replace_conf(file, old_str, new_str):303 """更改Dockerfile配置"""304 new_file = '/​data/​tmp/​replace_temp'305 with open(file, 'r+') as f:306 with open(new_file, 'w') as f1:307 for i in f.readlines():308 if old_str in i:309 i = i.replace(i, new_str)310 f1.write(i)311 os.remove(file)312 os.rename(new_file, file)313def container_run_check():314 """检查是否已经有相同的容器在运行 如果有 那么返回机器所在的ip地址"""315 for i in serverInfo.keys():316 container_check = 'docker ps -a | grep %s%s%s' % (env, tag, project)317 stout, err = connection_server(i, serverInfo[i]['port'], serverInfo[i]['user'],318 container_check)319 if stout.read().decode('utf-8') != '':320 return i321def return_ip():322 """返回符合一分钟以内负载小于16 内存大于4G的服务器ip"""323 server_ip = ''324 if container_run_check() is not None:325 server_ip = container_run_check()326 else:327 for i in serverInfo.keys():328 if float(serverInfo[i]['load1'].strip()) < 16.0 and int(serverInfo[i]['memory'].strip()) >= 4:329 server_ip = i330 break331 return server_ip332def java_docker_cmd(host, port, user, run_project):333 """docker启动命令"""334 common_log_path = get_config(run_project, 'commonLogsPath', BUILD_CONF)335 app_port = get_config(run_project, 'port', BUILD_CONF)336 image_url = get_config(run_project, 'imageUrl', BUILD_CONF)337 zk_domain = "z01.betadubbo.zk.inf.bandubanxie.com"338 zk_ip = "172.17.1.88"339 sk_path = "/​home/​service/​skywalking/​agent/​skywalking-agent.jar"340 docker_run = ''341 catalina_args = "\"$CATALINA_OPTS -Xms1024m -Xmx1024m -XX:MaxPermSize=512m -javaagent:%s " \342 "-Ddubbo.provider.tag=%s.%s\"" % (sk_path, env, tag)343 option = get_config(run_project, 'option', BUILD_CONF)344 docker_del = 'docker ps -a | grep %s%s%s && docker stop %s%s%s && docker rm %s%s%s' % (345 env, tag, run_project, env, tag, run_project, env, tag, run_project)346 stout, err = connection_server(host, port, user, docker_del)347 if err.read().decode('utf-8'):348 print('painc: already exists container delete error %s')349 exit()350 if option == "jar":351 docker_run = 'docker pull %s/​%s:v%s && ' \352 'docker run -dit --name %s%s%s ' \353 '--add-host %s:%s -p :%s ' \354 '--restart=always ' \355 '-m 1G' \356 ' -e YSX_PROJECT_DOMAIN=\"%s\"' \357 ' -e SW_AGENT_INSTANCE_NAME=\"%s.%s\"' \358 ' -e SW_LOGGING_DIR=\"/​data/​logs/​tomcat_logs/​%s\"' \359 ' -e SK_AGENT=\"%s\"' \360 ' -e YSX_SW_AGENT_COLLECTOR_BACKEND_SERVICES=\"192.168.12.51:11800\"' \361 ' -e YSX_SW_AGENT_AUTHENTICATION=\"yunshuxie-test\"' \362 ' -v %s/​%s/​%s/​%s/​:/​data/​logs %s/​%s:v%s' % (363 image_url, run_project, build_number, env, tag, run_project,364 zk_domain, zk_ip, app_port, run_project, env, tag, run_project,365 sk_path, common_log_path, env, tag, run_project, image_url,366 run_project, build_number,)367 elif option == "war":368 docker_run = 'docker pull %s/​%s:v%s && ' \369 'docker run -dit --name %s%s%s ' \370 '--add-host %s:%s -p :%s ' \371 '--restart=always ' \372 '-m 1G' \373 ' -e YSX_PROJECT_DOMAIN=\"%s\"' \374 ' -e SW_AGENT_INSTANCE_NAME=\"%s.%s\"' \375 ' -e SW_LOGGING_DIR=\"/​data/​logs/​tomcat_logs/​%s\"' \376 ' -e SK_AGENT=\"%s\"' \377 ' -e YSX_SW_AGENT_COLLECTOR_BACKEND_SERVICES=\"192.168.12.51:11800\"' \378 ' -e YSX_SW_AGENT_AUTHENTICATION=\"yunshuxie-test\"' \379 ' -e CATALINA_OPTS=%s' \380 ' -v %s/​%s/​%s/​%s/​:/​data/​logs %s/​%s:v%s' % (381 image_url, run_project, build_number, env, tag, run_project,382 zk_domain, zk_ip, app_port, run_project, env, tag, run_project,383 sk_path, catalina_args, common_log_path, env, tag, run_project,384 image_url, run_project, build_number)385 print(docker_run)386 stout, err = connection_server(host, port, user, docker_run)387 if err.read().decode('utf-8'):388 print('painc: container run error %s')389 exit()390 create_ng_conf()391def image_build_cmd(project_img, image_url):392 """docker容器gou"""393 status = os.system('cd %s && docker build -t %s/​%s:v%s . && docker push %s/​%s:v%s' %394 (get_config(project_img, 'deployPath', BUILD_CONF),395 image_url, project_img, build_number, image_url, project_img, build_number))396 if status == 0:397 print("镜像构建完毕 容器正在启动")398def image_build(project_img):399 """构建docker镜像 并运行调用docker命令启动容器"""400 deploy_path = get_config(project_img, 'deployPath', BUILD_CONF)401 option = get_config(project_img, 'option', BUILD_CONF)402 domain = get_config(project_img, 'domain', BUILD_CONF)403 jar_file = get_config(project_img, 'jarFile', BUILD_CONF)404 image_url = get_config(project_img, 'imageUrl', BUILD_CONF)405 server_port = serverInfo[return_ip()]['port']406 server_user = serverInfo[return_ip()]['user']407 server_ip = ''408 if return_ip():409 print('retuIp == %s' % return_ip())410 server_ip = return_ip()411 else:412 print("memory or load not available error")413 exit()414 if option == "jar":415 SK_AGENT = "/​home/​service/​skywalking/​agent/​skywalking-agent.jar"416 start_cmd = '#!/​bin/​bash\njava -server -javaagent:%s ' \417 '-Xms1024m -Xmx1024m -XX:MaxPermSize=512m -jar' \418 ' -Ddubbo.provider.tag=%s.%s' \419 ' /​home/​work/​%s/​%s --spring.profiles.active=%s' % \420 (SK_AGENT, env, tag, domain, jar_file, env)421 with open('%s/​server.sh' % deploy_path, 'w') as f:422 f.write(start_cmd)423 image_build_cmd(project_img, image_url)424 java_docker_cmd(server_ip, server_port, server_user, project_img)425 else:426 image_build_cmd(project_img, image_url)427 java_docker_cmd(server_ip, server_port, server_user, project_img)428def java_project_deploy(project_dep, commonpath):429 """特殊项目构建发布 例如mall-api或者mall-admin"""430 if project_dep == "mall-api" or project_dep == "mall-admin":431 os.chdir('%s/​ysx-mall' % commonpath)432 else:433 os.chdir('%s/​%s' % (commonpath, project_dep))434 status = os.system('git pull && git checkout %s && mvn clean package -U '435 '-Dmaven.test.skip=true -P%s -Dconfig_namespace=%s && [ -f %s ]' % (436 branch, env, get_ns_conf(build_user),437 get_config(project_dep, 'resPath', BUILD_CONF)))438 if status == 0:439 print("代码更新完成 编译结束 开始构建镜像。。。")440 shutil.copyfile(get_config(project_dep, 'resPath', BUILD_CONF), '%s/​%s' % (441 get_config(project_dep, 'deployPath', BUILD_CONF),442 get_config(project_dep, 'jarFile', BUILD_CONF)))443 """调用构建镜像"""444 image_build(project_dep)445 else:446 print("代码更新失败或编译失败 已退出。。。")447 exit(status)448def node_container_run(contain_check, server, container_name, app_port, rsync_path, work_path, common_log_path):449 """nodejs项目容器运行指令"""450 print(contain_check)451 chk_out, chk_err = connection_server(server, serverInfo[server]['port'], serverInfo[server]['user'],452 contain_check)453 if chk_err.read().decode('utf-8'):454 print('painc:nodejs_container run_check error ')455 exit()456 container_run = "docker run -dit --name %s -p :%s --restart=always -v %s:%s -v %s/​%s/​%s/​%s:/​data/​logs " \457 "172.17.2.66:5000/​node:v1" % (container_name, app_port, rsync_path, work_path,458 common_log_path, env, tag, project)459 print(container_run)460 run_out, run_err = connection_server(server, serverInfo[server]['port'], serverInfo[server]['user'],461 container_run)462 if run_err.read().decode('utf-8'):463 print(run_err.read().decode('utf-8'))464 print('painc:nodejs_container run error ')465 exit()466def nodejs_project_deploy(node_project_dep, node_commonpath):467 """nodejs项目编译 容器化部署"""468 deploy_path = get_config(node_project_dep, 'deployPath', BUILD_CONF)469 os.chdir('%s/​%s' % (node_commonpath, node_project_dep))470 status = os.system('git pull && git checkout %s && npm install --unsafe-perm=true --allow-root && npm run build ' %471 branch)472 if status == 0:473 print("node代码更新完成 编译结束 开始同步数据。。。")474 server = return_ip()475 rsync_path = '%s/​%s/​%s/​%s/​%s/​%s' % (deploy_path, env, namespace, build_user, branch, tag)476 rsync_path_check = '[ -d %s ] || mkdir -p %s' % (rsync_path, rsync_path)477 stout, err = connection_server(server, serverInfo[server]['port'], serverInfo[server]['user'],478 rsync_path_check)479 if err.read().decode('utf-8'):480 print('painc: rsync_path_check error %s')481 exit()482 rsyn_sta = os.system('rsync -e "ssh -p 1662 -o StrictHostKeyChecking=no" -avz --delete %s/​%s %s:%s' %483 (node_commonpath, node_project_dep, server, rsync_path))484 if rsyn_sta == 0:485 print("数据同步完成 正在启动容器。。。")486 app_port = get_config(node_project_dep, 'port', BUILD_CONF)487 work_path = '/​home/​work/​project'488 common_log_path = get_config(node_project_dep, 'commonLogsPath', BUILD_CONF)489 contain_check = 'docker ps -a | grep %s && docker stop %s && docker rm %s' % (490 container_name, container_name, container_name)491 node_container_run(contain_check, server, container_name, app_port, rsync_path, work_path, common_log_path)492 create_ng_conf()493def java_deploy_check(project_check):494 """发布前 环境检查 例如 所需的数据目录以及日志路径等 """495 common_path = get_config(project_check, 'commonPath', BUILD_CONF)496 git_Url = get_config(project_check, 'gitUrl', BUILD_CONF)497 if project_check == "mall-api" or project_check == "mall-admin":498 if not os.path.exists('%s/​ysx-mall' % common_path):499 os.chdir(common_path)500 os.system('git clone %s' % git_Url)501 """调用部署"""502 java_project_deploy(project_check, common_path)503 else:504 if not os.path.exists('%s/​%s' % (common_path, project_check)):505 os.chdir(common_path)506 os.system('git clone %s' % git_Url)507 """调用部署"""508 java_project_deploy(project_check, common_path)509def nodejs_deploy_check(project_check):510 """发布前 环境检查 例如 所需的数据目录以及日志路径等 关于nodejs所有函数都是与java分离的"""511 common_path = get_config(project_check, 'commonPath', BUILD_CONF)512 git_Url = get_config(project_check, 'gitUrl', BUILD_CONF)513 if not os.path.exists('%s/​%s' % (common_path, project_check)):514 os.chdir(common_path)515 os.system('git clone %s' % git_Url)516 """调用部署"""517 nodejs_project_deploy(project_check, common_path)518if __name__ == '__main__':519 operation = sys.argv[1]520 language = sys.argv[2]521 project = sys.argv[3]522 branch = sys.argv[4]523 build_number = sys.argv[5]524 namespace = sys.argv[6]525 env = sys.argv[7]526 build_user = sys.argv[8]527 tag = sys.argv[9]528 container_name = '%s%s%s' % (env, tag, project)529 save_data(serverInfo)530 check_tag(tag)531 check_env(PROXYSERVER, project)532 if operation == 'push' and language == 'java':533 java_deploy_check(project)534 elif operation == 'push' and language == 'nodejs':535 nodejs_deploy_check(project)536 else:537 print("flag Error exit")...

base.py

Source:base.py

1#!/​usr/​bin/​python2#3# Copyright 2018-2021 Polyaxon, Inc.4#5# Licensed under the Apache License, Version 2.0 (the "License");6# you may not use this file except in compliance with the License.7# You may obtain a copy of the License at8#9# http:/​/​www.apache.org/​licenses/​LICENSE-2.010#11# Unless required by applicable law or agreed to in writing, software12# distributed under the License is distributed on an "AS IS" BASIS,13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.14# See the License for the specific language governing permissions and15# limitations under the License.16import time17import traceback18from concurrent.futures import ThreadPoolExecutor19from typing import Dict, Tuple20from kubernetes.client.rest import ApiException21import polyaxon_sdk22from polyaxon import live_state, settings23from polyaxon.agents import converter24from polyaxon.agents.spawners.spawner import Spawner25from polyaxon.client import PolyaxonClient26from polyaxon.env_vars.getters import get_run_info27from polyaxon.exceptions import PolypodException28from polyaxon.lifecycle import V1StatusCondition, V1Statuses29from polyaxon.logger import logger30from polyaxon.schemas.cli.checks_config import ChecksConfig31from polyaxon.utils.tz_utils import now32from polyaxon.utils.workers_utils import exit_context, get_pool_workers, get_wait33class BaseAgent:34 HEALTH_FILE = "/​tmp/​.healthz"35 SLEEP_STOP_TIME = 60 * 536 SLEEP_ARCHIVED_TIME = 60 * 6037 def __init__(self, sleep_interval=None):38 self.sleep_interval = sleep_interval39 self.spawner = Spawner()40 self._spawner_refreshed_at = now()41 self.client = PolyaxonClient()42 self._graceful_shutdown = False43 self.content = settings.AGENT_CONFIG.to_dict(dump=True)44 def get_info(self) -> polyaxon_sdk.V1Agent:45 raise NotImplementedError46 def get_state(self) -> polyaxon_sdk.V1AgentStateResponse:47 raise NotImplementedError48 def sync_compatible_updates(self, compatible_updates: Dict):49 raise NotImplementedError50 @classmethod51 def get_healthz_config(cls):52 try:53 return ChecksConfig.read(cls.HEALTH_FILE, config_type=".json")54 except Exception: # noqa55 return56 @classmethod57 def ping(cls):58 ChecksConfig.init_file(cls.HEALTH_FILE)59 config = cls.get_healthz_config()60 if config:61 config.last_check = now()62 config.write(cls.HEALTH_FILE)63 @classmethod64 def pong(cls, interval: int = 15) -> bool:65 config = cls.get_healthz_config()66 if not config:67 return False68 return not config.should_check(interval=interval)69 def refresh_spawner(self):70 if (71 now() - self._spawner_refreshed_at72 ).total_seconds() > settings.AGENT_CONFIG.get_spawner_refresh_interval():73 logger.debug("Refreshing spawner ... ")74 self.spawner.refresh()75 self._spawner_refreshed_at = now()76 def start(self) -> None:77 try:78 with exit_context() as exit_event:79 index = 080 workers = get_pool_workers()81 with ThreadPoolExecutor(workers) as pool:82 logger.debug("Thread pool Workers: {}".format(workers))83 timeout = self.sleep_interval or get_wait(index)84 while not exit_event.wait(timeout=timeout):85 index += 186 self.refresh_spawner()87 agent_state = self.process(pool)88 self._check_status(agent_state)89 if agent_state.state.full:90 index = 291 self.ping()92 timeout = self.sleep_interval or get_wait(index)93 logger.info("Sleeping for {} seconds".format(timeout))94 finally:95 self.end()96 def _check_status(self, agent_state):97 if agent_state.status == V1Statuses.STOPPED:98 print(99 "Agent has been stopped from the platform,"100 "but the deployment is still running."101 "Please either set the agent to starting or teardown the agent deployment."102 )103 self.end(sleep=self.SLEEP_STOP_TIME)104 elif agent_state.live_state < live_state.STATE_LIVE:105 print(106 "Agent has been archived from the platform,"107 "but the deployment is still running."108 "Please either restore the agent or teardown the agent deployment."109 )110 self.end(sleep=self.SLEEP_ARCHIVED_TIME)111 def end(self, sleep: int = None):112 self._graceful_shutdown = True113 if sleep:114 time.sleep(sleep)115 else:116 logger.info("Agent is shutting down.")117 def process(self, pool: "ThreadPoolExecutor") -> polyaxon_sdk.V1AgentStateResponse:118 try:119 agent_state = self.get_state()120 if agent_state.compatible_updates:121 self.sync_compatible_updates(agent_state.compatible_updates)122 if agent_state:123 logger.info("Starting runs submission process.")124 else:125 logger.info("No state was found.")126 return polyaxon_sdk.V1AgentStateResponse()127 state = agent_state.state128 for run_data in state.schedules or []:129 pool.submit(self.submit_run, run_data)130 for run_data in state.queued or []:131 pool.submit(self.submit_run, run_data)132 for run_data in state.checks or []:133 pool.submit(self.check_run, run_data)134 for run_data in state.stopping or []:135 pool.submit(self.stop_run, run_data)136 for run_data in state.apply or []:137 pool.submit(self.apply_run, run_data)138 for run_data in state.deleting or []:139 pool.submit(self.delete_run, run_data)140 for run_data in state.hooks or []:141 pool.submit(self.make_and_create_run, run_data)142 for run_data in state.watchdogs or []:143 pool.submit(self.make_and_create_run, run_data)144 for run_data in state.tuners or []:145 pool.submit(self.make_and_create_run, run_data, True)146 return agent_state147 except Exception as exc:148 logger.error(exc)149 return polyaxon_sdk.V1AgentStateResponse()150 def log_run_failed(151 self,152 run_owner: str,153 run_project: str,154 run_uuid: str,155 exc: Exception,156 message: str = None,157 ) -> None:158 message = message or "Agent failed deploying run.\n"159 message += "error: {}\n{}".format(repr(exc), traceback.format_exc())160 self.log_run_status(161 run_owner=run_owner,162 run_project=run_project,163 run_uuid=run_uuid,164 status=V1Statuses.FAILED,165 reason="AgentLogger",166 message=message,167 )168 logger.warning(message)169 def log_run_stopped(self, run_owner: str, run_project: str, run_uuid: str) -> None:170 message = "Run was not found. The agent assumed it was already stopped."171 self.log_run_status(172 run_owner=run_owner,173 run_project=run_project,174 run_uuid=run_uuid,175 status=V1Statuses.STOPPED,176 reason="AgentLogger",177 message=message,178 )179 logger.warning(message)180 def log_run_scheduled(181 self, run_owner: str, run_project: str, run_uuid: str182 ) -> None:183 message = "Run was scheduled by the agent."184 self.log_run_status(185 run_owner=run_owner,186 run_project=run_project,187 run_uuid=run_uuid,188 status=V1Statuses.SCHEDULED,189 reason="AgentLogger",190 message=message,191 )192 logger.info(message)193 def log_run_running(self, run_owner: str, run_project: str, run_uuid: str) -> None:194 message = "Run changes were applied by the agent."195 self.log_run_status(196 run_owner=run_owner,197 run_project=run_project,198 run_uuid=run_uuid,199 status=V1Statuses.RUNNING,200 reason="AgentLogger",201 message=message,202 )203 logger.info(message)204 def log_run_status(205 self,206 run_owner: str,207 run_project: str,208 run_uuid: str,209 status: str,210 reason: str = None,211 message: str = None,212 ):213 status_condition = V1StatusCondition.get_condition(214 type=status, status=True, reason=reason, message=message215 )216 self.client.runs_v1.create_run_status(217 owner=run_owner,218 project=run_project,219 uuid=run_uuid,220 body={"condition": status_condition},221 async_req=True,222 )223 def clean_run(self, run_uuid: str, run_kind: str):224 try:225 self.spawner.clean(run_uuid=run_uuid, run_kind=run_kind)226 self.spawner.stop(run_uuid=run_uuid, run_kind=run_kind)227 except ApiException as e:228 if e.status == 404:229 logger.info("Run does not exist.")230 except Exception as e:231 logger.info(232 "Run could not be cleaned: {}\n{}".format(233 repr(e), traceback.format_exc()234 )235 )236 def make_run_resource(237 self,238 owner_name: str,239 project_name: str,240 run_name: str,241 run_uuid: str,242 content: str,243 default_auth=False,244 ) -> Dict:245 try:246 return converter.make_and_convert(247 owner_name=owner_name,248 project_name=project_name,249 run_name=run_name,250 run_uuid=run_uuid,251 content=content,252 default_auth=default_auth,253 )254 except PolypodException as e:255 logger.info(256 "Run could not be cleaned. Agent failed converting run manifest: {}\n{}".format(257 repr(e), traceback.format_exc()258 )259 )260 except Exception as e:261 logger.info(262 "Agent failed during compilation with unknown exception: {}\n{}".format(263 repr(e), traceback.format_exc()264 )265 )266 def prepare_run_resource(267 self,268 owner_name: str,269 project_name: str,270 run_name: str,271 run_uuid: str,272 content: str,273 ) -> Dict:274 try:275 return converter.convert(276 owner_name=owner_name,277 project_name=project_name,278 run_name=run_name,279 run_uuid=run_uuid,280 content=content,281 default_auth=True,282 agent_content=self.content,283 )284 except PolypodException as e:285 self.log_run_failed(286 run_owner=owner_name,287 run_project=project_name,288 run_uuid=run_uuid,289 exc=e,290 message="Agent failed converting run manifest.\n",291 )292 except Exception as e:293 self.log_run_failed(294 run_owner=owner_name,295 run_project=project_name,296 run_uuid=run_uuid,297 exc=e,298 message="Agent failed during compilation with unknown exception.\n",299 )300 def submit_run(self, run_data: Tuple[str, str, str, str]):301 run_owner, run_project, run_uuid = get_run_info(run_instance=run_data[0])302 resource = self.prepare_run_resource(303 owner_name=run_owner,304 project_name=run_project,305 run_name=run_data[2],306 run_uuid=run_uuid,307 content=run_data[3],308 )309 if not resource:310 return311 try:312 self.spawner.create(313 run_uuid=run_uuid, run_kind=run_data[1], resource=resource314 )315 except ApiException as e:316 if e.status == 409:317 logger.info("Run already running, triggering an apply mechanism.")318 self.apply_run(run_data=run_data)319 else:320 logger.info("Run submission error.")321 self.log_run_failed(322 run_owner=run_owner,323 run_project=run_project,324 run_uuid=run_uuid,325 exc=e,326 )327 except Exception as e:328 self.log_run_failed(329 run_owner=run_owner,330 run_project=run_project,331 run_uuid=run_uuid,332 exc=e,333 )334 def make_and_create_run(335 self, run_data: Tuple[str, str, str, str], default_auth: bool = False336 ):337 run_owner, run_project, run_uuid = get_run_info(run_instance=run_data[0])338 resource = self.make_run_resource(339 owner_name=run_owner,340 project_name=run_project,341 run_name=run_data[2],342 run_uuid=run_uuid,343 content=run_data[3],344 default_auth=default_auth,345 )346 if not resource:347 return348 try:349 self.spawner.create(350 run_uuid=run_uuid, run_kind=run_data[1], resource=resource351 )352 except ApiException as e:353 if e.status == 409:354 logger.info("Run already running, triggering an apply mechanism.")355 else:356 logger.info("Run submission error.")357 except Exception as e:358 logger.info(359 "Run could not be cleaned. Agent failed converting run manifest: {}\n{}".format(360 repr(e), traceback.format_exc()361 )362 )363 def apply_run(self, run_data: Tuple[str, str, str, str]):364 run_owner, run_project, run_uuid = get_run_info(run_instance=run_data[0])365 resource = self.prepare_run_resource(366 owner_name=run_owner,367 project_name=run_project,368 run_name=run_data[2],369 run_uuid=run_uuid,370 content=run_data[3],371 )372 if not resource:373 return374 try:375 self.spawner.apply(376 run_uuid=run_uuid, run_kind=run_data[1], resource=resource377 )378 self.log_run_running(379 run_owner=run_owner, run_project=run_project, run_uuid=run_uuid380 )381 except Exception as e:382 self.log_run_failed(383 run_owner=run_owner, run_project=run_project, run_uuid=run_uuid, exc=e384 )385 self.clean_run(run_uuid=run_uuid, run_kind=run_data[1])386 def check_run(self, run_data: Tuple[str, str]):387 run_owner, run_project, run_uuid = get_run_info(run_instance=run_data[0])388 try:389 self.spawner.get(run_uuid=run_uuid, run_kind=run_data[1])390 except ApiException as e:391 if e.status == 404:392 logger.info(393 "Run does not exist anymore, it could have been stopped or deleted."394 )395 self.log_run_stopped(396 run_owner=run_owner, run_project=run_project, run_uuid=run_uuid397 )398 def stop_run(self, run_data: Tuple[str, str]):399 run_owner, run_project, run_uuid = get_run_info(run_instance=run_data[0])400 try:401 self.spawner.stop(run_uuid=run_uuid, run_kind=run_data[1])402 except ApiException as e:403 if e.status == 404:404 logger.info("Run does not exist anymore, it could have been stopped.")405 self.log_run_stopped(406 run_owner=run_owner, run_project=run_project, run_uuid=run_uuid407 )408 except Exception as e:409 self.log_run_failed(410 run_owner=run_owner,411 run_project=run_project,412 run_uuid=run_uuid,413 exc=e,414 message="Agent failed stopping run.\n",415 )416 def delete_run(self, run_data: Tuple[str, str, str, str]):417 run_owner, run_project, run_uuid = get_run_info(run_instance=run_data[0])418 self.clean_run(run_uuid=run_uuid, run_kind=run_data[1])419 if run_data[3]:...

Show more

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.

LambdaTest Learning Hubs:

• JUnit Tutorial
• TestNG Tutorial
• Webdriver Tutorial
• WebDriverIO Tutorial
• Protractor Tutorial
• Selenium 4 Tutorial
• Jenkins Tutorial
• NUnit Tutorial
• Jest Tutorial
• Playwright Tutorial
• Cypress Tutorial
• PyTest Tutorial

YouTube

You could also refer to video tutorials over LambdaTest YouTube channel to get step by step demonstration from industry experts.

Run Lemoncheesecake automation tests on LambdaTest cloud grid

Perform automation testing on 3000+ real desktop and mobile devices online.

View sample repo View DocumentationSign up Free

›

_