How to use test_backslash method in autotest

Best Python code snippet using autotest_python

pbs_hook_set_jobenv.py

Source:pbs_hook_set_jobenv.py Github

copy

Full Screen

1# coding: utf-82# Copyright (C) 1994-2020 Altair Engineering, Inc.3# For more information, contact Altair at www.altair.com.4#5# This file is part of both the OpenPBS software ("OpenPBS")6# and the PBS Professional ("PBS Pro") software.7#8# Open Source License Information:9#10# OpenPBS is free software. You can redistribute it and/or modify it under11# the terms of the GNU Affero General Public License as published by the12# Free Software Foundation, either version 3 of the License, or (at your13# option) any later version.14#15# OpenPBS is distributed in the hope that it will be useful, but WITHOUT16# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or17# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public18# License for more details.19#20# You should have received a copy of the GNU Affero General Public License21# along with this program. If not, see <http://www.gnu.org/licenses/>.22#23# Commercial License Information:24#25# PBS Pro is commercially licensed software that shares a common core with26# the OpenPBS software. For a copy of the commercial license terms and27# conditions, go to: (http://www.pbspro.com/agreement.html) or contact the28# Altair Legal Department.29#30# Altair's dual-license business model allows companies, individuals, and31# organizations to create proprietary derivative works of OpenPBS and32# distribute them - whether embedded or bundled with other software -33# under a commercial license agreement.34#35# Use of Altair's trademarks, including but not limited to "PBS™",36# "OpenPBS®", "PBS Professional®", and "PBS Pro™" and Altair's logos is37# subject to Altair's trademark licensing policies.38import os39from tests.functional import *40from ptl.utils.pbs_logutils import PBSLogUtils41@skipOnShasta42class TestPbsHookSetJobEnv(TestFunctional):43 """44 This test suite to make sure hooks properly45 handle environment variables with special characters,46 values, in particular newline (\n), commas (,), semicolons (;),47 single quotes ('), double quotes ("), and backaslashes (\).48 PRE: Set up currently executing user's environment to have variables49 whose values have the special characters.50 Job A: Submit a job using the -V option (pass current environment)51 where there are NO hooks in the system.52 Introduce execjob_begin and execjob_launch hooks in the system.53 Let the former update pbs.event().job.Variable_List while the latter54 update pbs.event().env.55 Job B: Submit a job using the -V option (pass current environment)56 where there are now mom hooks in the system.57 POST: Job A and Job B would see the same environment variables, with58 Job B also seeing the changes made to the job by the 2 mom hooks.59 """60 # List of environment variables not to compare between61 # job ran without hooks, job ran with hooks.62 exclude_env = []63 env_nohook = {}64 env_nohook_exclude = {}65 env_hook = {}66 env_hook_exclude = {}67 def setUp(self):68 """69 Set environment variables70 """71 TestFunctional.setUp(self)72 # Set environment variables with special characters73 os.environ['TEST_COMMA'] = '1,2,3,4'74 os.environ['TEST_RETURN'] = """'3,754,765'"""77 os.environ['TEST_SEMICOLON'] = ';'78 os.environ['TEST_ENCLOSED'] = '\',\''79 os.environ['TEST_COLON'] = ':'80 os.environ['TEST_BACKSLASH'] = '\\'81 os.environ['TEST_DQUOTE'] = '"'82 os.environ['TEST_DQUOTE2'] = 'happy days"are"here to stay'83 os.environ['TEST_DQUOTE3'] = 'nothing compares" to you'84 os.environ['TEST_DQUOTE4'] = '"music makes the people"'85 os.environ['TEST_DQUOTE5'] = 'music "makes \'the\'"people'86 os.environ['TEST_DQUOTE6'] = 'lalaland"'87 os.environ['TEST_SQUOTE'] = '\''88 os.environ['TEST_SQUOTE2'] = 'happy\'days'89 os.environ['TEST_SQUOTE3'] = 'the days\'are here now\'then'90 os.environ['TEST_SQUOTE4'] = '\'the way that was\''91 os.environ['TEST_SQUOTE5'] = 'music \'makes "the\'"people'92 os.environ['TEST_SQUOTE6'] = 'loving\''93 os.environ['TEST_SPECIAL'] = "{}[]()~@#$%^&*!"94 os.environ['TEST_SPECIAL2'] = "<dumb-test_text>"95 # List of environment variables not to compare between96 # job ran without hooks, job ran with hooks.97 self.exclude_env = ['PBS_NODEFILE']98 self.exclude_env += ['PBS_JOBID']99 self.exclude_env += ['PBS_JOBCOOKIE']100 # Each job submitted by default gets a unique jobname101 self.exclude_env += ['PBS_JOBNAME']102 self.exclude_env += ['TMPDIR']103 self.exclude_env += ['happy']104 self.ATTR_V = 'Full_Variable_List'105 api_to_cli.setdefault(self.ATTR_V, 'V')106 # temporary files107 fn = self.du.create_temp_file(prefix="job_out1")108 self.job_out1_tempfile = fn109 fn = self.du.create_temp_file(prefix="job_out2")110 self.job_out2_tempfile = fn111 fn = self.du.create_temp_file(prefix="job_out3")112 self.job_out3_tempfile = fn113 def tearDown(self):114 TestFunctional.tearDown(self)115 try:116 os.remove(self.job_out1_tempfile)117 os.remove(self.job_out2_tempfile)118 os.remove(self.job_out3_tempfile)119 except OSError:120 pass121 def read_env(self, outputfile, ishook):122 """123 Parse the output file and store the124 variable list in a dictionary125 """126 with open(outputfile) as fd:127 pkey = ""128 tmpenv = {}129 penv = {}130 penv_exclude = {}131 for line in fd:132 l = line.split("=", 1)133 if (len(l) == 2):134 pkey = l[0]135 if pkey not in self.exclude_env:136 penv[pkey] = l[1]137 tmpenv = penv138 else:139 penv_exclude[pkey] = l[1]140 tmpenv = penv_exclude141 elif pkey != "":142 # append to previous dictionary entry143 tmpenv[pkey] += l[0]144 if (ishook == "hook"):145 self.env_hook = penv146 self.env_hook_exclude = penv_exclude147 else:148 self.env_nohook = penv149 self.env_nohook_exclude = penv_exclude150 def common_log_match(self, daemon):151 """152 Validate the env variable output in daemon logs153 """154 logutils = PBSLogUtils()155 logmsg = ["TEST_COMMA=1\,2\,3\,4",156 "TEST_SEMICOLON=;",157 "TEST_ENCLOSED=\\'\,\\'",158 "TEST_COLON=:",159 "TEST_BACKSLASH=\\\\",160 "TEST_DQUOTE=\\\"",161 "TEST_DQUOTE2=happy days\\\"are\\\"here to stay",162 "TEST_DQUOTE3=nothing compares\\\" to you",163 "TEST_DQUOTE4=\\\"music makes the people\\\"",164 "TEST_DQUOTE5=music \\\"makes \\'the\\'\\\"people",165 "TEST_DQUOTE6=lalaland\\\"",166 "TEST_SQUOTE=\\'",167 "TEST_SQUOTE2=happy\\'days",168 "TEST_SQUOTE3=the days\\'are here now\\'then",169 "TEST_SQUOTE4=\\'the way that was\\'",170 "TEST_SQUOTE5=music \\'makes \\\"the\\'\\\"people",171 "TEST_SQUOTE6=loving\\'",172 "TEST_SPECIAL={}[]()~@#$%^&*!",173 "TEST_SPECIAL2=<dumb-test_text>",174 "TEST_RETURN=\\'3\,",175 # Cannot add '\n' here because '\n' is not included in176 # the items of the list returned by log_lines(), (though177 # lines are split by '\n')178 "4\,",179 "5\\',"]180 if (daemon == "mom"):181 self.logger.info("Matching in mom logs")182 logfile_type = self.mom183 elif (daemon == "server"):184 self.logger.info("Matching in server logs")185 logfile_type = self.server186 else:187 self.logger.info("Provide a valid daemon name; server or mom")188 return189 lines = None190 ret_linenum = 0191 search_msg = 'log match: searching for '192 nomatch_msg = ' No match for '193 for msg in logmsg:194 for attempt in range(1, 61):195 lines = self.server.log_lines(196 logfile_type, starttime=self.server.ctime)197 match = logutils.match_msg(lines, msg=msg)198 if match:199 # Dont want the test to pass if there are200 # unwanted matched for "4\," and "5\\'.201 if msg == "TEST_RETURN=\\'3\,":202 ret_linenum = match[0]203 if (msg == "4\," and match[0] != (ret_linenum - 1)) or \204 (msg == "5\\'" and match[0] != (ret_linenum - 2)):205 pass206 else:207 self.logger.info(search_msg + msg + ' ... OK')208 break209 else:210 self.logger.info(nomatch_msg + msg +211 ' attempt ' + str(attempt))212 time.sleep(0.5)213 if match is None:214 _msg = nomatch_msg + msg215 raise PtlLogMatchError(rc=1, rv=False, msg=_msg)216 def common_validate(self):217 """218 This is a common function to validate the219 environment values with and without hook220 """221 self.assertEqual(self.env_nohook, self.env_hook)222 self.logger.info("Environment variables are same"223 " with and without hooks")224 match_str = self.env_hook['TEST_COMMA'].rstrip('\n')225 self.assertEqual(os.environ['TEST_COMMA'], match_str)226 self.logger.info(227 "TEST_COMMA matched - " + os.environ['TEST_COMMA'] +228 " == " + match_str)229 self.assertEqual(os.environ['TEST_RETURN'],230 self.env_hook['TEST_RETURN'].rstrip('\n'))231 self.logger.info(232 "TEST_RETURN matched - " + os.environ['TEST_RETURN'] +233 " == " + self.env_hook['TEST_RETURN'].rstrip('\n'))234 self.assertEqual(os.environ['TEST_SEMICOLON'],235 self.env_hook['TEST_SEMICOLON'].rstrip('\n'))236 self.logger.info(237 "TEST_SEMICOLON matched - " + os.environ['TEST_SEMICOLON'] +238 " == " + self.env_hook['TEST_SEMICOLON'].rstrip('\n'))239 self.assertEqual(240 os.environ['TEST_ENCLOSED'],241 self.env_hook['TEST_ENCLOSED'].rstrip('\n'))242 self.logger.info(243 "TEST_ENCLOSED matched - " + os.environ['TEST_ENCLOSED'] +244 " == " + self.env_hook['TEST_ENCLOSED'].rstrip('\n'))245 self.assertEqual(os.environ['TEST_COLON'],246 self.env_hook['TEST_COLON'].rstrip('\n'))247 self.logger.info("TEST_COLON matched - " + os.environ['TEST_COLON'] +248 " == " + self.env_hook['TEST_COLON'].rstrip('\n'))249 self.assertEqual(250 os.environ['TEST_BACKSLASH'],251 self.env_hook['TEST_BACKSLASH'].rstrip('\n'))252 self.logger.info(253 "TEST_BACKSLASH matched - " + os.environ['TEST_BACKSLASH'] +254 " == " + self.env_hook['TEST_BACKSLASH'].rstrip('\n'))255 self.assertEqual(os.environ['TEST_DQUOTE'],256 self.env_hook['TEST_DQUOTE'].rstrip('\n'))257 self.logger.info("TEST_DQUOTE matched - " +258 os.environ['TEST_DQUOTE'] +259 " == " + self.env_hook['TEST_DQUOTE'].rstrip('\n'))260 self.assertEqual(os.environ['TEST_DQUOTE2'],261 self.env_hook['TEST_DQUOTE2'].rstrip('\n'))262 self.logger.info("TEST_DQUOTE2 matched - " +263 os.environ['TEST_DQUOTE2'] +264 " == " + self.env_hook['TEST_DQUOTE2'].rstrip('\n'))265 self.assertEqual(os.environ['TEST_DQUOTE3'],266 self.env_hook['TEST_DQUOTE3'].rstrip('\n'))267 self.logger.info("TEST_DQUOTE3 matched - " +268 os.environ['TEST_DQUOTE3'] +269 " == " + self.env_hook['TEST_DQUOTE3'].rstrip('\n'))270 self.assertEqual(os.environ['TEST_DQUOTE4'],271 self.env_hook['TEST_DQUOTE4'].rstrip('\n'))272 self.logger.info("TEST_DQUOTE4 matched - " +273 os.environ['TEST_DQUOTE4'] +274 " == " + self.env_hook['TEST_DQUOTE4'].rstrip('\n'))275 self.assertEqual(os.environ['TEST_DQUOTE5'],276 self.env_hook['TEST_DQUOTE5'].rstrip('\n'))277 self.logger.info("TEST_DQUOTE5 matched - " +278 os.environ['TEST_DQUOTE5'] +279 " == " + self.env_hook['TEST_DQUOTE5'].rstrip('\n'))280 self.assertEqual(os.environ['TEST_DQUOTE6'],281 self.env_hook['TEST_DQUOTE6'].rstrip('\n'))282 self.logger.info("TEST_DQUOTE6 matched - " +283 os.environ['TEST_DQUOTE6'] +284 " == " + self.env_hook['TEST_DQUOTE6'].rstrip('\n'))285 self.assertEqual(os.environ['TEST_SQUOTE'],286 self.env_hook['TEST_SQUOTE'].rstrip('\n'))287 self.logger.info("TEST_SQUOTE matched - " + os.environ['TEST_SQUOTE'] +288 " == " + self.env_hook['TEST_SQUOTE'].rstrip('\n'))289 self.assertEqual(os.environ['TEST_SQUOTE2'],290 self.env_hook['TEST_SQUOTE2'].rstrip('\n'))291 self.logger.info("TEST_SQUOTE2 matched - " +292 os.environ['TEST_SQUOTE2'] +293 " == " + self.env_hook['TEST_SQUOTE2'].rstrip('\n'))294 self.assertEqual(os.environ['TEST_SQUOTE3'],295 self.env_hook['TEST_SQUOTE3'].rstrip('\n'))296 self.logger.info("TEST_SQUOTE3 matched - " +297 os.environ['TEST_SQUOTE3'] +298 " == " + self.env_hook['TEST_SQUOTE3'].rstrip('\n'))299 self.assertEqual(os.environ['TEST_SQUOTE4'],300 self.env_hook['TEST_SQUOTE4'].rstrip('\n'))301 self.logger.info("TEST_SQUOTE4 matched - " +302 os.environ['TEST_SQUOTE4'] +303 " == " + self.env_hook['TEST_SQUOTE4'].rstrip('\n'))304 self.assertEqual(os.environ['TEST_SQUOTE5'],305 self.env_hook['TEST_SQUOTE5'].rstrip('\n'))306 self.logger.info("TEST_SQUOTE5 matched - " +307 os.environ['TEST_SQUOTE5'] +308 " == " + self.env_hook['TEST_SQUOTE5'].rstrip('\n'))309 self.assertEqual(os.environ['TEST_SQUOTE6'],310 self.env_hook['TEST_SQUOTE6'].rstrip('\n'))311 self.logger.info("TEST_SQUOTE6 matched - " +312 os.environ['TEST_SQUOTE6'] +313 " == " + self.env_hook['TEST_SQUOTE6'].rstrip('\n'))314 self.assertEqual(os.environ['TEST_SPECIAL'],315 self.env_hook['TEST_SPECIAL'].rstrip('\n'))316 self.logger.info("TEST_SPECIAL matched - " +317 os.environ['TEST_SPECIAL'] +318 " == " + self.env_hook['TEST_SPECIAL'].rstrip('\n'))319 self.assertEqual(os.environ['TEST_SPECIAL2'],320 self.env_hook['TEST_SPECIAL2'].rstrip('\n'))321 self.logger.info("TEST_SPECIAL2 matched - " +322 os.environ['TEST_SPECIAL2'] +323 " == " + self.env_hook['TEST_SPECIAL2'].rstrip('\n'))324 def create_and_submit_job(self, user=None, attribs=None, content=None,325 content_interactive=None, preserve_env=False):326 """327 create the job object and submit it to the server328 as 'user', attributes list 'attribs' script329 'content' or 'content_interactive', and to330 'preserve_env' if interactive job.331 """332 # A user=None value means job will be executed by current user333 # where the environment is set up334 if attribs is None:335 use_attribs = {}336 else:337 use_attribs = attribs338 retjob = Job(username=user, attrs=use_attribs)339 if content is not None:340 retjob.create_script(body=content)341 elif content_interactive is not None:342 retjob.interactive_script = content_interactive343 retjob.preserve_env = preserve_env344 return self.server.submit(retjob)345 def test_begin_launch(self):346 """347 Test to verify that job environment variables having special348 characters are not truncated with execjob_launch and349 execjob_begin hook350 """351 self.exclude_env += ['HAPPY']352 self.exclude_env += ['happy']353 a = {'Resource_List.select': '1:ncpus=1',354 'Resource_List.walltime': 10,355 self.ATTR_V: None}356 script = ['env\n']357 script += ['sleep 5\n']358 # Submit a job without hooks in the system359 jid = self.create_and_submit_job(attribs=a, content=script)360 qstat = self.server.status(JOB, ATTR_o, id=jid)361 job_outfile = qstat[0][ATTR_o].split(':')[1]362 self.server.expect(JOB, 'queue', op=UNSET, id=jid, offset=10)363 # Read the env variables from job output364 self.env_nohook = {}365 self.env_nohook_exclude = {}366 self.read_env(job_outfile, "nohook")367 # Now start introducing hooks368 hook_body = """369import pbs370e=pbs.event()371e.job.Variable_List["happy"] = "days"372pbs.logmsg(pbs.LOG_DEBUG,"Variable List is %s" % (e.job.Variable_List,))373"""374 hook_name = "begin"375 a2 = {'event': "execjob_begin", 'enabled': 'True', 'debug': 'True'}376 rv = self.server.create_import_hook(377 hook_name,378 a2,379 hook_body,380 overwrite=True)381 self.assertTrue(rv)382 hook_body = """383import pbs384e=pbs.event()385e.env["HAPPY"] = "nights"386"""387 hook_name = "launch"388 a2 = {'event': "execjob_launch", 'enabled': 'True', 'debug': 'True'}389 rv = self.server.create_import_hook(390 hook_name,391 a2,392 hook_body,393 overwrite=True)394 self.assertTrue(rv)395 # Submit a job with hooks in the system396 jid2 = self.create_and_submit_job(attribs=a, content=script)397 qstat = self.server.status(JOB, ATTR_o, id=jid2)398 job_outfile = qstat[0][ATTR_o].split(':')[1]399 self.server.expect(JOB, 'queue', op=UNSET, id=jid2, offset=10)400 self.env_hook = {}401 self.env_hook_exclude = {}402 self.read_env(job_outfile, "hook")403 # Validate the values printed in job output file404 self.assertTrue('HAPPY' not in self.env_nohook_exclude)405 self.assertTrue('happy' not in self.env_nohook_exclude)406 self.assertEqual(self.env_hook_exclude['HAPPY'], 'nights\n')407 self.assertEqual(self.env_hook_exclude['happy'], 'days\n')408 self.common_validate()409 # Check the values in mom logs as well410 self.common_log_match("mom")411 def test_que(self):412 """413 Test that variable_list do not change with and without414 queuejob hook415 """416 self.exclude_env += ['happy']417 a = {'Resource_List.select': '1:ncpus=1',418 'Resource_List.walltime': 10}419 script = ['#PBS -V']420 script += ['env\n']421 script += ['sleep 5\n']422 # Submit a job without hooks in the system423 jid = self.create_and_submit_job(attribs=a, content=script)424 qstat = self.server.status(JOB, ATTR_o, id=jid)425 job_outfile = qstat[0][ATTR_o].split(':')[1]426 self.server.expect(JOB, 'queue', op=UNSET, id=jid, offset=10)427 # Read the env variable from job output file428 self.env_nohook = {}429 self.env_nohook_exclude = {}430 self.read_env(job_outfile, "nohook")431 # Now start introducing hooks432 hook_body = """433import pbs434e=pbs.event()435e.job.Variable_List["happy"] = "days"436pbs.logmsg(pbs.LOG_DEBUG,"Variable List is %s" % (e.job.Variable_List,))437"""438 hook_name = "qjob"439 a2 = {'event': "queuejob", 'enabled': 'True', 'debug': 'True'}440 rv = self.server.create_import_hook(441 hook_name,442 a2,443 hook_body,444 overwrite=True)445 self.assertTrue(rv)446 # Submit a job with hooks in the system447 jid2 = self.create_and_submit_job(attribs=a, content=script)448 qstat = self.server.status(JOB, ATTR_o, id=jid2)449 job_outfile = qstat[0][ATTR_o].split(':')[1]450 self.server.expect(JOB, 'queue', op=UNSET, id=jid2, offset=10)451 self.env_hook = {}452 self.env_hook_exclude = {}453 self.read_env(job_outfile, "hook")454 # Validate the env values from job output file455 # with and without queuejob hook456 self.assertTrue('happy' not in self.env_nohook_exclude)457 self.assertEqual(self.env_hook_exclude['happy'], 'days\n')458 self.common_validate()459 self.common_log_match("server")460 def test_execjob_epi(self):461 """462 Test that Variable_List will contain environment variable463 with commas, newline and all special characters even for464 other mom hooks465 """466 self.exclude_env += ['happy']467 a = {'Resource_List.select': '1:ncpus=1',468 'Resource_List.walltime': 10}469 script = ['#PBS -V']470 script += ['env\n']471 script += ['sleep 5\n']472 # Submit a job without hooks in the system473 jid = self.create_and_submit_job(attribs=a, content=script)474 qstat = self.server.status(JOB, ATTR_o, id=jid)475 job_outfile = qstat[0][ATTR_o].split(':')[1]476 self.server.expect(JOB, 'queue', op=UNSET, id=jid, offset=10)477 # Read the output file and parse the values478 self.env_nohook = {}479 self.env_nohook_exclude = {}480 self.read_env(job_outfile, "nohook")481 # Now start the hooks482 hook_name = "test_epi"483 hook_body = """484import pbs485e = pbs.event()486j = e.job487j.Variable_List["happy"] = "days"488pbs.logmsg(pbs.LOG_DEBUG,"Variable_List is %s" % (j.Variable_List,))489"""490 a2 = {'event': "execjob_epilogue", 'enabled': "true", 'debug': "true"}491 self.server.create_import_hook(492 hook_name,493 a2,494 hook_body,495 overwrite=True)496 # Submit a job with hooks in the system497 jid2 = self.create_and_submit_job(attribs=a, content=script)498 qstat = self.server.status(JOB, ATTR_o, id=jid2)499 job_outfile = qstat[0][ATTR_o].split(':')[1]500 self.server.expect(JOB, 'queue', op=UNSET, id=jid2, offset=10)501 # read the output file for env with hooks502 self.env_hook = {}503 self.env_hook_exclude = {}504 self.read_env(job_outfile, "hook")505 # Validate506 self.common_validate()507 # Verify the env variables in logs too508 self.common_log_match("mom")509 def test_execjob_pro(self):510 """511 Test that environment variable not gets truncated512 for execjob_prologue hook513 """514 a = {'Resource_List.select': '1:ncpus=1',515 'Resource_List.walltime': 10}516 script = ['#PBS -V']517 script += ['env\n']518 script += ['sleep 5\n']519 # Submit a job without hooks in the system520 jid = self.create_and_submit_job(attribs=a, content=script)521 qstat = self.server.status(JOB, ATTR_o, id=jid)522 job_outfile = qstat[0][ATTR_o].split(':')[1]523 self.server.expect(JOB, 'queue', op=UNSET, id=jid, offset=10)524 # read the output file for env without hook525 self.env_nohook = {}526 self.env_nohook_exclude = {}527 self.read_env(job_outfile, "nohook")528 # Now start the hooks529 hook_name = "test_pro"530 hook_body = """531import pbs532e = pbs.event()533j = e.job534j.Variable_List["happy"] = "days"535pbs.logmsg(pbs.LOG_DEBUG,"Variable_List is %s" % (j.Variable_List,))536"""537 a2 = {'event': "execjob_prologue", 'enabled': "true", 'debug': "true"}538 rv = self.server.create_import_hook(539 hook_name,540 a2,541 hook_body,542 overwrite=True)543 self.assertTrue(rv)544 # Submit a job with hooks in the system545 jid2 = self.create_and_submit_job(attribs=a, content=script)546 qstat = self.server.status(JOB, ATTR_o, id=jid2)547 job_outfile = qstat[0][ATTR_o].split(':')[1]548 self.server.expect(JOB, 'queue', op=UNSET, id=jid2, offset=10)549 # Read the job ouput file550 self.env_hook = {}551 self.env_hook_exclude = {}552 self.read_env(job_outfile, "hook")553 # Validate the env values with and without hook554 self.common_validate()555 # compare the values in mom_logs as well556 self.common_log_match("mom")557 @checkModule("pexpect")558 def test_interactive(self):559 """560 Test that interactive jobs do not have truncated environment561 variable list with execjob_launch hook562 """563 self.exclude_env += ['happy']564 # submit an interactive job without hook565 cmd = 'env > ' + self.job_out1_tempfile566 a = {ATTR_inter: '', self.ATTR_V: None}567 interactive_script = [('hostname', '.*'), (cmd, '.*')]568 jid = self.create_and_submit_job(569 attribs=a,570 content_interactive=interactive_script,571 preserve_env=True)572 # Once all commands sent and matched, job exits573 self.server.expect(JOB, 'queue', op=UNSET, id=jid, offset=10)574 # read the environment list from the job without hook575 self.env_nohook = {}576 self.env_nohook_exclude = {}577 self.read_env(self.job_out1_tempfile, "nohook")578 # now do the same with the hook579 hook_name = "launch"580 hook_body = """581import pbs582e = pbs.event()583j = e.job584j.Variable_List["happy"] = "days"585pbs.logmsg(pbs.LOG_DEBUG, "Variable_List is %s" % (j.Variable_List,))586"""587 a2 = {'event': "execjob_launch", 'enabled': 'true', 'debug': 'true'}588 self.server.create_import_hook(hook_name, a2, hook_body)589 # submit an interactive job without hook590 cmd = 'env > ' + self.job_out2_tempfile591 interactive_script = [('hostname', '.*'), (cmd, '.*')]592 jid2 = self.create_and_submit_job(593 attribs=a,594 content_interactive=interactive_script,595 preserve_env=True)596 # Once all commands sent and matched, job exits597 self.server.expect(JOB, 'queue', op=UNSET, id=jid2, offset=10)598 # read the environment list from the job without hook599 self.env_hook = {}600 self.env_hook_exclude = {}601 self.read_env(self.job_out2_tempfile, "hook")602 # validate the environment values603 self.common_validate()604 # verify the env values in logs605 self.common_log_match("mom")606 def test_no_hook(self):607 """608 Test to verify that environment variables are609 not truncated and also not modified by PBS when610 no hook is present611 """612 os.environ['BROL'] = 'hii\\\haha'613 os.environ['BROL1'] = """'hii614haa'"""615 a = {'Resource_List.select': '1:ncpus=1',616 'Resource_List.walltime': 10}617 script = ['#PBS -V']618 script += ['env\n']619 script += ['sleep 5\n']620 # Submit a job without hooks in the system621 jid = self.create_and_submit_job(attribs=a, content=script)622 qstat = self.server.status(JOB, id=jid)623 job_outfile = qstat[0]['Output_Path'].split(':')[1]624 job_var = qstat[0]['Variable_List']625 self.logger.info("job variable list is %s" % job_var)626 self.server.expect(JOB, 'queue', op=UNSET, id=jid, offset=10)627 # Read the env variable from job output file628 self.env_nohook = {}629 self.env_nohook_exclude = {}630 self.read_env(job_outfile, "nohook")631 # Verify the output with and without job632 self.assertEqual(os.environ['TEST_COMMA'],633 self.env_nohook['TEST_COMMA'].rstrip('\n'))634 self.logger.info(635 "TEST_COMMA matched - " + os.environ['TEST_COMMA'] +636 " == " + self.env_nohook['TEST_COMMA'].rstrip('\n'))637 self.assertEqual(os.environ['TEST_RETURN'],638 self.env_nohook['TEST_RETURN'].rstrip('\n'))639 self.logger.info(640 "TEST_RETURN matched - " + os.environ['TEST_RETURN'] +641 " == " + self.env_nohook['TEST_RETURN'].rstrip('\n'))642 self.assertEqual(os.environ['TEST_SEMICOLON'],643 self.env_nohook['TEST_SEMICOLON'].rstrip('\n'))644 self.logger.info(645 "TEST_SEMICOLON macthed - " + os.environ['TEST_SEMICOLON'] +646 " == " + self.env_nohook['TEST_SEMICOLON'].rstrip('\n'))647 self.assertEqual(648 os.environ['TEST_ENCLOSED'],649 self.env_nohook['TEST_ENCLOSED'].rstrip('\n'))650 self.logger.info(651 "TEST_ENCLOSED matched - " + os.environ['TEST_ENCLOSED'] +652 " == " + self.env_nohook['TEST_ENCLOSED'].rstrip('\n'))653 self.assertEqual(os.environ['TEST_COLON'],654 self.env_nohook['TEST_COLON'].rstrip('\n'))655 self.logger.info("TEST_COLON macthed - " + os.environ['TEST_COLON'] +656 " == " + self.env_nohook['TEST_COLON'].rstrip('\n'))657 self.assertEqual(658 os.environ['TEST_BACKSLASH'],659 self.env_nohook['TEST_BACKSLASH'].rstrip('\n'))660 self.logger.info(661 "TEST_BACKSLASH matched - " + os.environ['TEST_BACKSLASH'] +662 " == " + self.env_nohook['TEST_BACKSLASH'].rstrip('\n'))663 self.assertEqual(os.environ['TEST_DQUOTE'],664 self.env_nohook['TEST_DQUOTE'].rstrip('\n'))665 self.logger.info("TEST_DQUOTE - " + os.environ['TEST_DQUOTE'] +666 " == " + self.env_nohook['TEST_DQUOTE'].rstrip('\n'))667 self.assertEqual(os.environ['TEST_DQUOTE2'],668 self.env_nohook['TEST_DQUOTE2'].rstrip('\n'))669 self.logger.info("TEST_DQUOTE2 - " + os.environ['TEST_DQUOTE2'] +670 " == " + self.env_nohook['TEST_DQUOTE2'].rstrip('\n'))671 self.assertEqual(os.environ['TEST_DQUOTE3'],672 self.env_nohook['TEST_DQUOTE3'].rstrip('\n'))673 self.logger.info("TEST_DQUOTE3 - " + os.environ['TEST_DQUOTE3'] +674 " == " + self.env_nohook['TEST_DQUOTE3'].rstrip('\n'))675 self.assertEqual(os.environ['TEST_DQUOTE4'],676 self.env_nohook['TEST_DQUOTE4'].rstrip('\n'))677 self.logger.info("TEST_DQUOTE4 - " + os.environ['TEST_DQUOTE4'] +678 " == " + self.env_nohook['TEST_DQUOTE4'].rstrip('\n'))679 self.assertEqual(os.environ['TEST_DQUOTE5'],680 self.env_nohook['TEST_DQUOTE5'].rstrip('\n'))681 self.logger.info("TEST_DQUOTE5 - " + os.environ['TEST_DQUOTE5'] +682 " == " + self.env_nohook['TEST_DQUOTE5'].rstrip('\n'))683 self.assertEqual(os.environ['TEST_DQUOTE6'],684 self.env_nohook['TEST_DQUOTE6'].rstrip('\n'))685 self.logger.info("TEST_DQUOTE6 - " + os.environ['TEST_DQUOTE6'] +686 " == " + self.env_nohook['TEST_DQUOTE6'].rstrip('\n'))687 self.assertEqual(os.environ['TEST_SQUOTE'],688 self.env_nohook['TEST_SQUOTE'].rstrip('\n'))689 self.logger.info("TEST_SQUOTE - " + os.environ['TEST_SQUOTE'] +690 " == " + self.env_nohook['TEST_SQUOTE'].rstrip('\n'))691 self.assertEqual(os.environ['TEST_SQUOTE2'],692 self.env_nohook['TEST_SQUOTE2'].rstrip('\n'))693 self.logger.info("TEST_SQUOTE2 - " + os.environ['TEST_SQUOTE2'] +694 " == " + self.env_nohook['TEST_SQUOTE2'].rstrip('\n'))695 self.assertEqual(os.environ['TEST_SQUOTE3'],696 self.env_nohook['TEST_SQUOTE3'].rstrip('\n'))697 self.logger.info("TEST_SQUOTE3 - " + os.environ['TEST_SQUOTE3'] +698 " == " + self.env_nohook['TEST_SQUOTE3'].rstrip('\n'))699 self.assertEqual(os.environ['TEST_SQUOTE4'],700 self.env_nohook['TEST_SQUOTE4'].rstrip('\n'))701 self.logger.info("TEST_SQUOTE4 - " + os.environ['TEST_SQUOTE4'] +702 " == " + self.env_nohook['TEST_SQUOTE4'].rstrip('\n'))703 self.assertEqual(os.environ['TEST_SQUOTE5'],704 self.env_nohook['TEST_SQUOTE5'].rstrip('\n'))705 self.logger.info("TEST_SQUOTE5 - " + os.environ['TEST_SQUOTE5'] +706 " == " + self.env_nohook['TEST_SQUOTE5'].rstrip('\n'))707 self.assertEqual(os.environ['TEST_SQUOTE6'],708 self.env_nohook['TEST_SQUOTE6'].rstrip('\n'))709 self.logger.info("TEST_SQUOTE6 - " + os.environ['TEST_SQUOTE6'] +710 " == " + self.env_nohook['TEST_SQUOTE6'].rstrip('\n'))711 self.assertEqual(os.environ['BROL'],712 self.env_nohook['BROL'].rstrip('\n'))713 self.logger.info("BROL - " + os.environ['BROL'] + " == " +714 self.env_nohook['BROL'].rstrip('\n'))715 self.assertEqual(os.environ['BROL1'],716 self.env_nohook['BROL1'].rstrip('\n'))717 self.logger.info("BROL - " + os.environ['BROL1'] + " == " +718 self.env_nohook['BROL1'].rstrip('\n'))719 # match the values in qstat -f Variable_List720 # Following is blocked on PTL bug PP-1008721 # self.assertTrue("TEST_COMMA=1\,2\,3\,4" in job_var)722 # self.assertTrue("TEST_SEMICOLON=\;" in job_var)723 # self.assertTrue("TEST_COLON=:" in job_var)724 # self.assertTrue("TEST_DQUOTE=\"" in job_var)725 # self.assertTrue("TEST_SQUOTE=\'" in job_var)726 # self.assertTrue("TEST_BACKSLASH=\\" in job_var)727 # self.assertTrue("BROL=hii\\\\\\haha" in job_var)728 # self.assertTrue("TEST_ENCLOSED=\," in job_var)729 # self.assertTrue("BROL1=hii\nhaa" in job_var)730 # self.assertTrue("TEST_RETURN=3\,\n4\,\n5\," in job_var)731 @checkModule("pexpect")732 def test_interactive_no_hook(self):733 """734 Test to verify that environment variable values735 are not truncated or escaped wrongly whithin a736 job even when there is no hook present737 """738 os.environ['BROL'] = 'hii\\\haha'739 os.environ['BROL1'] = """'hii740haa'"""741 # submit an interactive job without hook742 cmd = 'env > ' + self.job_out3_tempfile743 a = {ATTR_inter: '', self.ATTR_V: None}744 interactive_script = [('hostname', '.*'), (cmd, '.*')]745 jid = self.create_and_submit_job(746 attribs=a,747 content_interactive=interactive_script,748 preserve_env=True)749 # Once all commands sent and matched, job exits750 self.server.expect(JOB, 'queue', op=UNSET, id=jid, offset=10)751 # read the environment list from the job without hook752 self.env_nohook = {}753 self.env_nohook_exclude = {}754 self.read_env(self.job_out3_tempfile, "nohook")755 # Verify the output with and without job756 self.logger.info("job Variable list is ")757 self.assertEqual(os.environ['TEST_COMMA'],758 self.env_nohook['TEST_COMMA'].rstrip('\n'))759 self.logger.info(760 "TEST_COMMA matched - " + os.environ['TEST_COMMA'] +761 " == " + self.env_nohook['TEST_COMMA'].rstrip('\n'))762 self.assertEqual(os.environ['TEST_RETURN'],763 self.env_nohook['TEST_RETURN'].rstrip('\n'))764 self.logger.info(765 "TEST_RETURN matched - " + os.environ['TEST_RETURN'] +766 " == " + self.env_nohook['TEST_RETURN'].rstrip('\n'))767 self.assertEqual(os.environ['TEST_SEMICOLON'],768 self.env_nohook['TEST_SEMICOLON'].rstrip('\n'))769 self.logger.info(770 "TEST_SEMICOLON macthed - " + os.environ['TEST_SEMICOLON'] +771 " == " + self.env_nohook['TEST_SEMICOLON'].rstrip('\n'))772 self.assertEqual(773 os.environ['TEST_ENCLOSED'],774 self.env_nohook['TEST_ENCLOSED'].rstrip('\n'))775 self.logger.info(776 "TEST_ENCLOSED matched - " + os.environ['TEST_ENCLOSED'] +777 " == " + self.env_nohook['TEST_ENCLOSED'].rstrip('\n'))778 self.assertEqual(os.environ['TEST_COLON'],779 self.env_nohook['TEST_COLON'].rstrip('\n'))780 self.logger.info("TEST_COLON macthed - " + os.environ['TEST_COLON'] +781 " == " + self.env_nohook['TEST_COLON'].rstrip('\n'))782 self.assertEqual(783 os.environ['TEST_BACKSLASH'],784 self.env_nohook['TEST_BACKSLASH'].rstrip('\n'))785 self.logger.info(786 "TEST_BACKSLASH matched - " + os.environ['TEST_BACKSLASH'] +787 " == " + self.env_nohook['TEST_BACKSLASH'].rstrip('\n'))788 self.assertEqual(os.environ['TEST_DQUOTE'],789 self.env_nohook['TEST_DQUOTE'].rstrip('\n'))790 self.logger.info("TEST_DQUOTE - " + os.environ['TEST_DQUOTE'] +791 " == " + self.env_nohook['TEST_DQUOTE'].rstrip('\n'))792 self.logger.info("TEST_DQUOTE2 - " + os.environ['TEST_DQUOTE2'] +793 " == " + self.env_nohook['TEST_DQUOTE2'].rstrip('\n'))794 self.logger.info("TEST_DQUOTE3 - " + os.environ['TEST_DQUOTE3'] +795 " == " + self.env_nohook['TEST_DQUOTE3'].rstrip('\n'))796 self.logger.info("TEST_DQUOTE4 - " + os.environ['TEST_DQUOTE4'] +797 " == " + self.env_nohook['TEST_DQUOTE4'].rstrip('\n'))798 self.logger.info("TEST_DQUOTE5 - " + os.environ['TEST_DQUOTE5'] +799 " == " + self.env_nohook['TEST_DQUOTE5'].rstrip('\n'))800 self.logger.info("TEST_DQUOTE6 - " + os.environ['TEST_DQUOTE6'] +801 " == " + self.env_nohook['TEST_DQUOTE6'].rstrip('\n'))802 self.logger.info("TEST_SQUOTE - " + os.environ['TEST_SQUOTE'] +803 " == " + self.env_nohook['TEST_SQUOTE'].rstrip('\n'))804 self.logger.info("TEST_SQUOTE2 - " + os.environ['TEST_SQUOTE2'] +805 " == " + self.env_nohook['TEST_SQUOTE2'].rstrip('\n'))806 self.logger.info("TEST_SQUOTE3 - " + os.environ['TEST_SQUOTE3'] +807 " == " + self.env_nohook['TEST_SQUOTE3'].rstrip('\n'))808 self.logger.info("TEST_SQUOTE4 - " + os.environ['TEST_SQUOTE4'] +809 " == " + self.env_nohook['TEST_SQUOTE4'].rstrip('\n'))810 self.logger.info("TEST_SQUOTE5 - " + os.environ['TEST_SQUOTE5'] +811 " == " + self.env_nohook['TEST_SQUOTE5'].rstrip('\n'))812 self.logger.info("TEST_SQUOTE6 - " + os.environ['TEST_SQUOTE6'] +813 " == " + self.env_nohook['TEST_SQUOTE6'].rstrip('\n'))814 self.assertEqual(os.environ['BROL'],815 self.env_nohook['BROL'].rstrip('\n'))816 self.logger.info("BROL - " + os.environ['BROL'] + " == " +817 self.env_nohook['BROL'].rstrip('\n'))818 self.assertEqual(os.environ['BROL1'],819 self.env_nohook['BROL1'].rstrip('\n'))820 self.logger.info("BROL - " + os.environ['BROL1'] + " == " +821 self.env_nohook['BROL1'].rstrip('\n'))822 def test_execjob_epi2(self):823 """824 Test that Variable_List will contain environment variable825 with commas, newline and all special characters for a job826 that has been recovered from a prematurely killed mom. This827 is a test from an execjob_epilogue hook's view.828 PRE: Set up currently executing user's environment to have variables829 whose values have the special characters.830 Submit a job using the -V option (pass current environment)831 where there is an execjob_epilogue hook that references832 Variable_List value.833 Now kill -9 pbs_mom and then restart it.834 This causes pbs_mom to read in job data from the *.JB file on835 disk, and pbs_mom immediately kills the job causing836 execjob_epilogue hook to execute.837 POST: The epilogue hook should see the proper value to the838 Variable_List.839 """840 a = {'Resource_List.select': '1:ncpus=1',841 'Resource_List.walltime': 60}842 j = Job(attrs=a)843 script = ['#PBS -V']844 script += ['env\n']845 script += ['sleep 30\n']846 j.create_script(body=script)847 # Now create/start the hook848 hook_name = "test_epi"849 hook_body = """850import pbs851import time852e = pbs.event()853j = e.job854pbs.logmsg(pbs.LOG_DEBUG,"Variable_List is %s" % (j.Variable_List,))855pbs.logmsg(pbs.LOG_DEBUG,856 "PBS_O_LOGNAME is %s" % j.Variable_List["PBS_O_LOGNAME"])857"""858 a = {'event': "execjob_epilogue", 'enabled': "true", 'debug': "true"}859 self.server.create_import_hook(860 hook_name,861 a,862 hook_body,863 overwrite=True)864 # Submit a job with hooks in the system865 jid = self.server.submit(j)866 # Wait for the job to start running.867 self.server.expect(JOB, {ATTR_state: 'R'}, id=jid)868 # kill -9 mom869 self.mom.signal('-KILL')870 # now restart mom871 self.mom.start()872 self.mom.log_match("Restart sent to server")873 # Verify the env variables are seen in logs874 self.common_log_match("mom")875 self.mom.log_match(...

Full Screen

Full Screen

urlutils.py

Source:urlutils.py Github

copy

Full Screen

1#!/usr/bin/env python2# -*- coding:utf-8 -*-3http = "http://"4https = "https://"5backslash = "/"6spot = "."7twoSpotBackslash = "../"8spotBackslash = "./"9def getMainUrl(url):10 """11 从url中抽取出站点主url12 Args:13 url:网站url,必须是绝对url14 Returns: 站点主url15 """16 mainUrl = str(url)17 # 如果以反斜杠开头那么它是一个相对路径,无法抽取主url,直接返回默认值18 if mainUrl.find(backslash) == 0:19 return mainUrl20 # 如果以点开头那么它是一个相对路径,无法抽取主url,直接返回默认值21 if url.find(spot) == 0:22 return mainUrl23 # 如果以http://开头,那么对它进行抽取24 if mainUrl.find(http) == 0:25 host = getHost(mainUrl)26 return http + host27 # 如果以https://开头,那么对它进行抽取28 if mainUrl.find(https) == 0:29 host = getHost(mainUrl)30 return https + host31 return mainUrl32def getHost(url):33 mainUrl = str(url)34 # 如果以反斜杠开头那么它是一个相对路径,无法抽取主url,直接返回默认值35 if mainUrl.find(backslash) == 0:36 return mainUrl37 # 如果以点开头那么它是一个相对路径,无法抽取主url,直接返回默认值38 if url.find(spot) == 0:39 return mainUrl40 startIndex = 041 endIndex = -142 # 如果以http://开头,那么对它进行抽取43 if mainUrl.find(http) == 0:44 mainUrl = mainUrl[len(http):len(mainUrl)]45 # 如果以https://开头,那么对它进行抽取46 elif mainUrl.find(https) == 0:47 mainUrl = mainUrl[len(https):len(mainUrl)]48 endIndex = mainUrl.find(backslash)49 if endIndex >= 0:50 mainUrl = mainUrl[startIndex:endIndex]51 return mainUrl52def assembleUrl(srcUrl, newUrl, srcMainUrl=None):53 """54 根据提供的站点主url和祖先srcUrl,把newUrl组装成一个绝对url55 Args:56 srcUrl:newUrl的祖先url57 newUrl:需要组装的url58 srcMainUrl:站点主url59 Returns:60 """61 # 如果以http://开头,那么直接返回不需要组装62 if newUrl.find(http) == 0:63 return newUrl64 # 如果以https://开头,那么直接返回不需要组装65 elif newUrl.find(https) == 0:66 return newUrl67 else:68 mainUrl = ""69 if srcMainUrl != None:70 mainUrl = srcMainUrl71 else:72 mainUrl = getMainUrl(srcUrl)73 resultUrl = newUrl74 index = 075 head = srcUrl76 # 如果以 / 开头 那么在newUrl前面加上mainUrl77 if newUrl.find(backslash) == 0:78 resultUrl = mainUrl + newUrl79 # 如果以../ 或者../../ 等开头的处理80 elif newUrl.find(twoSpotBackslash) == 0:81 count = 082 while 1:83 index = newUrl.find(twoSpotBackslash)84 if index != -1:85 index = index + len(twoSpotBackslash)86 newUrl = newUrl[index:len(newUrl)]87 count = count + 188 else:89 break90 # 去掉url最后比如 / index.html / index..91 end = head.rindex("/")92 head = head[0:end]93 while count > 0:94 end = head.rindex("/")95 head = head[0:end]96 count = count - 197 resultUrl = head + "/" + newUrl98 elif newUrl.find(spotBackslash) == 0:99 newUrl = newUrl.replace(spotBackslash, "/")100 resultUrl = head + newUrl101 else:102 end = head.rindex("/")103 head = head[0:end]104 resultUrl = head + "/" + newUrl105 return resultUrl106if __name__ == "__main__":107 url = "http://www.cnblogs.com/mrchige/p/6371783.html"108 mainUrl = getMainUrl(url)109 print mainUrl110 srcUrl = "http://www.cnblogs.com/mrchige/p/6371783.html"111 findUrl = "/test_backslash.html"112 result = "http://www.cnblogs.com/test_backslash.html"113 newUrl = assembleUrl(srcUrl, findUrl)114 print newUrl115 print newUrl == result116 srcUrl = "http://www.cnblogs.com/mrchige/p/6371783.html"117 findUrl = "../test_backslash.html"118 result = "http://www.cnblogs.com/mrchige/test_backslash.html"119 newUrl = assembleUrl(srcUrl, findUrl)120 print newUrl121 print newUrl == result122 srcUrl = "http://www.cnblogs.com/mrchige/p/6371783.html"123 findUrl = "../../test_backslash.html"124 result = "http://www.cnblogs.com/test_backslash.html"125 newUrl = assembleUrl(srcUrl, findUrl)126 print newUrl127 print newUrl == result128 srcUrl = "http://www.cnblogs.com/mrchige/p/6371783.html"129 findUrl = "./test_backslash.html"130 result = "http://www.cnblogs.com/mrchige/p/6371783.html/test_backslash.html"131 newUrl = assembleUrl(srcUrl, findUrl)132 print newUrl133 print newUrl == result134 srcUrl = "http://www.cnblogs.com/mrchige/p/6371783.html"135 findUrl = "test_backslash.html"136 result = "http://www.cnblogs.com/mrchige/p/test_backslash.html"137 newUrl = assembleUrl(srcUrl, findUrl)138 print newUrl...

Full Screen

Full Screen

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.

LambdaTest Learning Hubs:

YouTube

You could also refer to video tutorials over LambdaTest YouTube channel to get step by step demonstration from industry experts.

Run autotest automation tests on LambdaTest cloud grid

Perform automation testing on 3000+ real desktop and mobile devices online.

Try LambdaTest Now !!

Get 100 minutes of automation test minutes FREE!!

Next-Gen App & Browser Testing Cloud

Was this article helpful?

Helpful

NotHelpful