Test your AI Agents with the all-new Agent to Agent Testing Platform.Learn More
How to use run_output method in autotest
Best Python code snippet using autotest_python
test-fetchmail.py
Source:test-fetchmail.py 
1#!/usr/bin/python2#3# fetchmail.py quality assurance test script4# Copyright (C) 2008 Canonical Ltd.5#6# This program is free software: you can redistribute it and/or modify7# it under the terms of the GNU General Public License version 2,8# as published by the Free Software Foundation.9#10# This program is distributed in the hope that it will be useful,11# but WITHOUT ANY WARRANTY; without even the implied warranty of12# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the13# GNU General Public License for more details.14#15# You should have received a copy of the GNU General Public License16# along with this program. If not, see <http://www.gnu.org/licenses/>.17#18'''19 How to run against a clean schroot named 'edgy':20 schroot -c edgy -u root -- sh -c 'apt-get -y install fetchmail netbase dovecot-imapd dovecot-pop3d python-openssl ssl-cert && ./test-fetchmail.py -v'21 # Dapper uses python-pyopenssl instead...22'''23# QRT-Depends: testlib_dovecot.py fetchmail24# QRT-Packages: fetchmail netbase dovecot-imapd dovecot-pop3d ssl-cert25# QRT-Alternates: python-openssl python-pyopenssl26# QRT-Privilege: root27import unittest, subprocess, tempfile, os, re, shutil28import SocketServer29import testlib30import testlib_dovecot31class ReuseTCPServer(SocketServer.TCPServer):32 def __init__(self,*args):33 self.allow_reuse_address = True34 self.imap_capabilities = ''35 self.test_result = 'Failure: never connected'36 SocketServer.TCPServer.__init__(self,*args)37 def set_capabilities(self,capa):38 self.imap_capabilities = capa39 def get_capabilities(self):40 return self.imap_capabilities41 def set_test_result(self, result):42 self.test_result = result43 def get_test_result(self):44 return self.test_result45class ImapHandler(SocketServer.StreamRequestHandler):46 def handle(self):47 self.wfile.write("* OK TestRig ready.\r\n")48 self.default_result()49 self.imap_running = 150 while self.imap_running:51 prefix = cmd = ''52 parts = self.rfile.readline(512).strip().split(' ')53 try:54 prefix = parts.pop(0)55 cmd = parts.pop(0)56 except:57 pass58 if cmd == 'NOOP' or cmd == '*':59 self.wfile.write('%s OK That was hard work!\r\n' % prefix)60 elif cmd == 'CAPABILITY':61 self.wfile.write('* CAPABILITY IMAP4rev1 %s\r\n' % self.server.get_capabilities())62 self.wfile.write('%s OK Capability completed.\r\n' % prefix)63 elif cmd == 'AUTHENTICATE':64 self.wfile.write('%s BAD Not supported.\r\n' % prefix)65 elif cmd == '' or cmd == 'LOGOUT':66 # Empty cmd seems to be the "EOF" mode67 self.imap_running = 068 else:69 self.imap_cmd(prefix,cmd,parts)70# When running without --sslproto TLS1:71# * OK Dovecot ready.72# A0001 CAPABILITY73# * CAPABILITY IMAP4rev1 STARTTLS AUTH=PLAIN74# $1 OK Capability completed.75# A0002 STARTTLS76# $1 FAIL I tricked you77# A0003 NOOP78# $1 OK79# A0004 LOGIN "..." "..."80class ImapAllowAuthFallback(ImapHandler):81 def default_result(self):82 # Our last command should be "LOGIN", so everything else is a fail83 self.server.set_test_result('Failure: LOGIN not the last command')84 85 def imap_cmd(self,prefix,cmd,args):86 if cmd == 'STARTTLS':87 self.wfile.write('%s BAD I lied about the TLS.\r\n' % prefix)88 elif cmd == 'LOGIN':89 self.server.set_test_result('Success: LOGIN seen after STARTTLS failure')90 self.wfile.write('%s BAD Test finished.\r\n' % prefix)91 self.imap_running = 092 else:93 self.wfile.write('%s BAD Go away.\r\n' % prefix)94 self.imap_running = 095# When running with and without --sslproto TLS1:96# * OK Dovecot ready.97# A0001 CAPABILITY98# * CAPABILITY IMAP4rev1 STARTTLS LOGINDISABLED99# $1 OK Capability completed.100# A0002 STARTTLS101# $1 FAIL I tricked you102# ->CLOSE103#104# When running with --sslproto TLS1:105# * OK Dovecot ready.106# A0001 CAPABILITY107# * CAPABILITY IMAP4rev1 STARTTLS AUTH=PLAIN108# $1 OK Capability completed.109# A0002 STARTTLS110# $1 FAIL I tricked you111# ->CLOSE112class ImapDisallowAuthFallback(ImapHandler):113 def default_result(self):114 # Our last command should not be "LOGIN", so everything else is okay115 self.server.set_test_result('Success: did not LOGIN')116 def imap_cmd(self,prefix,cmd,args):117 if cmd == 'STARTTLS':118 self.wfile.write('%s BAD I lied about the TLS.\r\n' % prefix)119 elif cmd == 'LOGIN':120 self.server.set_test_result('Failure: LOGIN seen after STARTTLS failure')121 self.wfile.write('%s BAD Test finished.\r\n' % prefix)122 self.imap_running = 0123 else:124 self.server.set_test_result('Failure: unexpected cmd "%s"' % cmd)125 self.wfile.write('%s BAD Go away.\r\n' % prefix)126 self.imap_running = 0127class FetchmailAuth(unittest.TestCase):128 '''Test fetchmail authentication behavior (CVE-2006-5867).'''129 def setUp(self):130 tmpfd, self.tmppath = tempfile.mkstemp(prefix='fetchmail-test')131 def replace_tmpfile(self,cfg):132 self.tmpfile = file(self.tmppath,"w")133 self.tmpfile.write(cfg)134 self.tmpfile.close()135 check = file(self.tmppath,"r").read()136 self.assertEquals(check,cfg)137 def tearDown(self):138 os.unlink(self.tmppath)139 def _run_test(self,handler,capabilities,fetchmailrc,default_result='Failure: never ran'):140 server = ReuseTCPServer( ('', 1143), handler)141 server.set_capabilities(capabilities)142 server.set_test_result(default_result)143 self.replace_tmpfile(fetchmailrc);144 sp = subprocess.Popen(['fetchmail','-vk','-f',self.tmppath], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True)145 try:146 testlib.timeout(2,server.handle_request)147 finally:148 # get state of test149 report = server.get_test_result()150 # report state of fetchmail too151 out = sp.communicate(None)[0]152 # release server153 server = None154 # add fetchmail report on failure155 if report.find("Success") != 0:156 report +='\n%s' % out157 self.assertEquals(report.find("Success"),0,report)158 def test_auth_no_fallback_plain(self):159 '''Test [server: broken STARTTLS, LOGINDISABLED] [client: allow plaintext]'''160 self._run_test(ImapDisallowAuthFallback,161 'STARTTLS LOGINDISABLED', 162 '''163poll localhost protocol IMAP port 1143:164 user nobody password nothing165''')166 def test_auth_no_fallback_tls_required(self):167 '''Test [server: broken STARTTLS, LOGINDISABLED] [client: require TLS]'''168 self._run_test(ImapDisallowAuthFallback,169 'STARTTLS LOGINDISABLED', 170 '''171poll localhost protocol IMAP port 1143:172 user nobody password nothing173 sslproto TLS1174''')175 def test_auth_plain_fallback_allowed(self):176 '''Test [server: broken STARTTLS, AUTH=PLAIN] [client: allow plaintext]'''177 self._run_test(ImapAllowAuthFallback,178 'STARTTLS AUTH=PLAIN',179 '''180poll localhost protocol IMAP port 1143:181 user nobody password nothing182''')183 # CVE-2006-5867, Issue 2184 def test_auth_plain_fallback_disallowed(self):185 '''Test [server: broken STARTTLS, AUTH=PLAIN] [client: need TLS]'''186 self._run_test(ImapDisallowAuthFallback,187 'STARTTLS AUTH=PLAIN', 188 '''189poll localhost protocol IMAP port 1143:190 user nobody password nothing191 sslproto TLS1192''')193 # CVE-2006-5867, Issue 1194 def test_auth_plain_fallback_disallowed_sslcertck(self):195 '''Test [server: broken STARTTLS, AUTH=PLAIN] [client: need Cert]'''196 self._run_test(ImapDisallowAuthFallback,197 'STARTTLS AUTH=PLAIN', 198 '''199poll localhost protocol IMAP port 1143:200 user nobody password nothing201 sslcertck202''')203 # CVE-2006-5867, Issue 1204 def test_auth_plain_fallback_disallowed_sslfingerprint(self):205 '''Test [server: broken STARTTLS, AUTH=PLAIN] [client: need fingerprint]'''206 self._run_test(ImapDisallowAuthFallback,207 'STARTTLS AUTH=PLAIN', 208 '''209poll localhost protocol IMAP port 1143:210 user nobody password nothing211 sslfingerprint "DE:AD:BE:EF:00:00:00:00:00:00:00:00:00:00:00:00"212''')213 # CVE-2006-5867, Issue 4 (broken behavior not reproduced)214 def test_auth_plain_fallback_disallowed_gssapi(self):215 '''Test [server: broken STARTTLS, AUTH=PLAIN] [client: require MD5]'''216 self._run_test(ImapDisallowAuthFallback,217 'STARTTLS AUTH=PLAIN', 218 '''219poll localhost protocol IMAP port 1143 auth cram-md5:220 user nobody password nothing221''')222class FetchmailDovecot(testlib.TestlibCase):223 '''Helper functions for fetchmail/dovecot testing'''224 def setUp(self):225 tmpfd, self.tmppath = tempfile.mkstemp(prefix='fetchmail-test')226 self.user = testlib.TestUser()227 self.dovecot = testlib_dovecot.Dovecot(self, self.user)228 self.hostname = self.yank_commonname_from_cert(self.dovecot.get_cert())229 def replace_tmpfile(self,contents):230 self.tmpfile = file(self.tmppath,"w")231 self.tmpfile.write(contents)232 self.tmpfile.close()233 check = file(self.tmppath,"r").read()234 self.assertEquals(check,contents)235 def tearDown(self):236 self.dovecot = None237 self.user = None238 os.unlink(self.tmppath)239 def _run_fetchmail(self):240 sp = subprocess.Popen(['fetchmail','-vf',self.tmppath], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True)241 out = sp.communicate(None)[0]242 # If we didn't connect, the server didn't start up, try to figure243 # out why...244 if out.find('Connection refused')>0:245 out += file("/etc/dovecot/dovecot.conf","r").read()246 return out247 def _fetch_proto(self,protocol,user_opts='',server_opts='',badlogin_test=True):248 '''Test a specific protocol with options'''249 cfg = 'poll %s protocol %s ' + server_opts + '''250 user %s password %s ''' + user_opts + '''251 mda "cat > %s"252'''253 # This is slow due to the PAM delays, so make it configurable254 if badlogin_test:255 # Bad username256 self.replace_tmpfile(cfg % (self.hostname, protocol, 'nosuchuser', self.user.password, self.tmppath))257 out = self._run_fetchmail()258 self.assertTrue(out.find('fetchmail: Authorization failure')>=0 or259 'fetchmail: Query status=3 (AUTHFAIL)'>=0,out)260 # Bad password261 self.replace_tmpfile(cfg % (self.hostname, protocol, self.user.login, 'badpassword', self.tmppath))262 out = self._run_fetchmail()263 self.assertTrue(out.find('fetchmail: Authorization failure')>=0 or264 'fetchmail: Query status=3 (AUTHFAIL)'>=0,out)265 # Empty password266 self.replace_tmpfile(cfg % (self.hostname, protocol, self.user.login, '""', self.tmppath))267 out = self._run_fetchmail()268 self.assertTrue(out.find('fetchmail: Authorization failure')>=0 or269 'fetchmail: Query status=3 (AUTHFAIL)'>=0,out)270 # Good password271 self.replace_tmpfile(cfg % (self.hostname, protocol, self.user.login, self.user.password, self.tmppath))272 self.run_output = self._run_fetchmail()273 self.run_email = file(self.tmppath,"r").read()274 def _check_common(self):275 '''Check the run output for things specific to all behaviors'''276 self.assertTrue(self.run_output.find('2 messages')>=0,self.run_output)277 self.assertTrue(self.run_output.find('reading message')>=0,self.run_output)278 self.assertTrue(self.run_output.find('flushed')>=0,self.run_output)279 self.assertTrue(self.run_email.find('Received: from %s' % (self.hostname))==0,self.run_email)280 def _check_pop(self):281 '''Check the run output for things specific to POP behavior'''282 self._check_common()283 self.assertTrue(self.run_email.find('Date: Tue, 28 Nov 2006 11:29:34 +0100\nFrom: Test User 2 <test2@test2.com>\nTo: Dovecot tester <dovecot@test.com>\nSubject: Test 2\n')>0 and self.run_email.find('\n\nMore news.\n\nGet cracking!\n')>0,self.run_email)284 def _check_imap(self):285 '''Check the run output for things specific to IMAP behavior'''286 self._check_common()287 self.assertTrue(self.run_output.find('skipping message')>=0,self.run_output)288 self.assertTrue(self.run_email.find('Date: Thu, 16 Nov 2006 17:12:23 -0800\nFrom: Test User 1 <test1@test1.com>\nTo: Dovecot tester <dovecot@test.com>\nSubject: Test 1\n')>0 and self.run_email.find('\n\nSome really important news.\n')>0,self.run_email)289 def _check_cert_cn(self, expected=True):290 state = re.search(r'fetchmail: (Server|Issuer) CommonName: %s' % (self.hostname), self.run_output) != None291 self.assertEquals(state, expected, self.run_output)292class FetchmailDovecot00(FetchmailDovecot):293 '''Test basic fetchmail behavior with dovecot.'''294 def test_00_hashes(self):295 '''Rebuild PEM hashes'''296 # regenerate PEM hashes297 self.assertShellExitEquals(0, ["c_rehash"])298 def test_fetch_imap(self):299 '''Test IMAP fetching'''300 self._fetch_proto('IMAP','sslproto ""')301 self._check_imap()302 self._check_cert_cn(False)303 def test_fetch_imap_tls(self):304 '''Test IMAP TLS fetching'''305 self._fetch_proto('IMAP', 'sslcertck sslcertpath "/etc/ssl/certs"')306 self._check_imap()307 self._check_cert_cn()308 self._fetch_proto('IMAP', 'sslcertck sslcertpath "/dev/null"')309 self.assertTrue(self.run_output.find('fetchmail: Server certificate verification error: self signed certificate')>=0,self.run_output)310 def test_fetch_imaps(self):311 '''Test IMAP SSL fetching'''312 self._fetch_proto('IMAP','ssl sslcertck sslcertpath "/etc/ssl/certs"')313 self._check_imap()314 self._check_cert_cn()315 self._fetch_proto('IMAP', 'ssl sslcertck sslcertpath "/dev/null"')316 self.assertTrue(self.run_output.find('fetchmail: Server certificate verification error: self signed certificate')>=0,self.run_output)317 def test_ssl_fingerprint_good(self):318 '''Test IMAP TLS fetching with good SSL fingerprint'''319 self._fetch_proto('IMAP',user_opts='sslfingerprint "%s"' % self.dovecot.get_ssl_fingerprint())320 self._check_imap()321 self._check_cert_cn()322 def test_ssl_fingerprint_bad(self):323 '''Test IMAP TLS fetching with bad SSL fingerprint'''324 self._fetch_proto('IMAP',server_opts='auth password',user_opts='sslfingerprint "DE:AD:BE:EF:00:00:00:00:00:00:00:00:00:00:00:00"')325 self._check_cert_cn()326 self.assertTrue(self.run_output.find('fetchmail: %s fingerprints do not match' % (self.hostname))>=0,self.run_output)327 self.assertTrue(self.run_output.find('2 messages')<0,self.run_output)328# Not even dovecot wants to do POP2329# # CVE-2006-5867, Issue 5330# def test_fetch_pop2(self):331# '''Test POP2 fetching'''332# self._fetch_proto('POP2','sslproto ""')333# self._check_pop()334# self.assertTrue(self.run_output.find('fetchmail: Server CommonName')<0,self.run_output)335 def test_fetch_pop3(self):336 '''Test POP3 fetching'''337 self._fetch_proto('POP3','sslproto ""')338 self._check_pop()339 self._check_cert_cn(False)340 # CVE-2006-5867, Issue 3341 def test_fetch_pop3_nocapa(self):342 '''Test POP3 fetching TLS upgrade without Capabilities (CVE-2006-5867)'''343 self._fetch_proto('POP3',server_opts='auth cram-md5', user_opts='sslcertck sslcertpath "/etc/ssl/certs"')344 self._check_pop()345 self._check_cert_cn()346 # CVE-2006-5867, Issue 3347 def test_fetch_pop3_nocapa_tls(self):348 '''Test POP3 fetching TLS required without Capabilities (CVE-2006-5867)'''349 self._fetch_proto('POP3',server_opts='auth cram-md5', user_opts='sslproto tls1 sslcertck sslcertpath "/etc/ssl/certs"')350 self._check_pop()351 self._check_cert_cn()352 def test_fetch_pop3s_tls(self):353 '''Test POP3 TLS fetching'''354 self._fetch_proto('POP3', 'sslcertck sslcertpath "/etc/ssl/certs"')355 self._check_pop()356 self._check_cert_cn()357 self._fetch_proto('POP3', 'sslcertck sslcertpath "/dev/null"')358 self.assertTrue(self.run_output.find('fetchmail: Server certificate verification error: self signed certificate')>=0,self.run_output)359 def test_fetch_pop3s(self):360 '''Test POP3 SSL fetching'''361 self._fetch_proto('POP3','ssl sslcertck sslcertpath "/etc/ssl/certs"')362 self._check_pop()363 self._check_cert_cn()364 self._fetch_proto('POP3', 'ssl sslcertck sslcertpath "/dev/null"')365 self.assertTrue(self.run_output.find('fetchmail: Server certificate verification error: self signed certificate')>=0,self.run_output)366class FetchmailDovecot01(FetchmailDovecot):367 '''Test goofy SSL certs with fetchmail/dovecot'''368 def setUp(self):369 tmpfd, self.tmppath = tempfile.mkstemp(prefix='fetchmail-test')370 self.user = testlib.TestUser()371 # use NULL-byte certs372 self.certs = tempfile.mkdtemp(prefix='dovecot-certs-')373 self.cert_pub = self.certs + '/public.pem'374 self.cert_key = self.certs + '/private.pem'375 self.assertShellExitEquals(0, ['fetchmail/null-snakeoil.py',self.cert_pub,self.cert_key])376 self.dovecot = testlib_dovecot.Dovecot(self, self.user, cert_pub=self.cert_pub, cert_key=self.cert_key)377 self.hostname = self.yank_commonname_from_cert(self.dovecot.get_cert()).split('\x00')[0]378 def tearDown(self):379 self.dovecot = None380 self.user = None381 os.unlink(self.tmppath)382 shutil.rmtree(self.certs)383 def test_fetch_imap_tls(self):384 '''Test IMAP SSL fetching rejects NULL-byte CN (CVE-2009-2666)'''385 # cert check386 self._fetch_proto('IMAP', 'sslproto tls1')387 self._check_cert_cn()388 self.assertTrue(self.run_output.find('fetchmail: Bad certificate: Subject CommonName contains NUL, aborting')>=0,self.run_output)389if __name__ == '__main__':390 testlib.require_root()...
run.py
Source:run.py
1# -*- coding: utf-8 -*-2import os, argparse3# python run.py lib/ctms/reference.ctm index/reference.json lib/kws/queries.xml run_output/reference_out.xml run_scoring/reference_out --n_best 100 --alpha 1 --gamma 14# python run.py lib/ctms/decode.ctm index/decode.json lib/kws/queries.xml run_output/decode_out.xml run_scoring/decode_out --n_best 100 --alpha 1 --gamma 15# python run.py lib/ctms/decode-morph.ctm index/decode-morph.json output/queries_morph.kwslist.xml run_output/decode-morph_out.xml run_scoring/decode-morph_out --n_best 100 --alpha 1 --gamma 16# python run.py output/reference_morph.ctm index/reference_morph.json output/queries_morph.kwslist.xml run_output/reference_morph_out.xml run_scoring/reference_morph_out 7# python run.py output/decode_morph.ctm index/decode_morph.json output/queries_morph.kwslist.xml run_output/decode_morph_out.xml run_scoring/decode_morph_out 8parser = argparse.ArgumentParser()9parser.add_argument('ctm_file', help="ASR output file (.ctm)")10parser.add_argument('index_file', help="Index file (.ctm)")11parser.add_argument('queries_file', help="Queries file (.xml)")12parser.add_argument('output_file', help="Output hits file (.xml)")13parser.add_argument('scoring_dir', help="Scoring directory")14parser.add_argument('--n_best', help="Number of alternatives for OOV")15parser.add_argument('--alpha', help="weight of alternative query score")16parser.add_argument('--gamma', help="exponent for score normalization")17parser.add_argument('--rerun', help='boolean on whether compositions should be reperformed')18args = parser.parse_args()19ctm_file = args.ctm_file20index_file = args.index_file21queries_file = args.queries_file22output_file = args.output_file23scoring_dir = args.scoring_dir24n_best = " --n_best "+args.n_best if args.n_best else ""25alpha = " --alpha "+args.alpha if args.alpha else ""26gamma = " --gamma "+args.gamma if args.gamma else ""27rerun = " --rerun "+args.rerun if args.rerun else ""28os.system('python indexing.py %s %s' % (ctm_file, index_file))29os.system('python grapheme_confusion.py %s' % index_file)30print 'Done indexing and building grapheme confusion FST'31os.system('python querying.py %s %s %s %s %s %s %s' % (index_file, queries_file, output_file, n_best, alpha, gamma, rerun)) 32print 'Done querying'33os.system('rm -r %s' % (scoring_dir))34os.system('./scripts/score.sh %s %s' % (output_file, scoring_dir))35s_all = os.popen('./scripts/termselect.sh lib/terms/ivoov.map %s %s all' % (output_file, scoring_dir)).read()36s_iv = os.popen('./scripts/termselect.sh lib/terms/ivoov.map %s %s iv' % (output_file, scoring_dir)).read()37s_oov = os.popen('./scripts/termselect.sh lib/terms/ivoov.map %s %s oov' % (output_file, scoring_dir)).read()38TWVs = []39numbers = []40for s in [s_all, s_iv, s_oov]:41 s = s.split('=')42 TWV, threshold, number = [x.split(' ')[0]for x in s[1:]]43 TWVs.append(TWV)44 numbers.append(number)45numbers = [int(number) for number in numbers]46TWVs = [float(TWV) for TWV in TWVs]47threshold = float(threshold)48print index_file, n_best, alpha, gamma49print 'TWVs = \t%.6f\t%.6f\t%.6f' % (TWVs[0], TWVs[1], TWVs[2])50print 'threshold = %f' % threshold51print 'ALL = \t%df\t%d\t%d' % (numbers[0], numbers[1], numbers[2])52# part 353# python run.py lib/ctms/reference.ctm index/reference.json lib/kws/queries.xml run_output/reference_out.xml run_scoring/reference_out 54# python run.py output/reference_morph.ctm index/reference_morph.json output/queries_morph.kwslist.xml run_output/reference_morph_out.xml run_scoring/reference_morph_out 55# python run.py lib/ctms/decode.ctm index/decode.json lib/kws/queries.xml output/decode_nograph_out.xml scoring/decode_nograph_out 56# python run.py output/decode_morph.ctm index/decode_morph.json output/queries_morph.kwslist.xml run_output/decode_morph_out.xml run_scoring/decode_morph_out 57# python run.py lib/ctms/decode-morph.ctm index/decode-morph.json output/queries_morph.kwslist.xml run_output/decode-morph_out.xml run_scoring/decode-morph_out 58# part 459# python run.py lib/ctms/decode.ctm index/decode.json lib/kws/queries.xml run_output/decode_out.xml run_scoring/decode_out --gamma 0.660# python run.py lib/ctms/decode.ctm index/decode.json lib/kws/queries.xml run_output/decode_out.xml run_scoring/decode_out --gamma 161# python run.py lib/ctms/decode.ctm index/decode.json lib/kws/queries.xml run_output/decode_out.xml run_scoring/decode_out --gamma 1.562# python run.py lib/ctms/decode.ctm index/decode.json lib/kws/queries.xml run_output/decode_out.xml run_scoring/decode_out --gamma 263# python run.py lib/ctms/decode.ctm index/decode.json lib/kws/queries.xml run_output/decode_out.xml run_scoring/decode_out --gamma 2.564# python run.py lib/ctms/decode.ctm index/decode.json lib/kws/queries.xml run_output/decode_out.xml run_scoring/decode_out --gamma 365# python run.py lib/ctms/decode.ctm index/decode.json lib/kws/queries.xml run_output/decode_out.xml run_scoring/decode_out --gamma 3.566# python run.py lib/ctms/decode.ctm index/decode.json lib/kws/queries.xml run_output/decode_out.xml run_scoring/decode_out --gamma 467# python run.py lib/ctms/decode.ctm index/decode.json lib/kws/queries.xml run_output/decode_out.xml run_scoring/decode_out --gamma 568# python run.py lib/ctms/decode-morph.ctm index/decode-morph.json output/queries_morph.kwslist.xml run_output/decode-morph_out.xml run_scoring/decode-morph_out --gamma 0.6 69# python run.py lib/ctms/decode-morph.ctm index/decode-morph.json output/queries_morph.kwslist.xml run_output/decode-morph_out.xml run_scoring/decode-morph_out --gamma 170# python run.py lib/ctms/decode-morph.ctm index/decode-morph.json output/queries_morph.kwslist.xml run_output/decode-morph_out.xml run_scoring/decode-morph_out --gamma 1.571# python run.py lib/ctms/decode-morph.ctm index/decode-morph.json output/queries_morph.kwslist.xml run_output/decode-morph_out.xml run_scoring/decode-morph_out --gamma 2 72# python run.py lib/ctms/decode-morph.ctm index/decode-morph.json output/queries_morph.kwslist.xml run_output/decode-morph_out.xml run_scoring/decode-morph_out --gamma 3 73# python run.py lib/ctms/decode-morph.ctm index/decode-morph.json output/queries_morph.kwslist.xml run_output/decode-morph_out.xml run_scoring/decode-morph_out --gamma 574# part 575# python run.py lib/ctms/decode.ctm index/decode.json lib/kws/queries.xml run_output/decode_out.xml run_scoring/decode_out --n_best 1 --alpha 1 --gamma 2 # 0.36038376# python run.py lib/ctms/decode.ctm index/decode.json lib/kws/queries.xml run_output/decode_out.xml run_scoring/decode_out --n_best 2 --alpha 1 --gamma 2 # 0.36228777# python run.py lib/ctms/decode.ctm index/decode.json lib/kws/queries.xml run_output/decode_out.xml run_scoring/decode_out --n_best 3 --alpha 1 --gamma 2Â #Â 0.36500378# python run.py lib/ctms/decode.ctm index/decode.json lib/kws/queries.xml run_output/decode_out.xml run_scoring/decode_out --n_best 5 --alpha 1 --gamma 2 # 0.36264979# python run.py lib/ctms/decode.ctm index/decode.json lib/kws/queries.xml run_output/decode_out.xml run_scoring/decode_out --n_best 10 --alpha 1 --gamma 2 # 0.36205780# python run.py lib/ctms/decode.ctm index/decode.json lib/kws/queries.xml run_output/decode_out.xml run_scoring/decode_out --n_best 25 --alpha 1 --gamma 2 # 0.36160081# python run.py lib/ctms/decode.ctm index/decode.json lib/kws/queries.xml run_output/decode_out.xml run_scoring/decode_out --n_best 50 --alpha 1 --gamma 282# python run.py lib/ctms/decode.ctm index/decode.json lib/kws/queries.xml run_output/decode_out.xml run_scoring/decode_out --n_best 100 --alpha 1 --gamma 283# python run.py lib/ctms/decode.ctm index/decode.json lib/kws/queries.xml run_output/decode_out.xml run_scoring/decode_out --n_best 250 --alpha 1 --gamma 284# python run.py lib/ctms/decode.ctm index/decode.json lib/kws/queries.xml run_output/decode_out.xml run_scoring/decode_out --n_best 500 --alpha 1 --gamma 285# python run.py lib/ctms/decode.ctm index/decode.json lib/kws/queries.xml run_output/decode_out.xml run_scoring/decode_out --n_best 750 --alpha 1 --gamma 286# python run.py lib/ctms/decode.ctm index/decode.json lib/kws/queries.xml run_output/decode_out.xml run_scoring/decode_out --n_best 1000 --alpha 1 --gamma 287# python run.py lib/ctms/decode.ctm index/decode.json lib/kws/queries.xml run_output/decode_out.xml run_scoring/decode_out --n_best 1500 --alpha 1 --gamma 288####--> best is n_best = 189# python run.py lib/ctms/decode.ctm index/decode.json lib/kws/queries.xml run_output/decode_out.xml run_scoring/decode_out --n_best 1 --alpha 0.01 --gamma 290# python run.py lib/ctms/decode.ctm index/decode.json lib/kws/queries.xml run_output/decode_out.xml run_scoring/decode_out --n_best 1 --alpha 0.1 --gamma 291# python run.py lib/ctms/decode.ctm index/decode.json lib/kws/queries.xml run_output/decode_out.xml run_scoring/decode_out --n_best 1 --alpha 1 --gamma 292###-> alpha does not seem to matter if we score normalize...93# python run.py lib/ctms/decode-morph.ctm index/decode-morph.json output/queries_morph.kwslist.xml run_output/decode-morph_out.xml run_scoring/decode-morph_out --gamma 1.75 --n_best 1 --alpha 194# python run.py lib/ctms/decode-morph.ctm index/decode-morph.json output/queries_morph.kwslist.xml run_output/decode-morph_out.xml run_scoring/decode-morph_out --gamma 1.75 --n_best 2 --alpha 195# python run.py lib/ctms/decode-morph.ctm index/decode-morph.json output/queries_morph.kwslist.xml run_output/decode-morph_out.xml run_scoring/decode-morph_out --gamma 1.75 --n_best 5 --alpha 196# python run.py lib/ctms/decode-morph.ctm index/decode-morph.json output/queries_morph.kwslist.xml run_output/decode-morph_out.xml run_scoring/decode-morph_out --gamma 1.75 --n_best 10 --alpha 197# python run.py lib/ctms/decode-morph.ctm index/decode-morph.json output/queries_morph.kwslist.xml run_output/decode-morph_out.xml run_scoring/decode-morph_out --gamma 1.75 --n_best 50 --alpha 198# python run.py lib/ctms/decode-morph.ctm index/decode-morph.json output/queries_morph.kwslist.xml run_output/decode-morph_out.xml run_scoring/decode-morph_out --gamma 1.75 --n_best 100 --alpha 199# python run.py lib/ctms/decode-morph.ctm index/decode-morph.json output/queries_morph.kwslist.xml run_output/decode-morph_out.xml run_scoring/decode-morph_out --gamma 1.75 --n_best 1 --alpha 0.1100# python run.py lib/ctms/decode-morph.ctm index/decode-morph.json output/queries_morph.kwslist.xml run_output/decode-morph_out.xml run_scoring/decode-morph_out --gamma 1.75 --n_best 1 --alpha 1101# python run.py lib/ctms/decode-morph.ctm index/decode-morph.json output/queries_morph.kwslist.xml run_output/decode-morph_out.xml run_scoring/decode-morph_out --gamma 1.75 --n_best 1 --alpha 10102# python run.py lib/ctms/decode.ctm index/decode.json lib/kws/queries.xml run_output/decode_out.xml run_scoring/decode_out --n_best 750 --alpha 0.5 --gamma 0.5 -->0.363833103# python run.py lib/ctms/decode.ctm index/decode.json lib/kws/queries.xml run_output/decode_out.xml run_scoring/decode_out --n_best 750 --alpha 0.5 --gamma 0.75 -->0.374368104# python run.py lib/ctms/decode.ctm index/decode.json lib/kws/queries.xml run_output/decode_out.xml run_scoring/decode_out --n_best 750 --alpha 0.5 --gamma 1 -->0.382017105# python run.py lib/ctms/decode.ctm index/decode.json lib/kws/queries.xml run_output/decode_out.xml run_scoring/decode_out --n_best 750 --alpha 0.5 --gamma 1.5 -->0.373239106# python run.py lib/ctms/decode.ctm index/decode.json lib/kws/queries.xml run_output/decode_out.xml run_scoring/decode_out --n_best 750 --alpha 0.5 --gamma 2 -->0.367792107# python run.py lib/ctms/decode.ctm index/decode.json lib/kws/queries.xml run_output/decode_out.xml run_scoring/decode_out --n_best 750 --alpha 0.5 --gamma 3 -->0.358182108# python run.py lib/ctms/decode.ctm index/decode.json lib/kws/queries.xml run_output/decode_out.xml run_scoring/decode_out --n_best 750 --alpha 0.5 --gamma 0.5 ->0.363833109# python run.py lib/ctms/decode.ctm index/decode.json lib/kws/queries.xml run_output/decode_out.xml run_scoring/decode_out --n_best 750 --alpha 0.75 --gamma 0.5 ->0.374295110# python run.py lib/ctms/decode.ctm index/decode.json lib/kws/queries.xml run_output/decode_out.xml run_scoring/decode_out --n_best 750 --alpha 1 --gamma 0.5 111# python run.py lib/ctms/decode.ctm index/decode.json lib/kws/queries.xml run_output/decode_out.xml run_scoring/decode_out --n_best 750 --alpha 1.5 --gamma 0.5 112# python run.py lib/ctms/decode.ctm index/decode.json lib/kws/queries.xml run_output/decode_out.xml run_scoring/decode_out --n_best 750 --alpha 2 --gamma 0.5 113# python run.py lib/ctms/decode.ctm index/decode.json lib/kws/queries.xml run_output/decode_out.xml run_scoring/decode_out --n_best 750 --alpha 3 --gamma 0.5 114# part 3.3.1115# combine the best from decode+grapheme confusion and the best from decode-morph w/out grapheme confusion116########################### RUN THIS ##############################117# python run.py lib/ctms/decode-morph.ctm index/decode-morph.json output/queries_morph.kwslist.xml output/decode-morph_out.xml scoring/decode-morph_out --gamma 1.75 118# python run.py lib/ctms/decode.ctm index/decode.json lib/kws/queries.xml output/decode_out.xml scoring/decode_out --n_best 750 --gamma 0.25 --alpha 1.0119# python run.py lib/ctms/decode-morph.ctm index/decode-morph.json output/queries_morph.kwslist.xml output/decode-morph_unnorm_out.xml scoring/decode-morph_unnorm_out ...
task_collector.py
Source:task_collector.py
1from collections import Counter2from datetime import datetime3import os4from time import sleep5from RPA.Robocorp.Process import Process6from RPA.Robocorp.Vault import Vault7BOTS = [8 {9 "NAME": "236", # used as identification in the collected stats10 "LINE_ITEMS_DETAILS": True, # related to work item payload information of interest if True11 "MINIMUM_RUN_TIME": 200, # minimum process run time for a process to be "collected"12 },13 {14 "NAME": "22", # used as identification in the collected stats15 "LINE_ITEMS_DETAILS": False, # related to work item payload information of interest if True16 "MINIMUM_RUN_TIME": 0, # minimum process run time for a process to be "collected"17 },18]19def flush_print(message: str):20 print(message, flush=True)21class ControlRoomCollector:22 def __init__(self, runs_to_collect: int = 1):23 self.stats = {}24 self.bot = None25 self.runs_to_collect = runs_to_collect26 self.process = Process()27 self.output_dir = os.getenv("ROBOT_ARTIFACTS", "output")28 self.check_line_items = False29 self.api_throttle_time = 0.530 def set_bot(self, bot):31 self.bot = bot32 self.stats = {}33 secrets = Vault().get_secret(f"control_room_process_{self.bot['NAME']}")34 self.process.set_apikey(secrets["api_key"])35 self.process.set_workspace_id(secrets["workspace_id"])36 self.process.set_process_id(secrets["process_id"])37 self.check_line_items = (38 self.bot["LINE_ITEMS_DETAILS"]39 if "LINE_ITEMS_DETAILS" in self.bot.keys()40 else False41 )42 flush_print(f"Bot set: {self.bot['NAME']}")43 def collect_ok_runs_over_x_duration(self):44 if not self.bot:45 raise ValueError("Bot has not been set")46 collect = []47 runs = self.process.list_process_runs(limit=self.runs_to_collect)48 for run in runs:49 # Collect only runs which have state COMPL or PENDING50 if run["state"] in ["COMPL", "PENDING"]:51 # Collect only runs which have duration same or longer than MINIMUM_RUN_TIME52 if run["duration"] >= self.bot["MINIMUM_RUN_TIME"]:53 collect.append(run)54 else:55 flush_print(56 f"Run {run['runNo']} duration {run['duration']} under limit"57 )58 else:59 flush_print(run)60 flush_print(f"Run {run['runNo']} result is not OK")61 return collect62 def add_stats_for_date(self, date, stat):63 """Used for grouping stats by date"""64 if date not in self.stats.keys():65 self.stats[date] = [stat]66 else:67 self.stats[date].append(stat)68 def collect_data(self):69 if not self.bot:70 raise ValueError("Bot has not been set")71 runs = self.collect_ok_runs_over_x_duration()72 total_output = ""73 for run in runs:74 run_output = ""75 run_id = run["id"]76 run_no = run["runNo"]77 duration = 0 # run['duration']78 total_line_items = 079 run_status = self.process.get_process_run_status(run_id)80 step_runs = run_status["robotRuns"]81 real_start = None82 real_end = None83 for step_run in step_runs:84 one_step_run = self.process.get_process_run_status(85 run_id, step_run_id=step_run["id"]86 )87 step_run_start = datetime.strptime(88 one_step_run["startTs"], "%Y-%m-%dT%H:%M:%S.%fZ"89 )90 step_run_end = datetime.strptime(91 one_step_run["endTs"], "%Y-%m-%dT%H:%M:%S.%fZ"92 )93 if real_start == None or real_start > step_run_start:94 real_start = step_run_start95 if real_end == None or real_end < step_run_end:96 real_end = step_run_end97 duration += one_step_run["duration"]98 sleep(self.api_throttle_time)99 real_duration = real_end - real_start100 run_output += f"Run no {run_no}\n"101 items = self.process.list_process_run_work_items(run_id, include_data=True)102 run_output += f"\tnumber of step runs : {len(step_runs)}\n"103 run_output += f"\tnumber work items : {len(items)}\n"104 item_types = []105 if self.check_line_items:106 for item in items:107 if item["payload"]:108 line_items = item["payload"]["items"]109 total_line_items += len(line_items)110 if "item_type_code" in line_items[0].keys():111 item_types.extend(112 [item["item_type_code"] for item in line_items]113 )114 if total_line_items > 0:115 run_output += (116 f"\ttotal line items : {total_line_items} \n"117 )118 run_output += f"\taverage line item run duration : {round(duration/total_line_items,1)}s\n"119 run_output += f"\taverage step run duration : {round(duration/len(step_runs), 1)}s\n"120 run_output += f"\treal start : {real_start}\n"121 run_output += f"\treal end : {real_end}\n"122 run_output += (123 f"\treal duration : {real_duration.seconds}s\n"124 )125 run_output += f"\ttotal step run duration : {duration}s\n"126 run_output += "\n"127 flush_print(run_output)128 total_output += run_output129 date = datetime.strptime(items[0]["createTs"], "%Y-%m-%dT%H:%M:%S.%fZ")130 self.add_stats_for_date(131 date.strftime("%d/%b/%y"),132 {133 "runNo": run_no,134 "item_types": Counter(item_types),135 "real_duration": real_duration.seconds,136 "duration": duration,137 "step_runs": len(step_runs),138 "total_line_items": total_line_items,139 },140 )141 self.write_report("details", total_output)142 def write_report(self, report_type, content):143 output_filename = os.path.join(144 self.output_dir, f"bot_{self.bot['NAME']}_{report_type}_report.txt"145 )146 with open(output_filename, "w") as fout:147 fout.write(content)148 def print_collected_stats(self):149 if not self.bot:150 raise ValueError("Bot has not been set")151 output_string = ""152 for date in self.stats.keys():153 output_string += f"Day: {date}\n"154 total_duration = sum([s["duration"] for s in self.stats[date]])155 total_step_runs = sum([s["step_runs"] for s in self.stats[date]])156 if self.check_line_items:157 total_line_items = sum(158 [s["total_line_items"] for s in self.stats[date]]159 )160 total_item_types = sum(161 [s["item_types"] for s in self.stats[date]], Counter()162 )163 total_real_duration = sum([s["real_duration"] for s in self.stats[date]])164 output_string += (165 f"Duration (sum of step runs) : {total_duration}s\n"166 )167 output_string += (168 f"Effective runtime duration : {total_real_duration}s\n"169 )170 output_string += (171 f"Total step runs : {total_step_runs}\n"172 )173 if self.check_line_items:174 output_string += (175 f"Total line items : {total_line_items}\n"176 )177 output_string += "Line types : \n"178 for key, val in total_item_types.items():179 output_string += f"\t{key} = {val}\n"180 output_string += f"Average step run duration : {round(total_duration/total_step_runs,1)}s\n"181 output_string += f"Average step run real time duration : {round(total_real_duration/total_step_runs,1)}s\n"182 if self.check_line_items:183 output_string += f"Average line item duration : {round(total_duration/total_line_items,1)}s\n"184 output_string += f"Average line item real time duration : {round(total_real_duration/total_line_items,1)}s\n"185 output_string += "\n"186 flush_print(output_string)187 self.write_report("summary", output_string)188def main():189 collector = ControlRoomCollector(10)190 for bot in BOTS:191 collector.set_bot(bot)192 collector.collect_data()193 collector.print_collected_stats()194if __name__ == "__main__":...
Automation Testing Tutorials
Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.
LambdaTest Learning Hubs:
- JUnit Tutorial
- TestNG Tutorial
- Webdriver Tutorial
- WebDriverIO Tutorial
- Protractor Tutorial
- Selenium 4 Tutorial
- Jenkins Tutorial
- NUnit Tutorial
- Jest Tutorial
- Playwright Tutorial
- Cypress Tutorial
- PyTest Tutorial
YouTube
You could also refer to video tutorials over LambdaTest YouTube channel to get step by step demonstration from industry experts.
Run autotest automation tests on LambdaTest cloud grid
Perform automation testing on 3000+ real desktop and mobile devices online.
Try LambdaTest Now !!
Get 100 minutes of automation test minutes FREE!!
Was this article helpful?
