Unlock 30% off on Manual Testing Annual Plans this Holiday Season. 
Use Code: LCMYEARENDEXCL30
Use Code: LCMYEARENDEXCL30Check icon
Copied to Clipboard!
How to use mktagger method in stestr
Best Python code snippet using stestr_python
GCLogger.py
Source:GCLogger.py 
1#!/usr/bin/python -O2import re3import sys4import json5from monitor.metrics.Tail import LineParser6"""7This is a parser of GC log of Sun HotSpot JVM Version 6.8Required JVM option: -Xloggc=${GC_LOG_FILE} -XX:+PrintGCDetails9Usage:10 java -Xloggc=${GC_LOG_FILE} -XX:+PrintGCDetails ${ANY_OTHER_OPTIONS}11 this.py < ${GC_LOG_FILE} | grep -v ^### | ${YOUR_ANALYZER}12You can get all data as a python dictionary structure13in your analyer as follows:14look at testcase in test.metrics.log.GCLoggerTest.py15 16"""17################################################################################18# Parser generator from regular expression.19################################################################################20"""21Generate a parser from regex pattern and modifier.22Parser try to match input text by the pattern.23If matched, call data_modifier with list of matched strings.24The modifier add/update tag_str of the dictionary.25regexStr :: String26dataModifier :: (a, [String]) -> a | None27return :: (String, a) -> (String, a)28a :: ANY29dataModifier must not throw exceptions.30When some errors occur inside dataModifier, a must be not modified.31"""32def newP(regexStr, dataModifier):33 p = re.compile("(^%s)" % regexStr)34 def parse_(line, data):35 m = p.match(line)36 if m:37 if dataModifier is not None:38 data = dataModifier(data, m.groups()[1:])39 return (line[len(m.group(1)):], data)40 else:41 msg = "Parse failed: pattern \"%s\" for \"%s\"" % (regexStr, line)42 raise ParseError(msg)43 return parse_44################################################################################45# Utilities.46################################################################################47"""48Just modify data during parse.49dataModifier :: (a, [String]) -> a50return :: (String, a) -> (String, a)51a :: ANY52"""53def appP(dataModifier):54 def modify_(line, data):55 if dataModifier is not None:56 data = dataModifier(data)57 return (line, data)58 return modify_59# [String] -> String60def toString(strL):61 ret = "[%s" % strL[0]62 for str in strL[1:]:63 ret += ", %s" % str64 ret += "]"65 return ret66# Error type for parser.67class ParseError(Exception):68 pass69################################################################################70# Parser combinators.71################################################################################72"""73Parser combinator AND.74parsers :: [Parser]75return :: Parser76"""77def andP(parsers):78 def parseAnd_(text, data):79 text0 = text80 data0 = data81 for parser in parsers:82 (text1, data1) = parser(text0, data0)83 text0 = text184 data0 = data185 return (text0, data0)86 return parseAnd_87"""88Parser combinator OR.89parsers :: [Parser]90return :: Parser91"""92def orP(parsers):93 def parseOr_(text, data):94 msgL = []95 for parser in parsers:96 try:97 (ret_text, ret_data) = parser(text, data)98 return (ret_text, ret_data)99 except ParseError as msg:100 msgL.append(msg)101 msgs = toString(msgL)102 raise ParseError(msgs)103 return parseOr_104"""105Parser combinator MANY.106parsers :: [Parser]107return :: Parser108"""109def manyP(parser):110 def parseMany_(text, data):111 text0 = text112 data0 = data113 text1 = text114 data1 = data115 try:116 while True:117 (text1, data1) = parser(text0, data0)118 text0 = text1119 data0 = data1120 except ParseError as msg:121 if __debug__:122 print(msg)123 return (text1, data1)124 return parseMany_125################################################################################126# Utilities.127################################################################################128"""129A modifier for dictionary data.130tagStr :: String131dataConstructor :: [String] -> ANY132return :: (Dictionary, [String]) -> Dictionary133"""134def mkDictModifier(tagStr, dataConstructor):135 def modifyNothing_(dictData, matchStringL):136 return dictData137 if tagStr is None or dataConstructor is None:138 return modifyNothing_139 def modifyDict_(dictData, matchStringL):140 dictData[tagStr] = dataConstructor(matchStringL)141 return dictData142 return modifyDict_143"""144Behave like newP but that parses anything, just modify dictionary.145key :: String146value :: ANY147return :: (String, Dictionary) -> (String, Dictionary)148"""149def mkTagger(key, value):150 def tagger_(line, dictData):151 dictData[key] = value152 return (line, dictData)153 return tagger_154# match_strL :: [String] # length must be 1.155# return :: Float156def get_float(match_strL):157 assert len(match_strL) == 1158 return float(match_strL[0])159# match_strL :: [String] # length must be 3.160# return :: [Int] # length is 3.161def get_int3(match_strL):162 assert len(match_strL) == 3163 return [int(match_strL[0]), int(match_strL[1]), int(match_strL[2])]164# match_strL :: [String]165# return :: True166def get_true(match_strL):167 return True168################################################################################169# Regexp aliases.170################################################################################171regexp_float = r"(\d+.\d*)"172regexp_float_colon = regexp_float + r":\s+"173regexp_heap_info = r"(\d+)K->(\d+)K\((\d+)K\)"174regexp_float_secs = regexp_float + r"\s*secs\s+"175regexp_basic_string = r"([0-9a-zA-Z_-]+)"176################################################################################177# Parsers for gc log entries.178################################################################################179parseParNew = andP([ \180 mkTagger("type", "ParNew"), \181 newP(regexp_float_colon, mkDictModifier("timestamp", get_float)), \182 newP(r"\[GC\s+", None), \183 newP(regexp_float_colon, None), \184 newP(r"\[ParNew:\s+", None), \185 newP(regexp_heap_info + r",\s+", mkDictModifier("heap_new", get_int3)), \186 newP(regexp_float + r"\s*secs\]\s*", None), \187 newP(regexp_heap_info, mkDictModifier("heap_all", get_int3)), \188 newP(r"\s*(?:icms_dc=\d+\s*)?", None), \189 newP(r",\s*", None), \190 newP(regexp_float + r"\s*secs\]\s*", mkDictModifier("response", get_float)), \191 newP(r"\[Times:.*\]$", None), \192])193'''194if __debug__:195 text = r"2.380: [GC 2.380: [ParNew: 32768K->4204K(49152K), 0.0128980 secs] 32768K->4204K(114688K), 0.0130090 secs] [Times: user=0.04 sys=0.00, real=0.01 secs]"196 (ret, data) = parseParNew(text, {})197 print(text)198 print(len(ret))199 print(data)200if __debug__:201 text = r"9.815: [GC 9.815: [ParNew: 32768K->10796K(49152K), 0.0286700 secs] 52540K->30568K(114688K) icms_dc=0 , 0.0287550 secs] [Times: user=0.09 sys=0.00, real=0.03 secs]"202 (ret, data) = parseParNew(text, {})203 print(text)204 print(len(ret))205 print(data)206''' 207parseInitialMark = andP([ \208 mkTagger("type", "CMS-initial-mark"), \209 newP(regexp_float + r":\s*", mkDictModifier("timestamp", get_float)), \210 newP(r".*CMS-initial-mark:.*$", None), \211])212'''213if __debug__:214 text = r"3.072: [GC [1 CMS-initial-mark: 0K(65536K)] 19136K(114688K), 0.0215880 secs] [Times: user=0.04 sys=0.00, real=0.02 secs]"215 (ret, data) = parseInitialMark(text, {})216 print(text)217 print(len(ret))218 print(data)219'''220parseMarkStart = andP([ \221 mkTagger("type", "CMS-concurrent-mark-start"), \222 newP(regexp_float + r":\s*", mkDictModifier("timestamp", get_float)), \223 newP(r".*CMS-concurrent-mark-start.*$", None), \224])225'''226if __debug__:227 text = r"3.094: [CMS-concurrent-mark-start]"228 (ret, data) = parseMarkStart(text, {})229 print(text)230 print(len(ret))231 print(data)232'''233parseMark = andP([ \234 mkTagger("type", "CMS-concurrent-mark"), \235 newP(regexp_float + r":\s+", mkDictModifier("timestamp", get_float)), \236 newP(r"\[CMS-concurrent-mark:\s+", None), \237 newP(regexp_float + r"/", None), \238 newP(regexp_float + r"\s+secs\]\s+", mkDictModifier("response", get_float)), \239 newP(r"\[Times:.*\]$", None), \240])241'''242if __debug__:243 text = r"3.131: [CMS-concurrent-mark: 0.034/0.037 secs] [Times: user=0.12 sys=0.00, real=0.04 secs]"244 (ret, data) = parseMark(text, {})245 print(text)246 print(len(ret))247 print(data)248'''249parsePrecleanStart = andP([ \250 mkTagger("type", "CMS-concurrent-preclean-start"), \251 newP(regexp_float + r":\s*", mkDictModifier("timestamp", get_float)), \252 newP(r".*CMS-concurrent-preclean-start.*$", None), \253])254'''255if __debug__:256 text = r"3.132: [CMS-concurrent-preclean-start]"257 (ret, data) = parsePrecleanStart(text, {})258 print(text)259 print(len(ret))260 print(data)261'''262parsePreclean = andP([ \263 mkTagger("type", "CMS-concurrent-preclean"), \264 newP(regexp_float + r":\s+", mkDictModifier("timestamp", get_float)), \265 newP(r"\[CMS-concurrent-preclean:\s+", None), \266 newP(regexp_float + r"/", None), \267 newP(regexp_float + r"\s+secs\]\s+", mkDictModifier("response", get_float)), \268 newP(r"\[Times:.*\]$", None), \269])270'''271if __debug__:272 text = r"3.149: [CMS-concurrent-preclean: 0.014/0.018 secs] [Times: user=0.07 sys=0.00, real=0.01 secs]"273 (ret, data) = parsePreclean(text, {})274 print(text)275 print(len(ret))276 print(data)277'''278parseAbortablePrecleanStart = andP([ \279 mkTagger("type", "CMS-concurrent-abortable-preclean-start"), \280 newP(regexp_float + r":\s*", mkDictModifier("timestamp", get_float)), \281 newP(r".*CMS-concurrent-abortable-preclean-start.*$", None), \282])283'''284if __debug__:285 text = r"3.149: [CMS-concurrent-abortable-preclean-start]"286 (ret, data) = parseAbortablePrecleanStart(text, {})287 print(text)288 print(len(ret))289 print(data)290'''291parseAbortablePreclean = andP([ \292 mkTagger("type", "CMS-concurrent-abortable-preclean"), \293 newP(regexp_float + r":\s+", mkDictModifier("timestamp", get_float)), \294 newP(r"\[CMS-concurrent-abortable-preclean:\s+", None), \295 newP(regexp_float + r"/", None), \296 newP(regexp_float + r"\s+secs\]\s+", mkDictModifier("response", get_float)), \297 newP(r"\[Times:.*\]$", None), \298])299'''300if __debug__:301 text = r"17.418: [CMS-concurrent-abortable-preclean: 0.353/1.423 secs] [Times: user=4.60 sys=0.07, real=1.42 secs]"302 (ret, data) = parseAbortablePreclean(text, {})303 print(text)304 print(len(ret))305 print(data)306'''307parseAbortablePrecleanFullGC0 = andP([ \308 mkTagger("type", "CMS-concurrent-abortable-preclean-fullgc0"), \309 newP(regexp_float + r":\s+", mkDictModifier("timestamp", get_float)), \310 orP([ \311 newP(r"\[Full GC\s*\(System\)\s*" + regexp_float + r":\s+", mkDictModifier("system", get_true)), \312 newP(r"\[Full GC\s*" + regexp_float + r":\s+", None), \313 ]), \314 newP(r"\[CMS" + regexp_float + r":\s+", None), \315 newP(r"\[CMS-concurrent-abortable-preclean:\s+", None), \316 newP(regexp_float + r"/", None), \317 newP(regexp_float + r"\s+secs\]\s*", mkDictModifier("response", get_float)), \318 newP(r"\[Times:.*\]$", None), \319])320'''321if __debug__:322 text = r"3.242: [Full GC 3.242: [CMS3.243: [CMS-concurrent-abortable-preclean: 0.046/0.093 secs] [Times: user=0.36 sys=0.00, real=0.10 secs]"323 (ret, data) = parseAbortablePrecleanFullGC0(text, {})324 print(text)325 print(len(ret))326 print(data)327 text = r"63.533: [Full GC (System) 63.533: [CMS63.534: [CMS-concurrent-abortable-preclean: 0.316/1.244 secs] [Times: user=0.32 sys=0.01, real=1.25 secs]"328 (ret, data) = parseAbortablePrecleanFullGC0(text, {})329 print(text)330 print(len(ret))331 print(data)332'''333parseAbortablePrecleanFullGC1 = andP([ \334 mkTagger("type", "CMS-concurrent-abortable-preclean-fullgc1"), \335 newP(r"\s*\(concurrent mode (failure|interrupted)\):\s+", None), \336 newP(regexp_heap_info + r",\s+", mkDictModifier("heap_1", get_int3)), \337 newP(regexp_float + r"\s+secs\s*\]\s+", None), \338 newP(regexp_heap_info + r",\s+", mkDictModifier("heap_2", get_int3)), \339 newP(r"\[CMS Perm\s+:\s+", None), \340 newP(regexp_heap_info + r"\],\s+", mkDictModifier("perm", get_int3)), \341 newP(regexp_float + r"\s*secs\s*\]\s*", mkDictModifier("response", get_float)), \342 newP(r"\[Times:.*\]$", None), \343])344'''345if __debug__:346 text = r" (concurrent mode failure): 0K->7015K(65536K), 0.1244810 secs] 22902K->7015K(114688K), [CMS Perm : 21242K->21237K(21248K)], 0.1246890 secs] [Times: user=0.19 sys=0.05, real=0.13 secs]"347 (ret, data) = parseAbortablePrecleanFullGC1(text, {})348 print(text)349 print(len(ret))350 print(data)351 text = r" (concurrent mode interrupted): 44784K->40478K(65536K), 0.4974690 secs] 66630K->40478K(114688K), [CMS Perm : 77174K->77148K(128736K)], 0.4975800 secs] [Times: user=0.46 sys=0.03, real=0.50 secs]"352 (ret, data) = parseAbortablePrecleanFullGC1(text, {})353 print(text)354 print(len(ret))355 print(data)356'''357parseAbortablePrecleanFailureTime = andP([ \358 mkTagger("type", "CMS-concurrent-abortable-preclean-failure-time"), \359 newP(r"\s*CMS:\s*abort preclean due to time\s*", None), \360 newP(regexp_float_colon, mkDictModifier("timestamp", get_float)), \361 newP(r"\[CMS-concurrent-abortable-preclean:\s*", None), \362 newP(regexp_float + r"/", None), \363 newP(regexp_float + r"\s*secs\s*\]\s*", mkDictModifier("response", get_float)), \364 newP(r"\[Times:.*\]$", None), \365])366'''367if __debug__:368 text = r" CMS: abort preclean due to time 36.855: [CMS-concurrent-abortable-preclean: 1.280/5.084 secs] [Times: user=1.29 sys=0.00, real=5.09 secs]"369 (ret, data) = parseAbortablePrecleanFailureTime(text, {})370 print(text)371 print(len(ret))372 print(data)373 text = r"3.368: [GC [1 CMS-initial-mark: 7015K(65536K)] 7224K(114688K), 0.0004900 secs] [Times: user=0.00 sys=0.00, real=0.00 secs]"374 (ret, data) = parseInitialMark(text, {})375 print(text)376 print(len(ret))377 print(data)378 text = r"3.428: [CMS-concurrent-mark: 0.059/0.060 secs] [Times: user=0.22 sys=0.00, real=0.06 secs]"379 (ret, data) = parseMark(text, {})380 print(text)381 print(len(ret))382 print(data)383 text = r"3.431: [CMS-concurrent-preclean: 0.002/0.002 secs] [Times: user=0.00 sys=0.00, real=0.00 secs]"384 (ret, data) = parsePreclean(text, {})385 print(text)386 print(len(ret))387 print(data)388'''389parseRemark = andP([ \390 mkTagger("type", "CMS-remark"), \391 newP(regexp_float + r":\s*", mkDictModifier("timestamp", get_float)), \392 newP(r"\[GC\[YG occupancy.+CMS-remark:\s+\d+K\(\d+K\)\]\s*", None), \393 newP(r"\d+K\(\d+K\),\s*", None), \394 newP(regexp_float + r"\s*secs\]\s*", mkDictModifier("response", get_float)), \395 newP(r"\[Times:.*\]$", None), \396])397'''398if __debug__:399 text = r"3.431: [GC[YG occupancy: 1005 K (49152 K)]3.431: [Rescan (parallel) , 0.0080410 secs]3.439: [weak refs processing, 0.0000100 secs]3.439: [class unloading, 0.0014010 secs]3.441: [scrub symbol & string tables, 0.0032440 secs] [1 CMS-remark: 7015K(65536K)] 8021K(114688K), 0.0130490 secs] [Times: user=0.03 sys=0.00, real=0.02 secs]"400 (ret, data) = parseRemark(text, {})401 print(text)402 print(len(ret))403 print(data)404'''405parseSweepStart = andP([ \406 mkTagger("type", "CMS-concurrent-sweep-start"), \407 newP(regexp_float + r":\s*", mkDictModifier("timestamp", get_float)), \408 newP(r"\[CMS-concurrent-sweep-start\]$", None), \409])410'''411if __debug__:412 text = r"3.444: [CMS-concurrent-sweep-start]"413 (ret, data) = parseSweepStart(text, {})414 print(text)415 print(len(ret))416 print(data)417'''418parseSweep = andP([ \419 mkTagger("type", "CMS-concurrent-sweep"), \420 newP(regexp_float + r":\s+", mkDictModifier("timestamp", get_float)), \421 newP(r"\[CMS-concurrent-sweep:\s+", None), \422 newP(regexp_float + r"/", None), \423 newP(regexp_float + r"\s+secs\]\s+", mkDictModifier("response", get_float)), \424 newP(r"\[Times:.*\]$", None), \425])426'''427if __debug__:428 text = r"3.468: [CMS-concurrent-sweep: 0.024/0.024 secs] [Times: user=0.06 sys=0.00, real=0.02 secs]"429 (ret, data) = parseSweep(text, {})430 print(text)431 print(len(ret))432 print(data)433'''434parseResetStart = andP([ \435 mkTagger("type", "CMS-concurrent-reset-start"), \436 newP(regexp_float + r":\s*", mkDictModifier("timestamp", get_float)), \437 newP("\[CMS-concurrent-reset-start\]$", None), \438])439'''440if __debug__:441 text = r"3.468: [CMS-concurrent-reset-start]"442 (ret, data) = parseResetStart(text, {})443 print(text)444 print(len(ret))445 print(data)446'''447parseReset = andP([ \448 mkTagger("type", "CMS-concurrent-reset"), \449 newP(regexp_float + r":\s+", mkDictModifier("timestamp", get_float)), \450 newP(r"\[CMS-concurrent-reset:\s+", None), \451 newP(regexp_float + r"/", None), \452 newP(regexp_float + r"\s+secs\]\s+", mkDictModifier("response", get_float)), \453 newP(r"\[Times:.*\]$", None), \454])455'''456if __debug__:457 text = r"3.468: [CMS-concurrent-reset: 0.000/0.000 secs] [Times: user=0.00 sys=0.00, real=0.00 secs]"458 (ret, data) = parseReset(text, {})459 print(text)460 print(len(ret))461 print(data)462'''463parseFullGC = andP([ \464 mkTagger("type", "FullGC"), \465 newP(regexp_float_colon, mkDictModifier("timestamp", get_float)), \466 orP([ \467 newP(r"\[Full GC\s*\(System\)\s*", mkDictModifier("system", get_true)), \468 newP(r"\[Full GC\s*", None), \469 ]), \470 newP(regexp_float_colon, None), \471 newP(r"\[CMS:\s+", None), \472 newP(regexp_heap_info + r",\s+", mkDictModifier("heap_cms", get_int3)), \473 newP(regexp_float + r"\s*secs\]\s*", None), \474 newP(regexp_heap_info, mkDictModifier("heap_all", get_int3)), \475 newP(r"\s*,\s*\[CMS Perm\s*:\s*", None), \476 newP(regexp_heap_info, mkDictModifier("perm", get_int3)), \477 newP(r"\]\s*(?:icms_dc=\d+\s*)?", None), \478 newP(r",\s*", None), \479 newP(regexp_float + r"\s*secs\]\s*", mkDictModifier("response", get_float)), \480 newP(r"\[Times:.*\]$", None), \481])482'''483if __debug__:484 text = r"7.992: [Full GC 7.992: [CMS: 6887K->19772K(65536K), 0.4137230 secs] 34678K->19772K(114688K), [CMS Perm : 54004K->53982K(54152K)] icms_dc=0 , 0.4140100 secs] [Times: user=0.68 sys=0.14, real=0.41 secs]"485 (ret, data) = parseFullGC(text, {})486 print(text)487 print(len(ret))488 print(data)489 text = r"123.533: [Full GC (System) 123.533: [CMS: 39710K->34052K(65536K), 0.4852070 secs] 62832K->34052K(114688K), [CMS Perm : 77479K->76395K(128928K)], 0.4853310 secs] [Times: user=0.47 sys=0.01, real=0.48 secs]"490 (ret, data) = parseFullGC(text, {})491 print(text)492 print(len(ret))493 print(data)494'''495# This is for -XX:+UseParallelGC496parseParallelGC = andP([ \497 mkTagger("type", "ParallelGC"), \498 newP(regexp_float_colon, mkDictModifier("timestamp", get_float)), \499 newP(r"\[GC\s+\[PSYoungGen:\s*", None), \500 newP(regexp_heap_info + r"\s*\]\s*", mkDictModifier("heap_new", get_int3)), \501 newP(regexp_heap_info + r"\s*,\s*", mkDictModifier("heap_all", get_int3)), \502 newP(regexp_float + r"\s*secs\s*\]\s*", mkDictModifier("response", get_float)), \503 newP(r"\[Times:.*\]$", None), \504])505'''506if __debug__:507 text = r"162.002: [GC [PSYoungGen: 39323K->3653K(49152K)] 87187K->56999K(114688K), 0.0207580 secs] [Times: user=0.08 sys=0.00, real=0.02 secs]"508 (ret, data) = parseParallelGC(text, {})509 print(text)510 print(len(ret))511 print(data)512'''513# This is for -XX:+UseParallelGC514parseParallelFullGC = andP([ \515 mkTagger("type", "ParallelFullGC"), \516 newP(regexp_float_colon, mkDictModifier("timestamp", get_float)), \517 orP([ \518 newP(r"\[Full GC\s*\(System\)\s*\[PSYoungGen:\s*", mkDictModifier("system", get_true)), \519 newP(r"\[Full GC\s*\[PSYoungGen:\s*", None), \520 ]), \521 newP(regexp_heap_info + r"\s*\]\s*", mkDictModifier("heap_new", get_int3)), \522 orP([ \523 newP(r"\[ParOldGen:\s*", None), \524 newP(r"\[PSOldGen:\s*", None), \525 ]), \526 newP(regexp_heap_info + r"\s*\]\s*", mkDictModifier("heap_old", get_int3)), \527 newP(regexp_heap_info + r"\s*", mkDictModifier("heap_all", get_int3)), \528 newP(r"\[PSPermGen:\s*", None), \529 newP(regexp_heap_info + r"\s*\]\s*,\s*", mkDictModifier("perm", get_int3)), \530 newP(regexp_float + r"\s*secs\s*\]\s*", mkDictModifier("response", get_float)), \531 newP(r"\[Times:.*\]$", None), \532])533'''534if __debug__:535 text = r"162.657: [Full GC [PSYoungGen: 6189K->0K(50752K)] [PSOldGen: 58712K->43071K(65536K)] 64902K->43071K(116288K) [PSPermGen: 81060K->81060K(81152K)], 0.3032230 secs] [Times: user=0.30 sys=0.00, real=0.30 secs]"536 (ret, data) = parseParallelFullGC(text, {})537 print(text)538 print(len(ret))539 print(data)540'''541# This is for -XX:+UseSerialGC542parseSerialGC = andP([ \543 mkTagger("type", "SerialGC"), \544 newP(regexp_float_colon, mkDictModifier("timestamp", get_float)), \545 newP(r"\[GC\s+", None), \546 newP(regexp_float_colon + r"\[DefNew:\s*", None), \547 newP(regexp_heap_info + r"\s*,\s*", mkDictModifier("heap_new", get_int3)), \548 newP(regexp_float + r"\s*secs\s*\]\s*", None), \549 newP(regexp_heap_info + r"\s*,\s*", mkDictModifier("heap_all", get_int3)), \550 newP(regexp_float + r"\s*secs\s*\]\s*", mkDictModifier("response", get_float)), \551 newP(r"\[Times:.*\]$", None), \552])553'''554if __debug__:555 text = r"4.687: [GC 4.687: [DefNew: 33343K->649K(49152K), 0.0021450 secs] 45309K->12616K(114688K), 0.0021800 secs] [Times: user=0.00 sys=0.00, real=0.00 secs]"556 (ret, data) = parseSerialGC(text, {})557 print(text)558 print(len(ret))559 print(data)560'''561# This is for -XX:+UseSerialGC562parseSerialFullGC = andP([ \563 mkTagger("type", "SerialFullGC"), \564 newP(regexp_float_colon, mkDictModifier("timestamp", get_float)), \565 newP(r"\[Full GC\s+", None), \566 newP(regexp_float_colon + r"\s*", None), \567 newP(r"\[Tenured:\s*", None), \568 newP(regexp_heap_info + r"\s*,\s*", mkDictModifier("heap_old", get_int3)), \569 newP(regexp_float + r"\s*secs\]\s*", None), \570 newP(regexp_heap_info + r"\s*,\s*", mkDictModifier("heap_all", get_int3)), \571 newP(r"\[Perm\s*:\s*", None), \572 newP(regexp_heap_info + r"\s*\]\s*,\s*", mkDictModifier("perm", get_int3)), \573 newP(regexp_float + r"\s*secs\s*\]\s*", mkDictModifier("response", get_float)), \574 newP(r"\[Times:.*\]$", None), \575])576'''577if __debug__:578 text = r"4.899: [Full GC 4.899: [Tenured: 11966K->12899K(65536K), 0.1237750 secs] 22655K->12899K(114688K), [Perm : 32122K->32122K(32128K)], 0.1238590 secs] [Times: user=0.11 sys=0.00, real=0.13 secs]"579 (ret, data) = parseSerialFullGC(text, {})580 print(text)581 print(len(ret))582 print(data)583'''584"""585Java GC Log parser.586This supports almost kinds of GC provided by JVM.587-XX:+UseConcSweepGC (-XX:+UseParNewGC)588 parseParNew589 parseFullGC590-XX:+UseConcSweepGC -XX:CMSIncrementalMode (-XX:+UseParNewGC)591 parseParNew, parseFullGC,592 parse{InitialMark, MarkStart, Mark, PrecleanStart, Preclean,593 AbortablePrecleanStart, AbortablePreclean,594 AbortablePrecleanFullGC0, AbortablePrecleanFullGC1,595 AbortablePrecleanFailureTime,596 Remark,597 SweepStart, Sweep, ResetStart, Reset}598 parseAbortablePrecleanFullGC0 and parseAbortablePrecleanFullGC1599 must be always together.600-XX:+UseParallelGC601 parseParallelFullGC, parseParallelGC.602-XX:+UseSerialGC603 parseSerialFullGC, parseSerialGC.604 605"""606parseJavaGcLog = orP([ \607 parseParNew, \608 parseFullGC, \609 parseInitialMark, \610 parseMarkStart, parseMark, \611 parsePrecleanStart, parsePreclean, \612 parseAbortablePrecleanStart, parseAbortablePreclean, \613 parseAbortablePrecleanFullGC0, \614 parseAbortablePrecleanFullGC1, 615 parseAbortablePrecleanFailureTime, \616 parseRemark, \617 parseSweepStart, parseSweep, \618 parseResetStart, parseReset, \619 parseParallelFullGC, \620 parseParallelGC, \621 parseSerialFullGC, \622 parseSerialGC, \623])624################################################################################625# Parser of list of integer. This is for test.626################################################################################627"""628A modifier for list.629return :: ([[String]], [String]) -> [String]630"""631def mkListAppender():632 def listAppend_(list, matchStringL):633 if len(matchStringL) > 0:634 list.append(matchStringL[0])635 return list636 return listAppend_637"""638Convert last element to Int.639list :: [Int, Int, ..., Int, String]640return :: [Int, Int, ..., Int, Int]641"""642def convertLastToInt(list):643 list[-1] = int(list[-1])644 return list645# Parser of list of integer. This is for test.646parseIntList = andP([ \647 newP(r"\s*\[\s*", None), \648 manyP(andP([ \649 newP("(\d+)\s*(?:,\s*)?", mkListAppender()), \650 appP(convertLastToInt), \651 ])), \652 newP(r"\s*\]\s*", None), \653])654'''655if __debug__:656 text = r"[10, 20, 30]"657 (ret, data) = parseIntList(text, [])658 print(text)659 print(len(ret))660 print(data)661'''662 663################################################################################664# main665################################################################################666class GCLogLineParser(LineParser):667 array_clazz = [].__class__668 def parse(self,line):669 (ret,data) = parseJavaGcLog(line, {})670 values = dict()671 if not isinstance(data, dict):672 return None673 for key,value in data.items():674 if isinstance(value, self.array_clazz) and len(value) == 3:675 values[key+"_before"]=value[0]676 values[key+"_after"]=value[1]677 values[key+"_total"]=value[2]678 else:679 values[key] = value680 return values 681 682'''683data_prev = None684for line in sys.stdin:685 try:686 text = line.rstrip()687 #print(text)688 (ret, data) = parseJavaGcLog(text, {})689 if data["type"] == "CMS-concurrent-abortable-preclean-fullgc0":690 data_prev = data691 continue692 if data["type"] == "CMS-concurrent-abortable-preclean-fullgc1":693 assert data_prev["type"] == "CMS-concurrent-abortable-preclean-fullgc0"694 data_prev.update(data)695 data = data_prev696 data_prev = None697 data["type"] = "CMS-concurrent-abortable-preclean-fullgc"698 if __debug__:699 print(("len: %d" % len(ret)))700 print(json.dumps(data))701 except ParseError as msg:702 print(msg)703 print(("###%s" % text))704'''...
java_gc_log_parser.py
Source:java_gc_log_parser.py
1#!/usr/bin/python -O2import re3import sys4import json5"""6This is a parser of GC log of Sun HotSpot JVM Version 6.7Required JVM option: -Xloggc=${GC_LOG_FILE} -XX:+PrintGCDetails8Usage:9 java -Xloggc=${GC_LOG_FILE} -XX:+PrintGCDetails ${ANY_OTHER_OPTIONS}10 this.py < ${GC_LOG_FILE} | grep -v ^### | ${YOUR_ANALYZER}11You can get all data as a python dictionary structure12in your analyer as follows:13import sys14import json15list = []16for line in sys.stdin:17 line.rstrip()18 list.append(dict(json.loads(line)))19 20"""21################################################################################22# Parser generator from regular expression.23################################################################################24"""25Generate a parser from regex pattern and modifier.26Parser try to match input text by the pattern.27If matched, call data_modifier with list of matched strings.28The modifier add/update tag_str of the dictionary.29regexStr :: String30dataModifier :: (a, [String]) -> a | None31return :: (String, a) -> (String, a)32a :: ANY33dataModifier must not throw exceptions.34When some errors occur inside dataModifier, a must be not modified.35"""36def newP(regexStr, dataModifier):37 p = re.compile("(^%s)" % regexStr)38 def parse_(line, data):39 m = p.match(line)40 if m:41 if dataModifier is not None:42 data = dataModifier(data, m.groups()[1:])43 return (line[len(m.group(1)):], data)44 else:45 msg = "Parse failed: pattern \"%s\" for \"%s\"" % (regexStr, line)46 raise ParseError(msg)47 return parse_48################################################################################49# Utilities.50################################################################################51"""52Just modify data during parse.53dataModifier :: (a, [String]) -> a54return :: (String, a) -> (String, a)55a :: ANY56"""57def appP(dataModifier):58 def modify_(line, data):59 if dataModifier is not None:60 data = dataModifier(data)61 return (line, data)62 return modify_63# [String] -> String64def toString(strL):65 ret = "[%s" % strL[0]66 for str in strL[1:]:67 ret += ", %s" % str68 ret += "]"69 return ret70# Error type for parser.71class ParseError(Exception):72 pass73################################################################################74# Parser combinators.75################################################################################76"""77Parser combinator AND.78parsers :: [Parser]79return :: Parser80"""81def andP(parsers):82 def parseAnd_(text, data):83 text0 = text84 data0 = data85 for parser in parsers:86 (text1, data1) = parser(text0, data0)87 text0 = text188 data0 = data189 return (text0, data0)90 return parseAnd_91"""92Parser combinator OR.93parsers :: [Parser]94return :: Parser95"""96def orP(parsers):97 def parseOr_(text, data):98 msgL = []99 for parser in parsers:100 try:101 (ret_text, ret_data) = parser(text, data)102 return (ret_text, ret_data)103 except ParseError, msg:104 msgL.append(msg)105 msgs = toString(msgL)106 raise ParseError(msgs)107 return parseOr_108"""109Parser combinator MANY.110parsers :: [Parser]111return :: Parser112"""113def manyP(parser):114 def parseMany_(text, data):115 text0 = text116 data0 = data117 text1 = text118 data1 = data119 try:120 while True:121 (text1, data1) = parser(text0, data0)122 text0 = text1123 data0 = data1124 except ParseError, msg:125 if __debug__:126 print msg127 return (text1, data1)128 return parseMany_129################################################################################130# Utilities.131################################################################################132"""133A modifier for dictionary data.134tagStr :: String135dataConstructor :: [String] -> ANY136return :: (Dictionary, [String]) -> Dictionary137"""138def mkDictModifier(tagStr, dataConstructor):139 def modifyNothing_(dictData, matchStringL):140 return dictData141 if tagStr is None or dataConstructor is None:142 return modifyNothing_143 def modifyDict_(dictData, matchStringL):144 dictData[tagStr] = dataConstructor(matchStringL)145 return dictData146 return modifyDict_147"""148Behave like newP but that parses anything, just modify dictionary.149key :: String150value :: ANY151return :: (String, Dictionary) -> (String, Dictionary)152"""153def mkTagger(key, value):154 def tagger_(line, dictData):155 dictData[key] = value156 return (line, dictData)157 return tagger_158# match_strL :: [String] # length must be 1.159# return :: Float160def get_float(match_strL):161 assert len(match_strL) == 1162 return float(match_strL[0])163# match_strL :: [String] # length must be 3.164# return :: [Int] # length is 3.165def get_int3(match_strL):166 assert len(match_strL) == 3167 return [int(match_strL[0]), int(match_strL[1]), int(match_strL[2])]168# match_strL :: [String]169# return :: True170def get_true(match_strL):171 return True172################################################################################173# Regexp aliases.174################################################################################175regexp_float = r"(\d+.\d*)"176regexp_float_colon = regexp_float + r":\s+"177regexp_heap_info = r"(\d+)K->(\d+)K\((\d+)K\)"178regexp_float_secs = regexp_float + r"\s*secs\s+"179regexp_basic_string = r"([0-9a-zA-Z_-]+)"180################################################################################181# Parsers for gc log entries.182################################################################################183parseParNew = andP([ \184 mkTagger("type", "ParNew"), \185 newP(regexp_float_colon, mkDictModifier("timestamp", get_float)), \186 newP(r"\[GC\s+", None), \187 newP(regexp_float_colon, None), \188 newP(r"\[ParNew:\s+", None), \189 newP(regexp_heap_info + r",\s+", mkDictModifier("heap_new", get_int3)), \190 newP(regexp_float + r"\s*secs\]\s*", None), \191 newP(regexp_heap_info, mkDictModifier("heap_all", get_int3)), \192 newP(r"\s*(?:icms_dc=\d+\s*)?", None), \193 newP(r",\s*", None), \194 newP(regexp_float + r"\s*secs\]\s*", mkDictModifier("response", get_float)), \195 newP(r"\[Times:.*\]$", None), \196])197if __debug__:198 text = r"2.380: [GC 2.380: [ParNew: 32768K->4204K(49152K), 0.0128980 secs] 32768K->4204K(114688K), 0.0130090 secs] [Times: user=0.04 sys=0.00, real=0.01 secs]"199 (ret, data) = parseParNew(text, {})200 print text201 print len(ret)202 print data203if __debug__:204 text = r"9.815: [GC 9.815: [ParNew: 32768K->10796K(49152K), 0.0286700 secs] 52540K->30568K(114688K) icms_dc=0 , 0.0287550 secs] [Times: user=0.09 sys=0.00, real=0.03 secs]"205 (ret, data) = parseParNew(text, {})206 print text207 print len(ret)208 print data209 210parseInitialMark = andP([ \211 mkTagger("type", "CMS-initial-mark"), \212 newP(regexp_float + r":\s*", mkDictModifier("timestamp", get_float)), \213 newP(r".*CMS-initial-mark:.*$", None), \214])215if __debug__:216 text = r"3.072: [GC [1 CMS-initial-mark: 0K(65536K)] 19136K(114688K), 0.0215880 secs] [Times: user=0.04 sys=0.00, real=0.02 secs]"217 (ret, data) = parseInitialMark(text, {})218 print text219 print len(ret)220 print data221parseMarkStart = andP([ \222 mkTagger("type", "CMS-concurrent-mark-start"), \223 newP(regexp_float + r":\s*", mkDictModifier("timestamp", get_float)), \224 newP(r".*CMS-concurrent-mark-start.*$", None), \225])226if __debug__:227 text = r"3.094: [CMS-concurrent-mark-start]"228 (ret, data) = parseMarkStart(text, {})229 print text230 print len(ret)231 print data232parseMark = andP([ \233 mkTagger("type", "CMS-concurrent-mark"), \234 newP(regexp_float + r":\s+", mkDictModifier("timestamp", get_float)), \235 newP(r"\[CMS-concurrent-mark:\s+", None), \236 newP(regexp_float + r"/", None), \237 newP(regexp_float + r"\s+secs\]\s+", mkDictModifier("response", get_float)), \238 newP(r"\[Times:.*\]$", None), \239])240if __debug__:241 text = r"3.131: [CMS-concurrent-mark: 0.034/0.037 secs] [Times: user=0.12 sys=0.00, real=0.04 secs]"242 (ret, data) = parseMark(text, {})243 print text244 print len(ret)245 print data246parsePrecleanStart = andP([ \247 mkTagger("type", "CMS-concurrent-preclean-start"), \248 newP(regexp_float + r":\s*", mkDictModifier("timestamp", get_float)), \249 newP(r".*CMS-concurrent-preclean-start.*$", None), \250])251if __debug__:252 text = r"3.132: [CMS-concurrent-preclean-start]"253 (ret, data) = parsePrecleanStart(text, {})254 print text255 print len(ret)256 print data257parsePreclean = andP([ \258 mkTagger("type", "CMS-concurrent-preclean"), \259 newP(regexp_float + r":\s+", mkDictModifier("timestamp", get_float)), \260 newP(r"\[CMS-concurrent-preclean:\s+", None), \261 newP(regexp_float + r"/", None), \262 newP(regexp_float + r"\s+secs\]\s+", mkDictModifier("response", get_float)), \263 newP(r"\[Times:.*\]$", None), \264])265if __debug__:266 text = r"3.149: [CMS-concurrent-preclean: 0.014/0.018 secs] [Times: user=0.07 sys=0.00, real=0.01 secs]"267 (ret, data) = parsePreclean(text, {})268 print text269 print len(ret)270 print data271parseAbortablePrecleanStart = andP([ \272 mkTagger("type", "CMS-concurrent-abortable-preclean-start"), \273 newP(regexp_float + r":\s*", mkDictModifier("timestamp", get_float)), \274 newP(r".*CMS-concurrent-abortable-preclean-start.*$", None), \275])276if __debug__:277 text = r"3.149: [CMS-concurrent-abortable-preclean-start]"278 (ret, data) = parseAbortablePrecleanStart(text, {})279 print text280 print len(ret)281 print data282parseAbortablePreclean = andP([ \283 mkTagger("type", "CMS-concurrent-abortable-preclean"), \284 newP(regexp_float + r":\s+", mkDictModifier("timestamp", get_float)), \285 newP(r"\[CMS-concurrent-abortable-preclean:\s+", None), \286 newP(regexp_float + r"/", None), \287 newP(regexp_float + r"\s+secs\]\s+", mkDictModifier("response", get_float)), \288 newP(r"\[Times:.*\]$", None), \289])290if __debug__:291 text = r"17.418: [CMS-concurrent-abortable-preclean: 0.353/1.423 secs] [Times: user=4.60 sys=0.07, real=1.42 secs]"292 (ret, data) = parseAbortablePreclean(text, {})293 print text294 print len(ret)295 print data296parseAbortablePrecleanFullGC0 = andP([ \297 mkTagger("type", "CMS-concurrent-abortable-preclean-fullgc0"), \298 newP(regexp_float + r":\s+", mkDictModifier("timestamp", get_float)), \299 orP([ \300 newP(r"\[Full GC\s*\(System\)\s*" + regexp_float + r":\s+", mkDictModifier("system", get_true)), \301 newP(r"\[Full GC\s*" + regexp_float + r":\s+", None), \302 ]), \303 newP(r"\[CMS" + regexp_float + r":\s+", None), \304 newP(r"\[CMS-concurrent-abortable-preclean:\s+", None), \305 newP(regexp_float + r"/", None), \306 newP(regexp_float + r"\s+secs\]\s*", mkDictModifier("response", get_float)), \307 newP(r"\[Times:.*\]$", None), \308])309if __debug__:310 text = r"3.242: [Full GC 3.242: [CMS3.243: [CMS-concurrent-abortable-preclean: 0.046/0.093 secs] [Times: user=0.36 sys=0.00, real=0.10 secs]"311 (ret, data) = parseAbortablePrecleanFullGC0(text, {})312 print text313 print len(ret)314 print data315 text = r"63.533: [Full GC (System) 63.533: [CMS63.534: [CMS-concurrent-abortable-preclean: 0.316/1.244 secs] [Times: user=0.32 sys=0.01, real=1.25 secs]"316 (ret, data) = parseAbortablePrecleanFullGC0(text, {})317 print text318 print len(ret)319 print data320parseAbortablePrecleanFullGC1 = andP([ \321 mkTagger("type", "CMS-concurrent-abortable-preclean-fullgc1"), \322 newP(r"\s*\(concurrent mode (failure|interrupted)\):\s+", None), \323 newP(regexp_heap_info + r",\s+", mkDictModifier("heap_1", get_int3)), \324 newP(regexp_float + r"\s+secs\s*\]\s+", None), \325 newP(regexp_heap_info + r",\s+", mkDictModifier("heap_2", get_int3)), \326 newP(r"\[CMS Perm\s+:\s+", None), \327 newP(regexp_heap_info + r"\],\s+", mkDictModifier("perm", get_int3)), \328 newP(regexp_float + r"\s*secs\s*\]\s*", mkDictModifier("response", get_float)), \329 newP(r"\[Times:.*\]$", None), \330])331if __debug__:332 text = r" (concurrent mode failure): 0K->7015K(65536K), 0.1244810 secs] 22902K->7015K(114688K), [CMS Perm : 21242K->21237K(21248K)], 0.1246890 secs] [Times: user=0.19 sys=0.05, real=0.13 secs]"333 (ret, data) = parseAbortablePrecleanFullGC1(text, {})334 print text335 print len(ret)336 print data337 text = r" (concurrent mode interrupted): 44784K->40478K(65536K), 0.4974690 secs] 66630K->40478K(114688K), [CMS Perm : 77174K->77148K(128736K)], 0.4975800 secs] [Times: user=0.46 sys=0.03, real=0.50 secs]"338 (ret, data) = parseAbortablePrecleanFullGC1(text, {})339 print text340 print len(ret)341 print data342parseAbortablePrecleanFailureTime = andP([ \343 mkTagger("type", "CMS-concurrent-abortable-preclean-failure-time"), \344 newP(r"\s*CMS:\s*abort preclean due to time\s*", None), \345 newP(regexp_float_colon, mkDictModifier("timestamp", get_float)), \346 newP(r"\[CMS-concurrent-abortable-preclean:\s*", None), \347 newP(regexp_float + r"/", None), \348 newP(regexp_float + r"\s*secs\s*\]\s*", mkDictModifier("response", get_float)), \349 newP(r"\[Times:.*\]$", None), \350])351if __debug__:352 text = r" CMS: abort preclean due to time 36.855: [CMS-concurrent-abortable-preclean: 1.280/5.084 secs] [Times: user=1.29 sys=0.00, real=5.09 secs]"353 (ret, data) = parseAbortablePrecleanFailureTime(text, {})354 print text355 print len(ret)356 print data357 text = r"3.368: [GC [1 CMS-initial-mark: 7015K(65536K)] 7224K(114688K), 0.0004900 secs] [Times: user=0.00 sys=0.00, real=0.00 secs]"358 (ret, data) = parseInitialMark(text, {})359 print text360 print len(ret)361 print data362 text = r"3.428: [CMS-concurrent-mark: 0.059/0.060 secs] [Times: user=0.22 sys=0.00, real=0.06 secs]"363 (ret, data) = parseMark(text, {})364 print text365 print len(ret)366 print data367 text = r"3.431: [CMS-concurrent-preclean: 0.002/0.002 secs] [Times: user=0.00 sys=0.00, real=0.00 secs]"368 (ret, data) = parsePreclean(text, {})369 print text370 print len(ret)371 print data372parseRemark = andP([ \373 mkTagger("type", "CMS-remark"), \374 newP(regexp_float + r":\s*", mkDictModifier("timestamp", get_float)), \375 newP(r"\[GC\[YG occupancy.+CMS-remark:\s+\d+K\(\d+K\)\]\s*", None), \376 newP(r"\d+K\(\d+K\),\s*", None), \377 newP(regexp_float + r"\s*secs\]\s*", mkDictModifier("response", get_float)), \378 newP(r"\[Times:.*\]$", None), \379])380if __debug__:381 text = r"3.431: [GC[YG occupancy: 1005 K (49152 K)]3.431: [Rescan (parallel) , 0.0080410 secs]3.439: [weak refs processing, 0.0000100 secs]3.439: [class unloading, 0.0014010 secs]3.441: [scrub symbol & string tables, 0.0032440 secs] [1 CMS-remark: 7015K(65536K)] 8021K(114688K), 0.0130490 secs] [Times: user=0.03 sys=0.00, real=0.02 secs]"382 (ret, data) = parseRemark(text, {})383 print text384 print len(ret)385 print data386parseSweepStart = andP([ \387 mkTagger("type", "CMS-concurrent-sweep-start"), \388 newP(regexp_float + r":\s*", mkDictModifier("timestamp", get_float)), \389 newP(r"\[CMS-concurrent-sweep-start\]$", None), \390])391if __debug__:392 text = r"3.444: [CMS-concurrent-sweep-start]"393 (ret, data) = parseSweepStart(text, {})394 print text395 print len(ret)396 print data397parseSweep = andP([ \398 mkTagger("type", "CMS-concurrent-sweep"), \399 newP(regexp_float + r":\s+", mkDictModifier("timestamp", get_float)), \400 newP(r"\[CMS-concurrent-sweep:\s+", None), \401 newP(regexp_float + r"/", None), \402 newP(regexp_float + r"\s+secs\]\s+", mkDictModifier("response", get_float)), \403 newP(r"\[Times:.*\]$", None), \404])405if __debug__:406 text = r"3.468: [CMS-concurrent-sweep: 0.024/0.024 secs] [Times: user=0.06 sys=0.00, real=0.02 secs]"407 (ret, data) = parseSweep(text, {})408 print text409 print len(ret)410 print data411parseResetStart = andP([ \412 mkTagger("type", "CMS-concurrent-reset-start"), \413 newP(regexp_float + r":\s*", mkDictModifier("timestamp", get_float)), \414 newP("\[CMS-concurrent-reset-start\]$", None), \415])416if __debug__:417 text = r"3.468: [CMS-concurrent-reset-start]"418 (ret, data) = parseResetStart(text, {})419 print text420 print len(ret)421 print data422parseReset = andP([ \423 mkTagger("type", "CMS-concurrent-reset"), \424 newP(regexp_float + r":\s+", mkDictModifier("timestamp", get_float)), \425 newP(r"\[CMS-concurrent-reset:\s+", None), \426 newP(regexp_float + r"/", None), \427 newP(regexp_float + r"\s+secs\]\s+", mkDictModifier("response", get_float)), \428 newP(r"\[Times:.*\]$", None), \429])430if __debug__:431 text = r"3.468: [CMS-concurrent-reset: 0.000/0.000 secs] [Times: user=0.00 sys=0.00, real=0.00 secs]"432 (ret, data) = parseReset(text, {})433 print text434 print len(ret)435 print data436parseFullGC = andP([ \437 mkTagger("type", "FullGC"), \438 newP(regexp_float_colon, mkDictModifier("timestamp", get_float)), \439 orP([ \440 newP(r"\[Full GC\s*\(System\)\s*", mkDictModifier("system", get_true)), \441 newP(r"\[Full GC\s*", None), \442 ]), \443 newP(regexp_float_colon, None), \444 newP(r"\[CMS:\s+", None), \445 newP(regexp_heap_info + r",\s+", mkDictModifier("heap_cms", get_int3)), \446 newP(regexp_float + r"\s*secs\]\s*", None), \447 newP(regexp_heap_info, mkDictModifier("heap_all", get_int3)), \448 newP(r"\s*,\s*\[CMS Perm\s*:\s*", None), \449 newP(regexp_heap_info, mkDictModifier("perm", get_int3)), \450 newP(r"\]\s*(?:icms_dc=\d+\s*)?", None), \451 newP(r",\s*", None), \452 newP(regexp_float + r"\s*secs\]\s*", mkDictModifier("response", get_float)), \453 newP(r"\[Times:.*\]$", None), \454])455if __debug__:456 text = r"7.992: [Full GC 7.992: [CMS: 6887K->19772K(65536K), 0.4137230 secs] 34678K->19772K(114688K), [CMS Perm : 54004K->53982K(54152K)] icms_dc=0 , 0.4140100 secs] [Times: user=0.68 sys=0.14, real=0.41 secs]"457 (ret, data) = parseFullGC(text, {})458 print text459 print len(ret)460 print data461 text = r"123.533: [Full GC (System) 123.533: [CMS: 39710K->34052K(65536K), 0.4852070 secs] 62832K->34052K(114688K), [CMS Perm : 77479K->76395K(128928K)], 0.4853310 secs] [Times: user=0.47 sys=0.01, real=0.48 secs]"462 (ret, data) = parseFullGC(text, {})463 print text464 print len(ret)465 print data466# This is for -XX:+UseParallelGC467parseParallelGC = andP([ \468 mkTagger("type", "ParallelGC"), \469 newP(regexp_float_colon, mkDictModifier("timestamp", get_float)), \470 newP(r"\[GC\s+\[PSYoungGen:\s*", None), \471 newP(regexp_heap_info + r"\s*\]\s*", mkDictModifier("heap_new", get_int3)), \472 newP(regexp_heap_info + r"\s*,\s*", mkDictModifier("heap_all", get_int3)), \473 newP(regexp_float + r"\s*secs\s*\]\s*", mkDictModifier("response", get_float)), \474 newP(r"\[Times:.*\]$", None), \475])476if __debug__:477 text = r"162.002: [GC [PSYoungGen: 39323K->3653K(49152K)] 87187K->56999K(114688K), 0.0207580 secs] [Times: user=0.08 sys=0.00, real=0.02 secs]"478 (ret, data) = parseParallelGC(text, {})479 print text480 print len(ret)481 print data482# This is for -XX:+UseParallelGC483parseParallelFullGC = andP([ \484 mkTagger("type", "ParallelFullGC"), \485 newP(regexp_float_colon, mkDictModifier("timestamp", get_float)), \486 orP([ \487 newP(r"\[Full GC\s*\(System\)\s*\[PSYoungGen:\s*", mkDictModifier("system", get_true)), \488 newP(r"\[Full GC\s*\[PSYoungGen:\s*", None), \489 ]), \490 newP(regexp_heap_info + r"\s*\]\s*", mkDictModifier("heap_new", get_int3)), \491 newP(r"\[PSOldGen:\s*", None), \492 newP(regexp_heap_info + r"\s*\]\s*", mkDictModifier("heap_old", get_int3)), \493 newP(regexp_heap_info + r"\s*", mkDictModifier("heap_all", get_int3)), \494 newP(r"\[PSPermGen:\s*", None), \495 newP(regexp_heap_info + r"\s*\]\s*,\s*", mkDictModifier("perm", get_int3)), \496 newP(regexp_float + r"\s*secs\s*\]\s*", mkDictModifier("response", get_float)), \497 newP(r"\[Times:.*\]$", None), \498])499if __debug__:500 text = r"162.657: [Full GC [PSYoungGen: 6189K->0K(50752K)] [PSOldGen: 58712K->43071K(65536K)] 64902K->43071K(116288K) [PSPermGen: 81060K->81060K(81152K)], 0.3032230 secs] [Times: user=0.30 sys=0.00, real=0.30 secs]"501 (ret, data) = parseParallelFullGC(text, {})502 print text503 print len(ret)504 print data505# This is for -XX:+UseSerialGC506parseSerialGC = andP([ \507 mkTagger("type", "SerialGC"), \508 newP(regexp_float_colon, mkDictModifier("timestamp", get_float)), \509 newP(r"\[GC\s+", None), \510 newP(regexp_float_colon + r"\[DefNew:\s*", None), \511 newP(regexp_heap_info + r"\s*,\s*", mkDictModifier("heap_new", get_int3)), \512 newP(regexp_float + r"\s*secs\s*\]\s*", None), \513 newP(regexp_heap_info + r"\s*,\s*", mkDictModifier("heap_all", get_int3)), \514 newP(regexp_float + r"\s*secs\s*\]\s*", mkDictModifier("response", get_float)), \515 newP(r"\[Times:.*\]$", None), \516])517if __debug__:518 text = r"4.687: [GC 4.687: [DefNew: 33343K->649K(49152K), 0.0021450 secs] 45309K->12616K(114688K), 0.0021800 secs] [Times: user=0.00 sys=0.00, real=0.00 secs]"519 (ret, data) = parseSerialGC(text, {})520 print text521 print len(ret)522 print data523# This is for -XX:+UseSerialGC524parseSerialFullGC = andP([ \525 mkTagger("type", "SerialFullGC"), \526 newP(regexp_float_colon, mkDictModifier("timestamp", get_float)), \527 newP(r"\[Full GC\s+", None), \528 newP(regexp_float_colon + r"\s*", None), \529 newP(r"\[Tenured:\s*", None), \530 newP(regexp_heap_info + r"\s*,\s*", mkDictModifier("heap_old", get_int3)), \531 newP(regexp_float + r"\s*secs\]\s*", None), \532 newP(regexp_heap_info + r"\s*,\s*", mkDictModifier("heap_all", get_int3)), \533 newP(r"\[Perm\s*:\s*", None), \534 newP(regexp_heap_info + r"\s*\]\s*,\s*", mkDictModifier("perm", get_int3)), \535 newP(regexp_float + r"\s*secs\s*\]\s*", mkDictModifier("response", get_float)), \536 newP(r"\[Times:.*\]$", None), \537])538if __debug__:539 text = r"4.899: [Full GC 4.899: [Tenured: 11966K->12899K(65536K), 0.1237750 secs] 22655K->12899K(114688K), [Perm : 32122K->32122K(32128K)], 0.1238590 secs] [Times: user=0.11 sys=0.00, real=0.13 secs]"540 (ret, data) = parseSerialFullGC(text, {})541 print text542 print len(ret)543 print data544"""545Java GC Log parser.546This supports almost kinds of GC provided by JVM.547-XX:+UseConcSweepGC (-XX:+UseParNewGC)548 parseParNew549 parseFullGC550-XX:+UseConcSweepGC -XX:CMSIncrementalMode (-XX:+UseParNewGC)551 parseParNew, parseFullGC,552 parse{InitialMark, MarkStart, Mark, PrecleanStart, Preclean,553 AbortablePrecleanStart, AbortablePreclean,554 AbortablePrecleanFullGC0, AbortablePrecleanFullGC1,555 AbortablePrecleanFailureTime,556 Remark,557 SweepStart, Sweep, ResetStart, Reset}558 parseAbortablePrecleanFullGC0 and parseAbortablePrecleanFullGC1559 must be always together.560-XX:+UseParallelGC561 parseParallelFullGC, parseParallelGC.562-XX:+UseSerialGC563 parseSerialFullGC, parseSerialGC.564 565"""566parseJavaGcLog = orP([ \567 parseParNew, \568 parseFullGC, \569 parseInitialMark, \570 parseMarkStart, parseMark, \571 parsePrecleanStart, parsePreclean, \572 parseAbortablePrecleanStart, parseAbortablePreclean, \573 parseAbortablePrecleanFullGC0, \574 parseAbortablePrecleanFullGC1, 575 parseAbortablePrecleanFailureTime, \576 parseRemark, \577 parseSweepStart, parseSweep, \578 parseResetStart, parseReset, \579 parseParallelFullGC, \580 parseParallelGC, \581 parseSerialFullGC, \582 parseSerialGC, \583])584################################################################################585# Parser of list of integer. This is for test.586################################################################################587"""588A modifier for list.589return :: ([[String]], [String]) -> [String]590"""591def mkListAppender():592 def listAppend_(list, matchStringL):593 if len(matchStringL) > 0:594 list.append(matchStringL[0])595 return list596 return listAppend_597"""598Convert last element to Int.599list :: [Int, Int, ..., Int, String]600return :: [Int, Int, ..., Int, Int]601"""602def convertLastToInt(list):603 list[-1] = int(list[-1])604 return list605# Parser of list of integer. This is for test.606parseIntList = andP([ \607 newP(r"\s*\[\s*", None), \608 manyP(andP([ \609 newP("(\d+)\s*(?:,\s*)?", mkListAppender()), \610 appP(convertLastToInt), \611 ])), \612 newP(r"\s*\]\s*", None), \613])614if __debug__:615 text = r"[10, 20, 30]"616 (ret, data) = parseIntList(text, [])617 print text618 print len(ret)619 print data620################################################################################621# main622################################################################################623 624data_prev = None625for line in sys.stdin:626 try:627 text = line.rstrip()628 #print text629 (ret, data) = parseJavaGcLog(text, {})630 if data["type"] == "CMS-concurrent-abortable-preclean-fullgc0":631 data_prev = data632 continue633 if data["type"] == "CMS-concurrent-abortable-preclean-fullgc1":634 assert data_prev["type"] == "CMS-concurrent-abortable-preclean-fullgc0"635 data_prev.update(data)636 data = data_prev637 data_prev = None638 data["type"] = "CMS-concurrent-abortable-preclean-fullgc"639 if __debug__:640 print ("len: %d" % len(ret))641 print json.dumps(data)642 except ParseError, msg:643 #print msg644 print ("###%s" % text)...
load.py
Source:load.py
1#2# Copyright (c) 2009 Testrepository Contributors3#4# Licensed under either the Apache License, Version 2.0 or the BSD 3-clause5# license at the users choice. A copy of both licenses are available in the6# project source as Apache-2.0 and BSD. You may not use this file except in7# compliance with one of these two licences.8#9# Unless required by applicable law or agreed to in writing, software10# distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the12# license you chose for the specific language governing permissions and13# limitations under that license.14"""Load data into a repository."""15from functools import partial16from operator import methodcaller17import optparse18import threading19import subunit.test_results20import testtools21from testrepository.arguments.path import ExistingPathArgument22from testrepository.commands import Command23from testrepository.repository import RepositoryNotFound24from testrepository.testcommand import TestCommand25class InputToStreamResult(object):26 """Generate Stream events from stdin.27 Really a UI responsibility?28 """29 def __init__(self, stream):30 self.source = stream31 self.stop = False32 def run(self, result):33 while True:34 if self.stop:35 return36 char = self.source.read(1)37 if not char:38 return39 if char == b'a':40 result.status(test_id='stdin', test_status='fail')41class load(Command):42 """Load a subunit stream into a repository.43 Failing tests are shown on the console and a summary of the stream is44 printed at the end.45 Unless the stream is a partial stream, any existing failures are discarded.46 """47 input_streams = ['subunit+', 'interactive?']48 args = [ExistingPathArgument('streams', min=0, max=None)]49 options = [50 optparse.Option("--partial", action="store_true",51 default=False, help="The stream being loaded was a partial run."),52 optparse.Option(53 "--force-init", action="store_true",54 default=False,55 help="Initialise the repository if it does not exist already"),56 optparse.Option("--subunit", action="store_true",57 default=False, help="Display results in subunit format."),58 optparse.Option("--full-results", action="store_true",59 default=False,60 help="No-op - deprecated and kept only for backwards compat."),61 ]62 # Can be assigned to to inject a custom command factory.63 command_factory = TestCommand64 def run(self):65 path = self.ui.here66 try:67 repo = self.repository_factory.open(path)68 except RepositoryNotFound:69 if self.ui.options.force_init:70 repo = self.repository_factory.initialise(path)71 else:72 raise73 testcommand = self.command_factory(self.ui, repo)74 # Not a full implementation of TestCase, but we only need to iterate75 # back to it. Needs to be a callable - its a head fake for76 # testsuite.add.77 # XXX: Be nice if we could declare that the argument, which is a path,78 # is to be an input stream - and thus push this conditional down into79 # the UI object.80 if self.ui.arguments.get('streams'):81 opener = partial(open, mode='rb')82 streams = map(opener, self.ui.arguments['streams'])83 else:84 streams = self.ui.iter_streams('subunit')85 mktagger = lambda pos, result:testtools.StreamTagger(86 [result], add=['worker-%d' % pos])87 def make_tests():88 for pos, stream in enumerate(streams):89 # Calls StreamResult API.90 case = subunit.ByteStreamToStreamResult(91 stream, non_subunit_name='stdout')92 decorate = partial(mktagger, pos)93 case = testtools.DecorateTestCaseResult(case, decorate)94 yield (case, str(pos))95 case = testtools.ConcurrentStreamTestSuite(make_tests)96 # One unmodified copy of the stream to repository storage97 inserter = repo.get_inserter(partial=self.ui.options.partial)98 # One copy of the stream to the UI layer after performing global99 # filters.100 try:101 previous_run = repo.get_latest_run()102 except KeyError:103 previous_run = None104 output_result, summary_result = self.ui.make_result(105 inserter.get_id, testcommand, previous_run=previous_run)106 result = testtools.CopyStreamResult([inserter, output_result])107 runner_thread = None108 result.startTestRun()109 try:110 # Convert user input into a stdin event stream111 interactive_streams = list(self.ui.iter_streams('interactive'))112 if interactive_streams:113 case = InputToStreamResult(interactive_streams[0])114 runner_thread = threading.Thread(115 target=case.run, args=(result,))116 runner_thread.daemon = True117 runner_thread.start()118 case.run(result)119 finally:120 result.stopTestRun()121 if interactive_streams and runner_thread:122 runner_thread.stop = True123 runner_thread.join(10)124 if not summary_result.wasSuccessful():125 return 1126 else:...
Automation Testing Tutorials
Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.
LambdaTest Learning Hubs:
- JUnit Tutorial
- TestNG Tutorial
- Webdriver Tutorial
- WebDriverIO Tutorial
- Protractor Tutorial
- Selenium 4 Tutorial
- Jenkins Tutorial
- NUnit Tutorial
- Jest Tutorial
- Playwright Tutorial
- Cypress Tutorial
- PyTest Tutorial
YouTube
You could also refer to video tutorials over LambdaTest YouTube channel to get step by step demonstration from industry experts.
Run stestr automation tests on LambdaTest cloud grid
Perform automation testing on 3000+ real desktop and mobile devices online.
Try LambdaTest Now !!
Get 100 minutes of automation test minutes FREE!!
Was this article helpful?
