Test your AI Agents with the all-new Agent to Agent Testing Platform.Learn More

How to use skip_task method in Lemoncheesecake

Best Python code snippet using lemoncheesecake

upgrade.py

Source:upgrade.py

...46            self.device["host"] = str(task.device.primary_ip.address.ip)47        else:48            msg = "No primary (mgmt) address"49            self.warning(msg)50            self.skip_task(msg, TaskFailReasonChoices.FAIL_CHECK)51    def debug(self, msg):52        log.debug(f"{self.log_id} - {msg}")53        self.task.log += f'{datetime.now(pytz.timezone(TIME_ZONE)).strftime("%Y-%m-%d %H:%M:%S")} - DEBUG - {msg}\n'54        self.task.save()55    def info(self, msg):56        log.info(f"{self.log_id} - {msg}")57        self.task.log += f'{datetime.now(pytz.timezone(TIME_ZONE)).strftime("%Y-%m-%d %H:%M:%S")} - INFO - {msg}\n'58        self.task.save()59    def warning(self, msg):60        log.warning(f"{self.log_id} - {msg}")61        self.task.log += f'{datetime.now(pytz.timezone(TIME_ZONE)).strftime("%Y-%m-%d %H:%M:%S")} - WARNING - {msg}\n'62        self.task.save()63    def error(self, msg):64        log.error(f"{self.log_id} - {msg}")65        self.task.log += f'{datetime.now(pytz.timezone(TIME_ZONE)).strftime("%Y-%m-%d %H:%M:%S")} - ERROR - {msg}\n'66        self.task.save()67    def action_task(self, action, msg, reason):68        self.task.status = action69        self.task.message = msg70        self.task.fail_reason = reason71        self.task.save()72        raise UpgradeException(73            reason=reason,74            message=msg,75        )76    def skip_task(self, msg="", reason=""):77        self.action_task(TaskStatusChoices.STATUS_SKIPPED, msg, reason)78    def drop_task(self, msg="", reason=""):79        self.action_task(TaskStatusChoices.STATUS_FAILED, msg, reason)80    def check(self):81        if not hasattr(self.task.device.device_type, "golden_image"):82            msg = f"No Golden Image for {self.task.device.device_type.model}"83            self.warning(msg)84            self.skip_task(msg, TaskFailReasonChoices.FAIL_CHECK)85        else:86            self.debug(87                f"Golden Image for {self.task.device.device_type.model} is {self.task.device.device_type.golden_image.sw}"88            )89        if self.task.start_time > self.task.scheduled_time + timedelta(hours=int(self.task.mw_duration)):90            msg = "Maintenance Window is over"91            self.warning(msg)92            self.skip_task(msg, TaskFailReasonChoices.FAIL_CHECK)93        else:94            self.debug("MW is still active")95        if self.task.task_type == TaskTypeChoices.TYPE_UPGRADE:96            q = get_queue(UPGRADE_QUEUE)97            active_jobs = q.started_job_registry.count98            non_ack = ScheduledTask.objects.filter(start_time__isnull=False, confirmed=False).count()99            if non_ack >= active_jobs + UPGRADE_THRESHOLD:100                msg = f"Reached failure threshold: Unconfirmed: {non_ack}, active: {active_jobs}, failed: {non_ack-active_jobs}, threshold: {UPGRADE_THRESHOLD}"101                self.warning(msg)102                self.skip_task(msg, TaskFailReasonChoices.FAIL_CHECK)103            else:104                self.debug(105                    f"Unconfirmed: {non_ack}, active: {active_jobs}, failed: {non_ack - active_jobs}, threshold: {UPGRADE_THRESHOLD}"106                )107        else:108            self.debug(f"Task type is {self.task.task_type}, check against threshold was skipped")109    def connect_cli(self, **kwargs):110        def to_telnet(cli, **kwargs):111            try:112                cli.close()113            except Exception:114                pass115            cli = False116            if self.device["port"] != 23:117                self.debug("Swiching to telnet")118                self.device["port"] = 23119                self.device["transport"] = "telnet"120                cli = self.connect_cli(**kwargs)121            return cli122        cli = IOSXEDriver(**self.device, **kwargs)123        try:124            self.debug(f'Trying to connect via TCP/{self.device["port"]} ...')125            cli.open()126        except ScrapliAuthenticationFailed:127            self.debug(f'Incorrect username while connecting to the device via TCP/{self.device["port"]}')128            cli = to_telnet(cli, **kwargs)129        except ScrapliConnectionError:130            self.debug(f'Device closed connection on TCP/{self.device["port"]}')131            # raise132            cli = to_telnet(cli, **kwargs)133        except Exception:134            self.debug(f'Unknown error while connecting to the device via TCP/{self.device["port"]}')135            cli = to_telnet(cli, **kwargs)136        else:137            self.debug(f'Login successful while connecting to the device via TCP/{self.device["port"]}')138        return cli139    def is_alive(self):140        try:141            with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:142                s.settimeout(self.device.get("timeout_socket", 5))143                s.connect((self.device["host"], 22))144        except Exception:145            self.debug("no response on TCP/22")146            try:147                with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:148                    s.settimeout(self.device.get("timeout_socket", 5))149                    s.connect((self.device["host"], 23))150            except Exception:151                self.debug("no response on TCP/23")152                time.sleep(2)153                return False154            else:155                self.debug("got response on TCP/23")156        else:157            self.debug("got response on TCP/22")158        time.sleep(2)159        return True160    def check_device(self):161        pid = ""162        sn = ""163        cmd = [164            "show version",165            "dir /all",166        ]167        cli = self.connect_cli()168        if not cli:169            msg = "Can not connect to device CLI"170            self.error(msg)171            self.skip_task(msg, TaskFailReasonChoices.FAIL_CONNECT)172        output = cli.send_commands(cmd)173        cli.close()174        if output.failed:175            msg = "Can not collect outputs from device"176            self.error(msg)177            self.skip_task(msg, TaskFailReasonChoices.FAIL_CONFIG)178        self.debug("----------vv Outputs vv----------")179        self.debug(output.result)180        self.debug("----------^^ Outputs ^^----------")181        r = re.search(r"\n\w+\s+(\S+)\s+.*\(revision\s+", output[0].result)182        if r:183            pid = r.group(1)184            # pid = re.sub('\+','plus',r.group(1))185            self.info(f"PID: {r.group(1)}")186        else:187            msg = "Can not get device PID"188            self.error(msg)189            self.skip_task(msg, reason=TaskFailReasonChoices.FAIL_CONFIG)190        r = re.search(r"\n.*\s+board\s+ID\s+(\S+)", output[0].result)191        if r:192            sn = r.group(1)193            self.info(f"SN: {sn}")194        else:195            msg = "Can not get device SN"196            self.error(msg)197            self.skip_task(msg, reason=TaskFailReasonChoices.FAIL_CONFIG)198        if pid.upper() != self.task.device.device_type.model.upper() or sn.lower() != self.task.device.serial.lower():199            msg = "Device PID/SN does not match with NetBox data"200            self.error(msg)201            self.skip_task(msg, reason=TaskFailReasonChoices.FAIL_CONFIG)202        self.info(f"Device {pid}/{sn} matches with NetBox data")203        self.files = output[1].textfsm_parse_output()204        self.file_system = self.files[0]["file_system"].strip("/")205        self.target_image = self.task.device.device_type.golden_image.sw.filename206        self.target_path = self.task.device.device_type.golden_image.sw.image.path207        self.image_on_device = list(filter(lambda x: x["name"] == self.target_image, self.files))208        self.debug(f"File system: {self.file_system}")209        self.debug(f"Target Image: {self.target_image}")210        self.debug(f"Target Path: {self.target_path}")211        self.debug(f"Target Image on box: {self.image_on_device}")212        return True213    def file_upload(self):214        if self.task.transfer_method == TaskTransferMethod.METHOD_FTP:215            cmd_copy = f"copy ftp://{FTP_USERNAME}:{FTP_PASSWORD}@{FTP_SERVER}/{self.target_image} {self.file_system}/{self.target_image}"216        elif self.task.transfer_method == TaskTransferMethod.METHOD_HTTP:217            cmd_copy = f"copy {HTTP_SERVER}{self.target_image} {self.file_system}/{self.target_image}"218        else:219            msg = "Unknown transfer method"220            self.error(msg)221            self.skip_task(msg, reason=TaskFailReasonChoices.FAIL_UPLOAD)222        config = [223            "file prompt quiet",224            "line vty 0 15",225            "exec-timeout 180 0",226        ]227        config_undo = [228            "no file prompt quiet",229            "line vty 0 15",230            "exec-timeout 30 0",231        ]232        cli = self.connect_cli(timeout_ops=7200, timeout_transport=7200)233        if not cli:234            msg = "Unable to connect to the device"235            self.error(msg)236            self.skip_task(msg, TaskFailReasonChoices.FAIL_CONNECT)237        if not len(self.image_on_device):238            self.info("No image on the device. Need to transfer")239            self.debug(240                f'Free on {self.file_system} {self.files[0]["total_free"]}, \241                Image size (+10%) {int(int(self.task.device.device_type.golden_image.sw.image.size)*1.1)}'242            )243            if int(self.files[0]["total_free"]) < int(244                int(self.task.device.device_type.golden_image.sw.image.size) * 1.1245            ):246                try:247                    cli.close()248                except Exception:249                    pass250                msg = f"No enough space on {self.file_system}"251                self.error(msg)252                self.skip_task(msg, TaskFailReasonChoices.FAIL_UPLOAD)253            self.info("Download image from FTP...")254            output = cli.send_configs(config)255            self.debug(f"Preparing for copy:\n{output.result}")256            if output.failed:257                try:258                    cli.close()259                except Exception:260                    pass261                msg = "Can not change configuration"262                self.error(msg)263                self.skip_task(msg, TaskFailReasonChoices.FAIL_UPLOAD)264            self.debug(f"Copy command: {cmd_copy}")265            output = cli.send_command(cmd_copy)266            self.debug(f"Copying process:\n{output.result}")267            if output.failed or not (re.search(r"OK", output.result) or re.search(r"bytes copied in", output.result)):268                try:269                    cli.close()270                except Exception:271                    pass272                msg = "Can not download image from server"273                self.error(msg)274                self.skip_task(msg, TaskFailReasonChoices.FAIL_UPLOAD)275            output = cli.send_configs(config_undo)276            self.debug(f"Rollback after copy:\n{output.result}")277            if output.failed:278                try:279                    cli.close()280                except Exception:281                    pass282                msg = "Can not do rollback configuration"283                self.error(msg)284                self.skip_task(msg, TaskFailReasonChoices.FAIL_UPLOAD)285        else:286            self.info(f"Image {self.target_image} already exists")287        self.info("MD5 verification ...")288        md5 = cli.send_command(289            f"verify /md5 {self.file_system}/{self.target_image} {self.task.device.device_type.golden_image.sw.md5sum}"290        )291        self.debug(f"MD5 verication result:\n{md5.result[-200:]}")292        if md5.failed:293            try:294                cli.close()295            except Exception:296                pass297            msg = "Can not check MD5"298            self.error(msg)299            self.skip_task(msg, TaskFailReasonChoices.FAIL_CHECK)300        if re.search(r"Verified", md5.result):301            self.info("MD5 was verified")302        else:303            try:304                cli.close()305            except Exception:306                pass307            msg = "Wrong M5"308            self.error(msg)309            self.skip_task(msg, TaskFailReasonChoices.FAIL_CHECK)310        try:311            cli.close()312        except Exception:313            pass314        self.info("File was uploaded and verified")315        return True316    def device_reload(self):317        cmd = [318            "show run | i boot system",319            "show version",320        ]321        cli = self.connect_cli()322        if not cli:323            msg = "Unable to connect to the device"324            self.error(msg)325            self.skip_task(msg, TaskFailReasonChoices.FAIL_CONNECT)326        output = cli.send_commands(cmd)327        self.debug(f"Collected outputs:------vvvvv\n{output.result}\n-----^^^^^")328        if output.failed:329            try:330                cli.close()331            except Exception:332                pass333            msg = "Can not collect outputs for upgrade"334            self.error(msg)335            self.skip_task(msg, TaskFailReasonChoices.FAIL_UPGRADE)336        parsed = output[1].textfsm_parse_output()337        sw_current = parsed[0].get("version", "N/A")338        sw_target = self.task.device.device_type.golden_image.sw.version339        self.debug(f"Current version is {sw_current}")340        if sw_current.upper() == sw_target.upper():341            msg = f"Current version {sw_current} matches with target {sw_target}"342            self.warning(msg)343            self.info("Update custom field")344            self.task.device.custom_field_data[CF_NAME_SW_VERSION] = sw_current345            self.task.device.save()346            self.skip_task(msg, TaskFailReasonChoices.FAIL_UPGRADE)347        if not len(self.image_on_device):348            msg = "No target image on the box"349            self.error(msg)350            self.skip_task(msg, TaskFailReasonChoices.FAIL_UPGRADE)351        self.info("Image exists on the box")352        cli.timeout_ops = 600353        self.info("MD5 verification ...")354        md5 = cli.send_command(355            f"verify /md5 {self.file_system}/{self.target_image} {self.task.device.device_type.golden_image.sw.md5sum}"356        )357        self.debug(f"MD5 verication result:\n{md5.result[-200:]}")358        if md5.failed:359            try:360                cli.close()361            except Exception:362                pass363            msg = "Can not check MD5"364            self.error(msg)365            self.skip_task(msg, TaskFailReasonChoices.FAIL_CHECK)366        if re.search(r"Verified", md5.result):367            self.info("MD5 was verified")368        else:369            try:370                cli.close()371            except Exception:372                pass373            msg = "Wrong M5"374            self.error(msg)375            self.skip_task(msg, TaskFailReasonChoices.FAIL_CHECK)376        cli.timeout_ops = 10377        self.info("Preparing boot system config")378        new_boot_lines = []379        old_boot_lines = output[0].result.splitlines()380        self.debug(f"Orginal boot lines:\n{old_boot_lines}")381        for line in old_boot_lines:382            new_boot_lines.append(f"no {line}")383        new_boot_lines.append(f"boot system {self.file_system}/{self.target_image}")384        if len(old_boot_lines):385            new_boot_lines.append(old_boot_lines[0])386        self.debug(f"New boot lines:\n{new_boot_lines}")387        output = cli.send_configs(new_boot_lines)388        self.debug(f"Changnig Boot vars:\n{output.result}")389        if output.failed:390            msg = "Unable to change bootvar"391            self.error(msg)392            self.drop_task(msg, TaskFailReasonChoices.FAIL_UPGRADE)393        else:394            self.info("Bootvar was changed")395        self.info("Write memory before reload")396        try:397            output = cli.send_command("write memory")398        except (ScrapliTimeout, ScrapliConnectionError):399            self.info("Interactive prompt was detected")400            time.sleep(2)401            cli.open()402            try:403                output_tmp = cli.send_interactive(404                    [405                        ("write", "[confirm]", False),406                        ("\n", "#", False),407                    ]408                )409            except (ScrapliTimeout, ScrapliConnectionError):410                msg = "Unable to write memory: ScrapliTimeout"411                self.error(msg)412                self.drop_task(msg, TaskFailReasonChoices.FAIL_UPGRADE)413            else:414                output = output_tmp415        if re.search(r"\[OK\]", output.result):416            self.info("Config was saved")417        else:418            msg = "Can not save config"419            self.error(msg)420            self.drop_task(msg, TaskFailReasonChoices.FAIL_UPGRADE)421        self.info("Reloading the box")422        try:423            output = cli.send_interactive(424                [425                    ("reload in 1", "[confirm]", False),426                    ("\n", "#", False),427                ]428            )429        except ScrapliTimeout:430            msg = "Unable to reload: ScrapliTimeout"431            self.error(msg)432            self.drop_task(msg, TaskFailReasonChoices.FAIL_UPGRADE)433        else:434            self.info("Reload was requested")435        try:436            cli.close()437        except Exception:438            pass439        return True440    def post_check(self):441        cmd = [442            "show version",443        ]444        cli = self.connect_cli()445        if not cli:446            msg = "Unable to connect to the device"447            self.error(msg)448            self.drop_task(msg, TaskFailReasonChoices.FAIL_CONNECT)449        output = cli.send_commands(cmd)450        self.debug(f"Commands output\n{output.result}")451        if output.failed:452            msg = "Can not collect outputs for post-chech"453            self.error(msg)454            self.drop_task(msg, TaskFailReasonChoices.FAIL_UPGRADE)455        parsed = output[0].textfsm_parse_output()456        self.info(f'New version is {parsed[0].get("version", "N/A")}')457        self.info("Write memory after reload")458        try:459            output = cli.send_command("write memory")460        except (ScrapliTimeout, ScrapliConnectionError):461            self.info("Interactive prompt was detected")462            time.sleep(2)463            cli.open()464            try:465                output_tmp = cli.send_interactive(466                    [467                        ("write", "[confirm]", False),468                        ("\n", "#", False),469                    ]470                )471            except (ScrapliTimeout, ScrapliConnectionError):472                msg = "Unable to write memory: ScrapliTimeout"473                self.error(msg)474                self.drop_task(msg, TaskFailReasonChoices.FAIL_UPGRADE)475            else:476                output = output_tmp477        if re.search(r"\[OK\]", output.result):478            self.info("Config was saved")479        else:480            msg = "Can not save config"481            self.error(msg)482            self.drop_task(msg, TaskFailReasonChoices.FAIL_UPGRADE)483        cli.close()484        self.info("Update custom field")485        self.task.device.custom_field_data[CF_NAME_SW_VERSION] = parsed[0].get("version", "N/A")486        self.task.device.save()487        self.info("Post-checks have been done")488        return True489    def execute_task(self):490        self.info(f"New Job {self.task.job_id} was started. Type {self.task.task_type}")491        self.info("Initial task checking...")492        self.check()493        self.info("Initial task check has been completed")494        self.info("Checking if device alive...")495        if not self.is_alive():496            msg = f"Device {self.task.device.name}:{self.task.device.primary_ip.address.ip} is not reachable"497            self.warning(msg)498            self.skip_task(msg, TaskFailReasonChoices.FAIL_CONNECT)499        else:500            msg = f"Device {self.task.device.name}:{self.task.device.primary_ip.address.ip} is reachable"501            self.info(msg)502        self.info("Device valiation...")503        self.check_device()504        self.info("Device has been validated")505        if self.task.task_type == TaskTypeChoices.TYPE_UPLOAD:506            self.info("Uploadng image on the box...")507            self.file_upload()508        elif self.task.task_type == TaskTypeChoices.TYPE_UPGRADE:509            self.info("Reloading the box...")510            self.device_reload()511            hold_timer = 240512            self.info(f"Hold for {hold_timer} seconds")...

bot.py

Source:bot.py

...11from accounts import ACCOUNTS_LIST # list of api_ids and api_hashes12from messages import get_bot_dialogue, balance, withdraw_all13import keyboard14programm_is_running = True # changed by ctrl + shift + ] hotkey, exit while loop if false15async def skip_task(phone, err_msg, arg):16	'''17	Skips task(not always really skips)18	arg -- dictionary, {'tl_bot_chat': bot_chat_variable, 'msg': last_message(not string, but message object),19						'really_skip':True or False(read explanation), 'client': telethon_telgram_client}20	really_skip - some exceptions that occurs while getting on page do not mean that reward won't be given.21	I discovered that only captcha do not allow bot to get reward(see in captcha exception in main()). 22	So if variable is True(only captcha case) we really skip on new task if opposite, wait for default amount23	of seconds and go forward(if we had to wait more, main() will help by waiting new message)24	'''25	print(phone + ': ' + err_msg)26	if arg['really_skip']:27		await arg['client'](GetBotCallbackAnswerRequest(arg['tl_bot_chat'], arg['msg'].id, data=arg['msg'].reply_markup.rows[1].buttons[1].data))28		await asyncio.sleep(3)29	else:30		#define how long to wait31		msg = await arg['client'].get_messages(arg['tl_bot_chat'], limit=1)32		msg = msg[0]33		wait = str(msg.message).replace('You must stay on the site for', '').replace('seconds to get your reward.', '')34		wait = str(wait).replace('Please stay on the site for at least', '').replace('seconds...', '')35		try:36			wait = int(wait)37		except ValueError:38			print('Can not define how long to wait. Standart value is 15 seconds')39			wait = 1040		print('Wait for 15 seconds')41		await asyncio.sleep(wait + 5) # +5 for insurance42		43	print(phone + ': ' + 'Skipped!')44	45        46async def main(browser, accounts_list_slice):47	'''48	Main function where we choose account and talk to bot, then change when we need and repeat49	args:50	browser -- webdriver browser variable51	accounts_list_slice -- list of accounts that change each other if needed(when there is no task for very long)52	'''53	print('Main started...')54	start_session = time.time() # for calculating session duration55	global programm_is_running56	no_ads_iterator = 0 # count how many times there were no ads error57	account_iterator = 0 # read name58	while programm_is_running:59		#log into account60		log_data = accounts_list_slice[account_iterator]61		client = await tl.TelegramClient(log_data['phone'], log_data['api_id'], log_data['api_hash']).start()62		print('**Loged in as: ' + log_data['phone'] + '**')63		PREFIX_ID = log_data['phone'] + ': ' # should be in every print(), shows what number message relates to64		#get ltc bot dialogue65		tl_bot_chat = await get_bot_dialogue(client)66		print(PREFIX_ID + 'Found a LTC Click Bot chat')67		#get new link68		await client.send_message(tl_bot_chat, '/visit')69		print(PREFIX_ID + 'First /visit sent')70		71		#previous setup72		old_msg = None73		msg = await client.get_messages(tl_bot_chat, limit=1)74		msg = msg[0]75		for_skip_task = {'tl_bot_chat': tl_bot_chat, 'msg': None, 'really_skip':False, 'client': client}76		while programm_is_running:77			#wait if bot is lagging78			await asyncio.sleep(2)79			if re.search(r'there are no new ads available', msg.message) and programm_is_running:80				#if there is mo more ad81				no_ads_iterator += 1 #increment82				if no_ads_iterator >= 5:83					#if there is no ads for 5 times -> change account84					print(PREFIX_ID + 'No ads for this account now. Changing account...')85					account_iterator += 186					if account_iterator >= len(accounts_list_slice):87						account_iterator = 088					no_ads_iterator = 0 # new account starts at 0 no_ads_iterator89					await client.disconnect()90					break91				found_task = False92				#try for 5 more times93				print(PREFIX_ID + 'No ads observed. It may be a lie. Try /visit for 5 times in a row')94				for i in range(5):95					await client.send_message(tl_bot_chat, '/visit')96					await asyncio.sleep(2)97					msg = await client.get_messages(tl_bot_chat, limit=1)98					msg = msg[0]99					if not re.search(r'there are no new ads available', msg.message):100						#if found task break out of this function and go to the website101						print(PREFIX_ID + 'Found')102						found_task = True103						break104					print(PREFIX_ID + '#{} - No ads'.format(i))105				if not found_task:106					#if bot really do not have tasks for this account107					print(PREFIX_ID + 'Threre is no ad for {} times\nIf there will be no ad for {} times then change account'.format(no_ads_iterator, 5 - no_ads_iterator))108					print(PREFIX_ID + 'There is no more new ad. Sleep for 1 minute')109					print(PREFIX_ID + 'For exit press: ctrl + shift + ]')110					await asyncio.sleep(60) # sleep for a minute and check111					await client.send_message(tl_bot_chat, '/visit')112					print(PREFIX_ID + 'Get up and work!')113					await asyncio.sleep(2)114	115			#set time point before loop116			time_start = time.time()117			#reset msg118			msg = await client.get_messages(tl_bot_chat, limit=1)	119			msg = msg[0]120			#get message121			while msg == old_msg and programm_is_running:122				# exit only if new message123				msg = await client.get_messages(tl_bot_chat, limit=1)124				msg = msg[0]125				await asyncio.sleep(1)126				#check if we have waited for new message for too long127				if time.time() - time_start >= 95:128					try: 129						for_skip_task['msg'] = msg130						for_skip_task['really_skip'] = True131						await skip_task(log_data['phone'], 'There is no new message for too long', for_skip_task)132						for_skip_task['really_skip'] = False133					except AttributeError:134						print(PREFIX_ID + 'Last message was not a link')135						await client.send_message(tl_bot_chat, '/visit')136						await asyncio.sleep(5)137						msg = await client.get_messages(tl_bot_chat, limit=1)138						msg = msg[0]139						for_skip_task['msg'] = msg140						for_skip_task['really_skip'] = True141						try:142							await skip_task(log_data['phone'], 'Try skip for one more time', for_skip_task)143						except AttributeError:144							print(PREFIX_ID + 'Failed one more time')145							break146						finally:147							for_skip_task['really_skip'] = False148			#set new old_msg149			old_msg = msg150			# if got a url151			if re.search(r'Press', msg.message) and programm_is_running:152				no_ads_iterator = 0 # count how many times there were no ads error153				print(PREFIX_ID + 'Ad message sent: {}'.format(msg.date))154				visit_url = msg.reply_markup.rows[0].buttons[0].url155				print(PREFIX_ID + 'Ad URL: ' + visit_url)156				for_skip_task['msg'] = msg # for exceptions157				try: 158					#check for captcha159					url_site = urllib.request.urlopen(Request(visit_url, headers={'User-Agent' : 'Mozilla/5.0'}))160					captcha_str = url_site.read().decode('utf-8')161					url_site.close()162					if not re.search(r'reCAPTCHA', captcha_str):163						#go to URL164						browser.get(visit_url)165						print(PREFIX_ID + 'Page was opened succesfully\n\n')166					else:167						for_skip_task['really_skip'] = True168						await skip_task(log_data['phone'], 'Captcha was found on site. Skipping...', for_skip_task)169						for_skip_task['really_skip'] = False170				except selenium.common.exceptions.TimeoutException:171					await skip_task(log_data['phone'], 'Page loading timeout. Skipping...', for_skip_task)172				except TimeoutError:173					for_skip_task['really_skip'] = True174					await skip_task(log_data['phone'], 'Socket timeout. Skipping...', for_skip_task)175					for_skip_task['really_skip'] = False176				except ConnectionResetError:177					await skip_task(log_data['phone'], 'Connection reset. Skipping...', for_skip_task)178				except ConnectionRefusedError:179					await skip_task(log_data['phone'], 'Connection refused. Skipping...', for_skip_task)180				except urllib.error.HTTPError as e:181					await skip_task(log_data['phone'], 'Can not access the site. Skipping...' + str(e.reason), for_skip_task)182				except urllib.error.URLError:183					await skip_task(log_data['phone'], 'Bad certificate. Skipping...', for_skip_task)184				except UnicodeDecodeError:185					await skip_task(log_data['phone'], 'Can not decode text for captcha check')186			elif re.search(r'no longer valid', msg.message):187				# if skipped or some error appeared188				await client.send_message(tl_bot_chat, '/visit')189	190	print('Disconnecting with current client')191	await client.disconnect()	192def browser_setup():193	'''194	Setups one browser and returns browser variable195	'''196	#make browser headless197	print('Starting headless browser')198	options = webdriver.firefox.options.Options()199	options.headless = True # True or False...

worker.py

Source:worker.py

1# Copyright 2016 The Chromium Authors. All rights reserved.2# Use of this source code is governed by a BSD-style license that can be3# found in the LICENSE file.4import argparse5import json6import logging7import os8import random9import sys10import time11from googleapiclient import discovery12from oauth2client.client import GoogleCredentials13# NOTE: The parent directory needs to be first in sys.path to avoid conflicts14# with catapult modules that have colliding names, as catapult inserts itself15# into the path as the second element. This is an ugly and fragile hack.16_CLOUD_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)),17                          os.pardir)18sys.path.insert(0, os.path.join(_CLOUD_DIR, os.pardir))19# Add _CLOUD_DIR to the path to access common code through the same path as the20# frontend.21sys.path.append(_CLOUD_DIR)22from common.clovis_task import ClovisTask23import common.google_bigquery_helper24from common.google_instance_helper import GoogleInstanceHelper25from clovis_task_handler import ClovisTaskHandler26from failure_database import FailureDatabase27from google_storage_accessor import GoogleStorageAccessor28class Worker(object):29  def __init__(self, config, logger):30    """See README.md for the config format."""31    self._project_name = config['project_name']32    self._taskqueue_tag = config['taskqueue_tag']33    self._src_path = config['src_path']34    self._instance_name = config.get('instance_name')35    self._worker_log_path = config.get('worker_log_path')36    self._credentials = GoogleCredentials.get_application_default()37    self._logger = logger38    self._self_destruct = config.get('self_destruct')39    if self._self_destruct and not self._instance_name:40      self._logger.error('Self destruction requires an instance name.')41    # Separate the task storage path into the bucket and the base path under42    # the bucket.43    storage_path_components = config['task_storage_path'].split('/')44    self._bucket_name = storage_path_components[0]45    self._base_path_in_bucket = ''46    if len(storage_path_components) > 1:47      self._base_path_in_bucket = '/'.join(storage_path_components[1:])48      if not self._base_path_in_bucket.endswith('/'):49        self._base_path_in_bucket += '/'50    self._google_storage_accessor = GoogleStorageAccessor(51        credentials=self._credentials, project_name=self._project_name,52        bucket_name=self._bucket_name)53    if self._instance_name:54      failure_database_filename = \55          'failure_database_%s.json' % self._instance_name56    else:57      failure_database_filename = 'failure_dabatase.json'58    self._failure_database_path = os.path.join(self._base_path_in_bucket,59                                               failure_database_filename)60    # Recover any existing failures in case the worker died.61    self._failure_database = self._GetFailureDatabase()62    if self._failure_database.ToJsonDict():63      # Script is restarting after a crash, or there are already files from a64      # previous run in the directory.65      self._failure_database.AddFailure(FailureDatabase.DIRTY_STATE_ERROR,66                                        'failure_database')67    bigquery_service = common.google_bigquery_helper.GetBigQueryService(68        self._credentials)69    self._clovis_task_handler = ClovisTaskHandler(70        self._project_name, self._base_path_in_bucket, self._failure_database,71        self._google_storage_accessor, bigquery_service,72        config['binaries_path'], config['ad_rules_filename'],73        config['tracking_rules_filename'], self._logger, self._instance_name)74    self._UploadFailureDatabase()75  def Start(self):76    """Main worker loop.77    Repeatedly pulls tasks from the task queue and processes them. Returns when78    the queue is empty.79    """80    task_api = discovery.build('taskqueue', 'v1beta2',81                               credentials=self._credentials)82    queue_name = 'clovis-queue'83    # Workaround for84    # https://code.google.com/p/googleappengine/issues/detail?id=1019985    project = 's~' + self._project_name86    while True:87      self._logger.debug('Fetching new task.')88      (clovis_task, task_id) = self._FetchClovisTask(project, task_api,89                                                     queue_name)90      if not clovis_task:91        break92      self._logger.info('Processing task %s' % task_id)93      self._clovis_task_handler.Run(clovis_task)94      self._UploadFailureDatabase()95      self._logger.debug('Deleting task %s' % task_id)96      task_api.tasks().delete(project=project, taskqueue=queue_name,97                              task=task_id).execute()98      self._logger.info('Finished task %s' % task_id)99    self._Finalize()100  def _GetFailureDatabase(self):101    """Downloads the failure database from CloudStorage."""102    self._logger.info('Downloading failure database')103    failure_database_string = self._google_storage_accessor.DownloadAsString(104        self._failure_database_path)105    return FailureDatabase(failure_database_string)106  def _UploadFailureDatabase(self):107    """Uploads the failure database to CloudStorage."""108    if not self._failure_database.is_dirty:109      return110    self._logger.info('Uploading failure database')111    self._google_storage_accessor.UploadString(112        self._failure_database.ToJsonString(),113        self._failure_database_path)114    self._failure_database.is_dirty = False115  def _FetchClovisTask(self, project_name, task_api, queue_name):116    """Fetches a ClovisTask from the task queue.117    Params:118      project_name(str): The name of the Google Cloud project.119      task_api: The TaskQueue service.120      queue_name(str): The name of the task queue.121    Returns:122      (ClovisTask, str): The fetched ClovisTask and its task ID, or (None, None)123                         if no tasks are found.124    """125    response = task_api.tasks().lease(126        project=project_name, taskqueue=queue_name, numTasks=1, leaseSecs=600,127        groupByTag=True, tag=self._taskqueue_tag).execute()128    if (not response.get('items')) or (len(response['items']) < 1):129      return (None, None)  # The task queue is empty.130    google_task = response['items'][0]131    task_id = google_task['id']132    # Delete the task without processing if it already failed multiple times.133    # TODO(droger): This is a workaround for internal bug b/28442122, revisit134    # once it is fixed.135    retry_count = google_task['retry_count']136    max_retry_count = 3137    skip_task = retry_count >= max_retry_count138    if skip_task:139      task_api.tasks().delete(project=project_name, taskqueue=queue_name,140                              task=task_id).execute()141    clovis_task = ClovisTask.FromBase64(google_task['payloadBase64'])142    if retry_count > 0:143      self._failure_database.AddFailure('task_queue_retry',144                                        clovis_task.ToJsonString())145      self._UploadFailureDatabase()146    if skip_task:147      return self._FetchClovisTask(project_name, task_api, queue_name)148    return (clovis_task, task_id)149  def _Finalize(self):150    """Called before exiting."""151    self._logger.info('Done')152    self._clovis_task_handler.Finalize()153    # Upload the worker log.154    if self._worker_log_path:155      self._logger.info('Uploading worker log.')156      remote_log_path = os.path.join(self._base_path_in_bucket, 'worker_log')157      if self._instance_name:158        remote_log_path += '_' + self._instance_name159      self._google_storage_accessor.UploadFile(self._worker_log_path,160                                               remote_log_path)161    # Self destruct.162    if self._self_destruct:163      # Workaround for ComputeEngine internal bug b/28760288.164      random_delay = random.random() * 600.0  # Up to 10 minutes.165      self._logger.info(166          'Wait %.0fs to avoid load spikes on compute engine.' % random_delay)167      time.sleep(random_delay)168      self._logger.info('Starting instance destruction: ' + self._instance_name)169      google_instance_helper = GoogleInstanceHelper(170          self._credentials, self._project_name, self._logger)171      success = google_instance_helper.DeleteInstance(self._taskqueue_tag,172                                                      self._instance_name)173      if not success:174        self._logger.error('Self destruction failed.')175    # Do not add anything after this line, as the instance might be killed at176    # any time.177if __name__ == '__main__':178  parser = argparse.ArgumentParser(179      description='ComputeEngine Worker for Clovis')180  parser.add_argument('--config', required=True,181                      help='Path to the configuration file.')182  args = parser.parse_args()183  # Configure logging.184  logging.basicConfig(level=logging.WARNING,185                      format='[%(asctime)s][%(levelname)s] %(message)s',186                      datefmt='%y-%m-%d %H:%M:%S')187  logging.Formatter.converter = time.gmtime188  worker_logger = logging.getLogger('worker')189  worker_logger.setLevel(logging.INFO)190  worker_logger.info('Reading configuration')191  with open(args.config) as config_json:192    worker = Worker(json.load(config_json), worker_logger)...

task.py

Source:task.py

...70            if isinstance(dep_task.result, (TaskResultFailure, TaskResultSkipped)):71                reason = dep_task.result.reason72            else:73                reason = None74            skip_task(task, context, completed_task_queue, reason)75            return76    # skip task on external trigger if any77    skip_reason = context.is_task_to_be_skipped(task)78    if skip_reason:79        skip_task(task, context, completed_task_queue, reason=skip_reason)80        return81    # run task when all conditions are met82    run_task(task, context, completed_task_queue)83def schedule_tasks_to_be_run(tasks, context, pool, completed_tasks_queue):84    for task in tasks:85        pool.apply_async(handle_task, args=(task, context, completed_tasks_queue))86def skip_task(task, context, completed_task_queue, reason=""):87    _debug("skip task %s" % task)88    try:89        task.skip(context, reason)90    except Exception:91        task.result = TaskResultException(serialize_current_exception())92    else:93        task.result = TaskResultSkipped(reason)94    completed_task_queue.put(task)95def skip_all_tasks(tasks, remaining_tasks, completed_tasks, context, pool, completed_tasks_queue, reason):96    # schedule all tasks to be skipped...97    for task in remaining_tasks:98        pool.apply_async(skip_task, args=(task, context, completed_tasks_queue, reason))99    # ... and wait for their completion100    while len(completed_tasks) != len(tasks):...

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.