How to use get_comments method in Kiwi

Best Python code snippet using Kiwi_python

douban_crawer.py

Source:douban_crawer.py

...18            #time.sleep(2)19        20        url = url_base%str(get_comments.run)21#        page = download_page(url)22#        get_comments(page,comments)23#        print('Page : {0}, Comments {1} have been crawered.'.format(str(get_comments.page),str(get_comments.run)))24        try:25            page = download_page(url)26            get_comments(page,comments)27            print('Page : {0}, Comments {1} have been crawered.'.format(str(get_comments.page),str(get_comments.run)))28        except AttributeError:29            get_comments.page -= 130            page = Login_douban(url)31            #print(page)32            try:33                get_comments(page,comments)34                print('Page : {0}, Comments {1} have been crawered.'.format(str(get_comments.page),str(get_comments.run)))35            except AttributeError:36                print('Crawer Over!')37                break38        except Exception as e:39            print(e)40            break41    save_json2jsonfile('./crawed/Comments.json',comments)42    save_json2normalfile('./crawed/Comments.txt',comments)43    44def Login_douban(redir_url):45    "Build a session to post login data"46    s = requests.Session()47    login_url = "https://accounts.douban.com/login"48    login_infor = {49            'redir':redir_url,50            'form_email':'input Douban User Login',51            'form_password':'input Douban User Password',52            'login':u'ç»å½'53            }54    login_infor_copy = login_infor.copy()55    while 1:56        content = s.post(login_url,data = login_infor_copy)57        "Input Identifying Code"58        try:59            soup = BeautifulSoup(content.text,'html.parser')60            captcha_url = soup.find('img',id='captcha_image')['src']61        except:62            return content.text63        if not len(captcha_url) == 0:64            "using re to obtain Identifying Code"65            pattern = re.compile('<input type="hidden" name="captcha-id" value="(.*?)"/')66            captcha_id = re.findall(pattern, content.text)67            "save Identifying Code into local"68            urllib.request.urlretrieve(captcha_url,"./crawed/captcha.jpg")69            captcha_img = Image.open("./crawed/captcha.jpg");captcha_img.show()70            captcha = input('please input the captcha:')71            login_infor['captcha-solution'] = captcha72            login_infor['captcha-id'] = captcha_id73            content = s.post(login_url,data = login_infor)74            #print(realcontent.text)75            if u"è¸éªå¯»æ¢" in content.text:76                print("Login Succeed.")77                break78            else:79                print("Login Failed.")80                continue81    return content.text82def download_page(url):83    r = requests.get(url)84    return r.text85def save_json2jsonfile(file_path,dicts):86    '''87        save json data into file88    '''89    with open(file_path,'w') as f:90        json.dump(dicts,sort_keys = True,indent = 4,fp = f,ensure_ascii=False)91    return None92def save_json2normalfile(file_path,dicts):93    '''94        save json data into file95    '''96    with open(file_path,'w') as f:97        f.writelines('Film : '+ dicts.pop('film')+'\n')98        for key in ['Comment '+ str(i+1) for i in range(len(dicts.keys()))]:99            line = key+'\n\t'+'\n\t'.join([i+' : '+dicts[key][i] for i in sorted(dicts[key].keys(),reverse = True)])+'\n'100            f.writelines(line)101    with open(file_path[:-4]+'_content'+file_path[-4:],'w') as f:102        for key in ['Comment '+ str(i+1) for i in range(len(dicts.keys()))]:103            line = dicts[key]['Comment Content'] + '\n'104            f.writelines(line)105    return None106def get_comments(page,comments):107    get_comments.page += 1108    soup = BeautifulSoup(page,'html.parser')109#    print(soup.find(id = "comments").get_text())110    con = soup.find(id = "comments")111    con_list = con.find_all('div', class_="comment-item")112    for single in con_list:113        get_comments.run += 1114        "comment html stucture"115        single_con = single.find('div',class_ = 'comment')116        "Get comment information: user_name, user_star, user_comment_time"117        single_infor = single_con.find('span',class_ = 'comment-info')118        single_user = single_infor.find('a').get_text()119        single_star = single_infor.find_all('span')[1]['class']120        if not single_star[-1] == 'rating':...

urls.py

Source:urls.py

1from django.conf.urls import url2from ppcn import views3urlpatterns = [4    url(5        r'^api/v1/ppcn/geographic/level/(?P<language>es|en)*/*',6        views.get_geographic_level,7        name='get_geographic_level'8    ),9    url(10        r'^api/v1/ppcn/required/level/(?P<language>es|en)/*',11        views.get_required_level,12        name='get_required_level'13    ),14    url(15        r'^api/v1/ppcn/recognition/type/(?P<language>[A-Za-z]*)/*$',16        views.get_recognition_type,17        name='get_recognition_type'18    ),19    url(20        r'^api/v1/ppcn/(?P<id>[0-9]+)/sector/(?P<language>[A-Za-z]*)/*$',21        views.get_sector,22        name='get_sector'23    ),24    url(25        r'^api/v1/ppcn/(?P<pk>[0-9]+)/subsector/(?P<language>[A-Za-z]*)/*$',26        views.get_sub_sector,27        name='get_sub_sector'28    ),29    url(r'^api/v1/ppcn/(?P<language>es|en)*/*$',30        views.get_post_ppcn,31        name='get_post_ppcn'32    ), 33    url(34        r'^api/v1/ppcn/(?P<id>[0-9a-fA-F-]+)/ppcn_file/(?P<ppcn_file_id>[0-9a-fA-F-]+)/*$',35        views.get_ppcn_file,36        name='get_ppcn_file'37    ),38     url(r'^api/v1/ppcn/file/*$',39        views.post_ppcn_file,40        name='post_ppcn_file'41    ),42    url(43        r'^api/v1/ppcn/ovv/*$',44        views.get_all_ovv,45        name='get_all_ovv'46    ),47    48    url(r'^api/v1/ppcn/all/*(?P<language>es|en)*/*$',49        views.get_all_ppcn,50        name='get_all_ppcn'51    ), 52    url(r'^api/v1/ppcn/(?P<id>[0-9a-f-]+)/send/*$',53        views.send_to_review,54        name='send_to_review'55    ), 56    url(r'^api/v1/ppcn/(?P<id>[0-9a-f-]+)/(?P<language>es|en)*/*$',57        views.get_one_ppcn,58        name='get_one_ppcn'59    ), 60    url(r'^api/v1/ppcn/(?P<id>[0-9a-f-]+)/*$',61        views.put_delete_patch_ppcn,62        name='put_delete_patch_ppcn'63    ),64    url(r'^api/v1/ppcn/form/(?P<geographicLevel_id>[0-9]+)/(?P<language>es|en)*/*$',65        views.get_form_ppcn,66        name='get_form_ppcn'67    ),68    url(69        r'^api/v1/ppcn/changelog/(?P<id>[0-9a-f-]+)$',70        views.get_ppcn_change_log,71        name='get_ppcn_change_log'72    ),73    url(74        r'^api/v1/ppcn/(?P<id>[0-9a-fA-F-]+)/file/(?P<ppcn_file_id>[0-9a-zA-Z-]+)/*$',75        views.get_ppcn_file_version,76        name='get_ppcn_file_version'77    ),78    url(r'^api/v1/ppcn/(?P<ppcn_id>[0-9a-f-]+)/comments/*$',79        views.get_comments,80        name='get_comments'81    ),82    url(r'^api/v1/ppcn/(?P<ppcn_id>[0-9a-f-]+)/(?P<fsm_state>[A-Za-z0-9\._-]+)/comments/*$',83        views.get_comments,84        name='get_comments'85    ),86    url(r'^api/v1/ppcn/(?P<ppcn_id>[0-9a-f-]+)/(?P<fsm_state>[A-Za-z0-9\._-]+)/review/(?P<review_number>[0-9]+)/comments/*$',87        views.get_comments,88        name='get_comments'89    ),90    url(r'^api/v1/ppcn/(?P<ppcn_id>[0-9a-f-]+)/review/(?P<review_number>[0-9]+)/comments/*$',91        views.get_comments,92        name='get_comments'93    ),94    url(r'^api/v1/ppcn/(?P<ppcn_id>[0-9a-f-]+)/comments/*$',95        views.get_comments,96        name='get_comments'97    ),...

test_commentgetlist.py

Source:test_commentgetlist.py

...11@unittest.mock.patch('pycamunda.task.Comment.load', unittest.mock.MagicMock())12@unittest.mock.patch('requests.Session.request')13def test_commentgetlist_calls_requests(mock, engine_url, task_input):14    get_comments = pycamunda.task.CommentGetList(url=engine_url, task_id='anId')15    get_comments()16    assert mock.called17    assert mock.call_args[1]['method'].upper() == 'GET'18@unittest.mock.patch('pycamunda.task.Comment.load', unittest.mock.MagicMock())19@unittest.mock.patch('requests.Session.request', raise_requests_exception_mock)20def test_commentgetlist_raises_pycamunda_exception(engine_url, task_input):21    get_comments = pycamunda.task.CommentGetList(url=engine_url, task_id='anId')22    with pytest.raises(pycamunda.PyCamundaException):23        get_comments()24@unittest.mock.patch('pycamunda.task.Comment.load', unittest.mock.MagicMock())25@unittest.mock.patch('requests.Session.request', not_ok_response_mock)26@unittest.mock.patch('pycamunda.base._raise_for_status')27def test_commentgetlist_raises_for_status(mock, engine_url, task_input):28    get_comments = pycamunda.task.CommentGetList(url=engine_url, task_id='anId')29    get_comments()30    assert mock.called31@unittest.mock.patch('requests.Session.request', unittest.mock.MagicMock())32@unittest.mock.patch('pycamunda.base.from_isoformat', unittest.mock.MagicMock())33def test_commentgetlist_returns_comment(engine_url, task_input):34    get_comments = pycamunda.task.CommentGetList(url=engine_url, task_id='anId')35    comments = get_comments()36    assert isinstance(comments, tuple)...

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.