Unlock 30% off on Manual Testing Annual Plans this Holiday Season. 
Use Code: LCMYEARENDEXCL30
Use Code: LCMYEARENDEXCL30Check icon
Copied to Clipboard!
How to use doesrowexist method in pyatom
Best Python code snippet using pyatom_python
etllibrary.py
Source:etllibrary.py 
1#External Libraries2import json3import pymysql4import sys5import datetime6#Local Libraries7sys.path.insert(0, './lib')8from scraperlibrary import (9 cleanHeader,10 cleanData,11 get_type212)13class ETL:14 host = ""15 user = ""16 pw = ""17 db = ""18 charset = ""19 logging = ""20 connection = ""21 cursor = ""22 jsonSchema = ""23 def __init__(self, host, user, pw, db, logging, schemaFile, charset='utf8'):24 self.host = host25 self.user = user26 self.pw = pw27 self.db = db28 self.charset = charset29 self.logging = logging30 ## Initialize MySQL Cursor31 self.connection = pymysql.connect(host=self.host, user=user, password=pw, charset=self.charset)32 self.cursor = self.connection.cursor(pymysql.cursors.DictCursor)33 34 self.connectDatabase()35 DDLFields = list()36 37 self.jsonSchema = json.load(open(schemaFile))38 39 ##initialize Logging40 self.logging = logging41 def connectDatabase(self):42 """43 ' connectDatabase: Checks if the database exists and if not it attempts to create it. 44 """45 self.cursor.execute("SHOW DATABASES")46 dbExists = False47 for value in self.cursor:48 if value['Database'] == self.db:49 dbExists = True50 self.logging.info("etlLibrary: connectDatabase: Database "+self.db+" already exists. Connecting to database.")51 if dbExists == False:52 self.cursor.execute("CREATE DATABASE "+self.db)53 self.logging.info("etlLibrary: connectDatabase: Database "+self.db+" doesn't yet exist. Creating Database.")54 self.connection = pymysql.connect(host=self.host, user=self.user, password=self.pw, db=self.db, charset=self.charset)55 self.cursor = self.connection.cursor(pymysql.cursors.DictCursor)56 def createSchema(self):57 '''58 '' createSchema - This function will retrieve a schema predefined in a json mapping59 '' and build the tables if it doesn't exist. 60 ''' 61 for key, value in self.jsonSchema.items() :62 existingMYSQLTable = self.cursor.execute('SELECT TABLE_NAME FROM information_schema.TABLES WHERE TABLE_SCHEMA = "'+self.db+'" and TABLE_NAME = "'+key+'";')63 DDLFields = []64 if type(value) == type(dict()):65 for field, datatype in value.items():66 if field == "UNIQUE INDEX":67 DDLFieldDef = field + " " +datatype+"_unique (" + datatype + " ASC), "68 DDLFields.append(DDLFieldDef)69 else:70 DDLFieldDef = field + " " + datatype + ", "71 DDLFields.append(DDLFieldDef)72 columnExists = self.cursor.execute('SELECT COLUMN_NAME FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = "'+self.db+'" and TABLE_NAME = "'+key+'" and COLUMN_NAME = "'+field+'";')73 if existingMYSQLTable > 0 and columnExists == 0:74 alterTable = "ALTER TABLE "+self.db+"."+key+" ADD COLUMN "+field+" " +datatype+";"75 self.logging.info("now executing: "+ alterTable)76 self.cursor.execute(alterTable)77 if existingMYSQLTable == 0:78 mySQLDDL = """CREATE TABLE """+key+ """ 79 (id INTEGER NOT NULL AUTO_INCREMENT, """+"""""".join(DDLFields)+""" 80 date_created TIMESTAMP NOT NULL ,81 date_updated TIMESTAMP NOT NULL, 82 PRIMARY KEY (id));"""83 self.logging.debug(mySQLDDL)84 self.cursor.execute(mySQLDDL)85 del DDLFields[:]86 def dataDump(self, table='dataDump'):87 '''88 '' dataDump - this method will create a table to store your json.89 '''90 existingMYSQLTable = self.cursor.execute('SELECT TABLE_NAME FROM information_schema.TABLES WHERE TABLE_SCHEMA = "'+self.db+'" and TABLE_NAME = "'+table+'";')91 if existingMYSQLTable == 0:92 ddl = """CREATE TABLE """+self.db+"""."""+table+""" (id INTEGER NOT NULL AUTO_INCREMENT,url varchar(150) NULL,record_id INT NULL,location varchar(100) NULL,source varchar(100) NULL, json text NULL, date_created TIMESTAMP NOT NULL, date_updated TIMESTAMP NOT NULL, parsed tinyint(1) default 0, PRIMARY KEY (id), UNIQUE INDEX url_UNIQUE (url ASC));"""93 self.logging.debug(ddl)94 self.cursor.execute(ddl)95 def dumpData(self, url, record_id, source, location, json):96 '''97 '' dumpData - This function will dump the json objects into a table98 '''99 table = 'dataDump'100 existingMYSQLTable = self.cursor.execute('SELECT TABLE_NAME FROM information_schema.TABLES WHERE TABLE_SCHEMA = "'+self.db+'" and TABLE_NAME = "'+table+'";')101 if existingMYSQLTable == 0:102 ddl = """CREATE TABLE """+self.db+"""."""+table+""" (103 `id` INTEGER NOT NULL AUTO_INCREMENT,104 `url` varchar(150) NULL,105 `record_id` INT NULL,106 `location` varchar(100) NULL,107 `source` varchar(100) NULL,108 `json` json NULL,109 `date_created` TIMESTAMP NOT NULL,110 `date_updated` TIMESTAMP NOT NULL,111 parsed tinyint(1) default 0,112 PRIMARY KEY (`id`),113 UNIQUE INDEX `url_UNIQUE` (`url` ASC));114 """115 self.logging.info('ETL:dumpData: Running the following DDL'+ddl)116 self.cursor.execute(ddl)117 118 theInsert = 'replace into '+self.db+'.'+table+' (url, record_id, source, location, json, parsed) values( "'+url+'", '+str(record_id)+', "'+source+'", "'+location+'", '+json+', 1 );'119 self.logging.info('ETL:dataDump: Running the following replace: '+theInsert)120 self.cursor.execute(theInsert)121 self.connection.commit()122 def analyzeArray(self, arrayName, location):123 '''124 '' analyzeArray - This function will grab all items parsed from your HTML, add them to an array_properties table for comparison.125 '' I use this to add similar properites to the same field as several towns use different names for things like bathrooms.126 '''127 self.logging.debug(arrayName)128 existingMYSQLTable = self.cursor.execute('SELECT TABLE_NAME FROM information_schema.TABLES WHERE TABLE_SCHEMA = "'+self.db+'" and TABLE_NAME = "array_properties";')129 if existingMYSQLTable == 0:130 mySQLDDL = """CREATE TABLE array_properties131 (id INTEGER NOT NULL AUTO_INCREMENT, 132 location varchar(125), 133 array_name varchar(100), 134 key_name varchar(125), 135 data_type varchar(100), 136 PRIMARY KEY (id),137 unique index key_unique (key_name asc));"""138 self.logging.info(mySQLDDL)139 self.cursor.execute(mySQLDDL)140 for key, value in arrayName.items():141 if key is not None:142 doesRowExistSQL = 'select id from array_properties where key_name = "'+key+'";'143 doesRowExist = self.cursor.execute(doesRowExistSQL)144 if doesRowExist == 0:145 insertStatement = """replace into array_properties (key_name, location, data_type) values ( """"+key+"""", """"+location+"""", """"+get_type2(value)+"""" ) ;"""146 self.cursor.execute(insertStatement)147 self.connection.commit()148 self.logging.debug(insertStatement)149 def writeTable(self, tableName, object):150 '''151 '' writeTable: this function takes a table and an object and then writes the data into the table in question.152 '''153 table = tableName154 DDLFields = list()155 insertKeys = list() #Used to hold the field names you will inser into156 insertValues = list() #Used to hold the actual values you are inserting157 tableStructure = list()158 existingMYSQLTable = self.cursor.execute('SELECT TABLE_NAME FROM information_schema.TABLES WHERE TABLE_SCHEMA = "'+self.db+'" and TABLE_NAME = "'+tableName+'";')159 whereParentDataIs = list()160 for key, value in object.items():161 if key != "" and key[0].isdigit() is False : #Checkes if the key is a digit, if it is then we don't do anything.162 if value is None:163 datatype = "TEXT"164 else:165 datatype = get_type2(value)166 DDLFieldDef = key + ' ' + datatype + ', '167 DDLFields.append(DDLFieldDef)168 insertKeys.append(key + ', ')169 columnExists = self.cursor.execute('SELECT COLUMN_NAME FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = "'+self.db+'" and TABLE_NAME = "'+tableName+'" and COLUMN_NAME = "'+key+'";')170 if existingMYSQLTable > 0 and columnExists == 0:171 alterTable = 'ALTER TABLE '+self.db+'.'+tableName+' ADD COLUMN '+key+' ' +datatype+';'172 self.logging.info('ETL: writeTable: Now executing: '+ alterTable)173 self.cursor.execute(alterTable)174 if value == "" or value is None:175 whereParentDataIs.append(key + """ is NULL and """)176 insertValues.append( """NULL , """) 177 elif value != "" and self.jsonSchema[tableName][key].upper().split('(')[0] in ['VARCHAR', 'TEXT']:178 whereParentDataIs.append( key + """ = """ + '"' + value.replace('"',"'") + '"' + """ and """)179 insertValues.append('"' + value.replace('"',"'") + '", ')180 elif value !="" and self.jsonSchema[tableName][key].upper() == 'DATETIME':181 value = datetime.datetime.strptime(value, '%m/%d/%Y').strftime('%Y-%m-%d')182 whereParentDataIs.append( key + """ = """ + '"' + value + '"' + """ and """)183 insertValues.append('"' + value + '", ')184 else:185 whereParentDataIs.append(key + """ = """ + str(value) + """ and """)186 insertValues.append( str(value)+', ') 187 if existingMYSQLTable == 0:188 mySQLDDL = """CREATE TABLE """+tableName+ """ 189 (id INTEGER NOT NULL AUTO_INCREMENT,"""+"""""".join(DDLFields)+""" PRIMARY KEY (id));"""190 self.logging.info(mySQLDDL)191 self.cursor.execute(mySQLDDL)192 doesRowExistSQL = """select * from """+ tableName + """ where """+"""""".join(whereParentDataIs)[:-4]+""";"""193 self.logging.info('ETL: writeTable: Running the following SQL: '+doesRowExistSQL) 194 doesRowExist = self.cursor.execute(doesRowExistSQL)195 if doesRowExist == 0:196 insertStatement = """replace into """+tableName+""" ("""+ """""".join(insertKeys)[:-2] +""",date_created,date_updated) values ("""+ """""".join(insertValues)[:-2] + """,now(),now()) ;"""197 self.logging.info(insertStatement)198 self.cursor.execute(insertStatement)199 self.connection.commit()200 else:201 self.logging.info('already exists')202 self.cursor.execute(doesRowExistSQL)203 addresses = self.cursor.fetchone()204 for key, value in addresses.items():205 if type(value) == type(datetime.datetime.now()):206 addresses.update({key : value.strftime('%Y-%m-%d') })207 return addresses208 def parseTable(self, dataArray, tableName, primaryTable=''):209 '''210 '' ParseTable - This method is used to parse 211 '' our tables and add the respective relationships.212 '' It's currently custom to my script and will need 213 '' to refactor it.214 '''215 foreignKey = primaryTable + '_id' #Used to link relationship to all tables. 216 DDLFields = list()217 insertKeys = list() #Used to hold the field names you will inser into218 insertValues = list() #Used to hold the actual values you are inserting219 subTableFields = list()220 table = dataArray[tableName]221 subTable = None222 tableStructure = list()223 whereParentDataIs = list()224 #Loop through the main tables.225 if type(table) != type(list()):226 for key, value in table.items():227 value = cleanData(value, self.jsonSchema[tableName][key])228 if type(value) != type(dict()) and type(value) != type(list()):229 insertKeys.append(key + ', ')230 if key in self.jsonSchema[tableName]: #Checkes if the key is a digit, if it is then we don't do anything.231 if value == "" or value is None:232 whereParentDataIs.append(key + """ is NULL and """)233 insertValues.append( """NULL , """) 234 elif value != "" and self.jsonSchema[tableName][key].upper().split('(')[0] in ['VARCHAR', 'TEXT']:235 whereParentDataIs.append( key + """ = """ + value + """ and """)236 insertValues.append( value + """, """)237 elif value != "" and self.jsonSchema[tableName][key] == "DATETIME":238 self.logging.debug( value)239 value = datetime.datetime.strptime(value, """%m/%d/%Y""").strftime("""%Y-%m-%d""")240 whereParentDataIs.append( key + """ = """ + value + """ and """)241 insertValues.append( value + """, """)242 else:243 whereParentDataIs.append(key + """ = """ + str(value) + """ and """)244 insertValues.append( str(value) + """, """) 245 else:246 subTable = key247 DDLFieldDef = key + '_id INTEGER, '248 DDLFields.append(DDLFieldDef)249 insertSubValues = list()250 insertSubKeys = list()251 whereDataIs = list()252 existingSubTable = self.cursor.execute('SELECT TABLE_NAME FROM information_schema.TABLES WHERE TABLE_SCHEMA = "'+self.db+'" and TABLE_NAME = "'+subTable+'";')253 254 for subKey, subValue in value.items():255 if subKey[0].isdigit() is False:256 subDataType = get_type2(subValue)257 subTableFieldDef = subKey + ' ' + subDataType + ', '258 subTableFields.append( subTableFieldDef)259 insertSubKeys.append(subKey + ', ')260 if subValue == "":261 whereDataIs.append(subKey + ' is NULL and ')262 elif subValue != "" and self.jsonSchema[subTable][subKey].upper().split('(')[0] in ['VARCHAR', 'TEXT']:263 whereDataIs.append(subKey + ' = ' + '"' + subValue.replace('"', '') + '"' + ' and ')264 insertSubValues.append('"' + subValue.replace('"', '') + '"' + ', ')265 elif subValue != "" and self.jsonSchema[subTable][subKey] == "DATETIME":266 self.logging.debug( str(subValue+ datetime.datetime.strptime(subValue, '%m/%d/%Y').strftime('%Y-%m-%d')))267 subValue = datetime.datetime.strptime(subValue, '%m/%d/%Y').strftime('%Y-%m-%d')268 whereDataIs.append(subKey + ' = ' + '"' + subValue.replace('"', '') + '"' + ' and ')269 insertSubValues.append('"' + subValue.replace('"', '') + '"' + ', ')270 else:271 insertSubValues.append( str(subValue) + ', ') 272 whereDataIs.append(subKey + '=' + '"' + str(subValue) + 'and ')273 columnExists = self.cursor.execute('SELECT COLUMN_NAME FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = "'+self.db+'" and TABLE_NAME = "'+subTable+'" and COLUMN_NAME = "'+subKey+'";')274 if existingSubTable > 0 and columnExists == 0:275 alterTable = 'ALTER TABLE '+self.db+'.'+subTable+' ADD COLUMN '+subKey+' ' +subDataType+';'276 self.logging.debug('now executing: '+ alterTable)277 self.cursor.execute(alterTable)278 if existingSubTable == 0:279 subTableDDL = """CREATE TABLE """+subTable+ """ 280 (id INTEGER NOT NULL AUTO_INCREMENT,"""+"""""".join(subTableFields)+""" 281 date_created TIMESTAMP NOT NULL,282 date_updated TIMESTAMP NOT NULL, 283 PRIMARY KEY (id));"""284 self.logging.debug(subTableDDL)285 self.cursor.execute(subTableDDL)286 insertSubStatement = """replace into """+subTable+""" ("""+ """""".join(insertSubKeys)[:-2] +""") values ("""+ """""".join(insertSubValues).replace('""','NULL')[:-2] + """) ;"""287 self.logging.debug(insertSubStatement)288 self.cursor.execute(insertSubStatement)289 self.connection.commit()290 291 insertKeys.append(subTable+'_id, ')292 self.logging.debug(doesRowExistSQL) 293 self.cursor.execute(doesRowExistSQL)294 for key, value in self.cursor.fetchone().items():295 insertValues.append(str(value)+', ')296 297 298 del subTableFields[:], insertSubValues[:]299 doesRowExistSQL = """select id from """+ tableName + """ where """+"""""".join(whereParentDataIs)[:-4]+""";"""300 self.logging.debug(doesRowExistSQL) 301 302 doesRowExist = self.cursor.execute(doesRowExistSQL)303 insertStatement = """replace into """+tableName+""" ("""+ """""".join(insertKeys)[:-2] +""") values ("""+ """""".join(insertValues).replace('""','NULL')[:-2] + """) ;"""304 self.logging.debug(insertStatement)305 if tableName == primaryTable:306 if doesRowExist == 0:307 self.cursor.execute(insertStatement)308 self.connection.commit()309 self.logging.debug(insertStatement)310 self.logging.debug(doesRowExistSQL)311 self.cursor.execute(doesRowExistSQL)312 fkey = self.cursor.fetchone()313 for key, value in fkey.items():314 global fKeyValue, fKeyName315 fKeyName = primaryTable+'_id, '316 fKeyValue = str(value)+', '317 else:318 insertKeys.append(fKeyName)319 insertValues.append(fKeyValue)320 insertStatement = """replace into """+tableName+""" ("""+ """""".join(insertKeys)[:-2] +""") values ("""+ """""".join(insertValues).replace('""','NULL')[:-2] + """) ;"""321 self.logging.debug(insertStatement)322 self.cursor.execute(insertStatement)323 self.connection.commit()324 325 326 self.logging.debug(tableName, DDLFields, subTable, subTableFields)327 del DDLFields[:], insertValues328 return tableStructure329 else:330 subTable = tableName331 insertSubValues = list()332 insertSubKeys = list()333 whereDataIs = list()334 for items in table:335 for subKey, subValue in items.items():336 if subKey in self.jsonSchema[tableName]: #Checkes if the key is a digit, if it is then we don't do anything.337 subDataType = get_type2(subValue)338 subTableFieldDef = subKey + ' ' + subDataType + ', '339 subTableFields.append( subTableFieldDef)340 insertSubKeys.append(subKey + ', ')341 if subValue == "" or subValue is None:342 whereDataIs.append(subKey + ' is NULL and ')343 insertSubValues.append(' NULL, ')344 elif subValue != "" and self.jsonSchema[subTable][subKey].upper().split('(')[0] in ['VARCHAR', 'TEXT']:345 whereDataIs.append(subKey + ' = ' + '"' + subValue.replace('"', '') + '"' + ' and ')346 insertSubValues.append('"' + subValue.replace('"', '') + '"' + ', ')347 elif subValue != "" and self.jsonSchema[subTable][subKey].upper() == "DATETIME":348 subValue = datetime.datetime.strptime(subValue, '%m/%d/%Y').strftime('%Y-%m-%d')349 whereDataIs.append(subKey + ' = ' + '"' + subValue.replace('"', '') + '"' + ' and ')350 insertSubValues.append('"' + subValue.replace('"', '') + '"' + ', ')351 else:352 insertSubValues.append( str(subValue) + ', ') 353 whereDataIs.append(subKey + ' = ' + str(subValue) + ' and ')354 columnExists = self.cursor.execute('SELECT COLUMN_NAME FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = "'+self.db+'" and TABLE_NAME = "'+subTable+'" and COLUMN_NAME = "'+subKey+'";')355 if columnExists == 0:356 alterTable = 'ALTER TABLE '+self.db+'.'+subTable+' ADD COLUMN '+subKey+' ' +subDataType+';'357 self.logging.debug('now executing: '+ alterTable)358 self.cursor.execute(alterTable)359 doesRowExistSQL = """select id from """+ subTable + """ where """+"""""".join(whereDataIs)[:-4]+""";"""360 self.logging.debug(doesRowExistSQL) 361 doesRowExist = self.cursor.execute(doesRowExistSQL)362 if doesRowExist == 0:363 insertSubKeys.append(fKeyName)364 insertSubValues.append(fKeyValue)365 insertSubStatement = """replace into """+subTable+""" ("""+ """""".join(insertSubKeys)[:-2] +""") values ("""+ """""".join(insertSubValues).replace('""','NULL')[:-2] + """) ;"""366 self.logging.debug(insertSubStatement)367 self.cursor.execute(insertSubStatement)368 self.connection.commit()369 insertKeys.append(subTable+'_id, ')370 self.logging.debug(doesRowExistSQL) 371 self.cursor.execute(doesRowExistSQL)372 if self.cursor.execute(doesRowExistSQL) > 0:373 for key, value in self.cursor.fetchone().items():374 insertValues.append(str(value)+', ')375 ...
Alchemy.py
Source:Alchemy.py
1try:2 # For Python 3.0 and later3 from urllib.request import urlopen4except ImportError:5 # Fall back to Python 2's urllib26 from urllib2 import urlopen7 8from bs4 import BeautifulSoup9import json10import MySQLdb11import sys 12reload(sys) 13sys.setdefaultencoding('utf8')14db = MySQLdb.connect("localhost","root","1105026","ThesisDatabase")15cursor = db.cursor()16keyWordSeparator = "+-+-"17fileNamingStartIndex = 118def deleteTable():19 stmt = "SHOW TABLES LIKE 'NewsTable'"20 cursor.execute(stmt)21 result = cursor.fetchone()22 if result:23 # there is a table named "NewsTable"24 print("Table is present already")25 cursor.execute("DROP TABLE IF EXISTS NewsTable ")26 27def createNewTable():28 sql = """\29 CREATE TABLE NewsTable (\30 row_id INT NOT NULL AUTO_INCREMENT,\31 news_id VARCHAR(50) NOT NULL,\32 newsURL VARCHAR(200) NOT NULL,\33 titleFileName VARCHAR(50) NOT NULL,\34 keywordsFileName VARCHAR(50) NOT NULL,\35 textFileName VARCHAR(50) NOT NULL,\36 PRIMARY KEY ( row_id )\37 )"""38 cursor.execute(sql)39def doesRowExist(newsId):40 cursor.execute("SELECT COUNT(*) from NewsTable where news_id ='%s'"% (newsId))41 result=cursor.fetchone()42 if(result[0]==1):43 return True44 return False45 46def insertInDatabase(newsId,url,title,keyWordString,entireText,fileIndex):47 48 49 50 textFileName = "textFiles/text"+ str(fileIndex)51 keyWordFileName = "keywordFiles/keys"+str(fileIndex)52 titleFileName = "titleFiles/keys"+str(fileIndex)53 54 textFile = open(textFileName, "wb")55 keyWordFile= open(keyWordFileName, "wb")56 titleFile= open(titleFileName, "wb")57 58 entireText = entireText.encode('ascii', 'ignore')59 keyWordString= keyWordString.encode('ascii', 'ignore')60 title= title.encode('ascii', 'ignore')61 62 textFile.write(entireText)63 keyWordFile.write(keyWordString)64 titleFile.write(title)65 66 titleFile.close()67 textFile.close()68 keyWordFile.close()69 70 if(doesRowExist(newsId)):71 print("this row has been inserted before")72 return73 74 # Prepare SQL query to INSERT a record into the database.75 sql = "INSERT INTO NewsTable(news_id,newsURL,\76 titleFileName, keywordsFileName,textFileName)\77 VALUES('%s','%s', '%s', '%s', '%s')" % \78 (newsId,url,titleFileName,keyWordFileName,textFileName)79 cursor.execute(sql) 80 81 82 try:83 # Execute the SQL command84 cursor.execute(sql)85 # Commit your changes in the database86 db.commit()87 print("Row inserted "+newsId)88 except:89 # Rollback in case there is any error90 print("Row not inserted")91 db.rollback()92 93 94 95 96 97def writeSampleHTML(entireText,fileNamingStartIndex):98 htmlFileName = "htmlFiles/html"+ str(fileNamingStartIndex)99 100 htmlFile = open(htmlFileName, "wb")101 entireText = entireText.encode('ascii', 'ignore')102 htmlFile.write(entireText)103 htmlFile.close()104#removes all the tags of the news through html and just returns the main content and more things105def parseHTMLofTheNews(url):106 107 108 try:109 html =urlopen(url).read() 110 writeSampleHTML(html)111 112 soup = BeautifulSoup(html)113 # kill all script and style elements114 for script in soup(["script", "style"]):115 script.extract() # rip it out116 # get text117 text = soup.body.get_text()118 text = text.encode('ascii', 'ignore')119 # break into lines and remove leading and trailing space on each120 lines = (line.strip() for line in text.splitlines())121 # break multi-headlines into a line each122 chunks = (phrase.strip() for line in lines for phrase in line.split(" "))123 # drop blank lines124 text = '\n'.join(chunk for chunk in chunks if chunk)125 except:126 return "-1"127 128 129 return text 130 131 132 133def readDatabase():134 135 # Prepare SQL query to INSERT a record into the database.136 sql = "SELECT * FROM NewsTable"137 try:138 # Execute the SQL command139 cursor.execute(sql)140 print(cursor.rowcount)141 '''142 143 # Fetch all the rows in a list of lists.144 results = cursor.fetchall()145 for row in results:146 fname = row[0]147 lname = row[1]148 age = row[2]149 sex = row[3]150 income = row[4]151 income1 = row[4]152 # Now print fetched result153 print "fname=%d,lname=%s,age=%s,sex=%s,income=%s,income1=%s" % \154 (fname, lname, age, sex, income,income1 )155 '''156 157 except:158 print "Error: unable to fetch data"159def main():160 161 #json =urlopen("https://gateway-a.watsonplatform.net/calls/data/GetNews? apikey="your won key"&outputMode=json&start=now-1d&end=now&maxResults=15&return=enriched.url.text,enriched.url")162 returnedJson =urlopen("https://gateway-a.watsonplatform.net/calls/data/GetNews?apikey=c4c9edc33ff0c219f9eba3dea28eb8f557cbe4ef&outputMode=json&start=now-10d&end=now&maxResults=15&return=enriched.url.url,enriched.url.keywords,enriched.url.title").read()163 164 print(returnedJson)165 parsed_json = json.loads(returnedJson)166 #fo = open("json.txt", "wb")167 168 #fo.write(returnedJson)169 N = len(parsed_json["result"]["docs"])170 for i in range(N):171 172 newsId = parsed_json["result"]["docs"][i]["id"]173 url = parsed_json["result"]["docs"][i]["source"]["enriched"]["url"]["url"]174 title = parsed_json["result"]["docs"][i]["source"]["enriched"]["url"]["title"]175 allKeyWords = parsed_json["result"]["docs"][i]["source"]["enriched"]["url"]["keywords"]176 177 keyWordString=""178 for index in range(len(allKeyWords)):179 keyWord = allKeyWords[index]180 keyWordString+=keyWord["text"]181 if(index != (len(allKeyWords)-1)):182 keyWordString+=keyWordSeparator183 #print keyWord["text"]184 #print url185 #print title186 #print keyWordString187 entireText = parseHTMLofTheNews(url)188 189 190 191 if(entireText == "-1"):192 continue193 #print entireText194 fileIndex = 0195 print "Inserting news with id: "+newsId196 insertInDatabase(newsId,url,title,keyWordString,entireText,fileNamingStartIndex + i)197 198 199 200 201if __name__ == "__main__":202 203 #deleteTable()204 #createNewTable()205 main() 206 #readDatabase() ...
rename-mail-folder.py
Source:rename-mail-folder.py
1#!/usr/bin/python2#3# Linux Desktop Testing Project http://ldtp.freedesktop.org4#5# Author:6# Khasim Shaheed <khasim.shaheed@gmail.com>7#8# Copyright 2004 Novell, Inc.9#10# This test script is free software; you can redistribute it and/or11# modify it under the terms of the GNU Library General Public12# License as published by the Free Software Foundation; either13# version 2 of the License, or (at your option) any later version.14#15# This library is distributed in the hope that it will be useful,16# but WITHOUT ANY WARRANTY; without even the implied warranty of17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU18# Library General Public License for more details.19#20# You should have received a copy of the GNU Library General Public21# License along with this library; if not, write to the22# Free Software Foundation, Inc., 59 Temple Place - Suite 330,23# Boston, MA 02111-1307, USA.24#25# Rename a mail folder26import time27from ldtp import *28from ldtputils import *29# Section to rename a mail folder30def rename_mail_folder (prev_label, new_label):31 try:32 selectrow ('frmEvolution-*', 'ttblMailFolderTree', prev_label)33 selectmenuitem ('frmEvolution-*', 'mnuFolder;mnuRename*')34 if waittillguiexist ('dlgRenameFolder') == 0:35 log ('Rename Folder dialog not opened', 'error')36 raise LdtpExecutionError (0)37 settextvalue ('dlgRenameFolder', 'txt0', new_label)38 click ('dlgRenameFolder', 'btnOK')39 if waittillguinotexist ('dlgRenameFolder') == 0:40 log ('Rename Folder dialog not closed', 'error')41 raise LdtpExecutionError (0)42 time.sleep (2)43 if guiexist ('dlgEvolutionError'):44 click ('dlgEvolutionError', 'btnOK')45 time.sleep (1)46 click ('dlgRenameFolder', 'btnCancel')47 48 # TODO: The reason for the error should be logged and 49 # Pass/Fail condition has to be changed accordingly50 log ('Renaming folder failed', 'cause')51 log ('Renaming folder failed', 'fail')52 else:53 if doesrowexist ('frmEvolution-*', 'ttblMailFolderTree', prev_label):54 log ('Renaming a folder failed, renaming not done properly', 'cause')55 log ('Renaming a folder failed, renaming not done properly', 'fail')56 elif doesrowexist ('frmEvolution-*', 'ttblMailFolderTree', new_label):57 log ('Renaming a folder passed successfully', 'pass')58 else:59 log ('Renaming a folder failed', 'cause')60 log ('Renaming a folder failed', 'fail')61 except ldtp.error, msg:62 log ('Renaming a mail folder failed, ' + str (msg), 'cause')63 log ('Renaming a mail folder failed', 'fail')64 if guiexist ('dlgRenameFolder'):65 click ('dlgRenameFolder', 'btnCancel')66 raise LdtpExecutionError (0) 67# Read input from file68data_object = LdtpDataFileParser (datafilename)69prev_label = data_object.gettagvalue ('prev_label')70new_label = data_object.gettagvalue ('new_label')71# Call the function72if prev_label and new_label:73 rename_mail_folder (prev_label[0], new_label[0])74else:75 if not (prev_label):76 log ('prev_label not provided in data xml file', 'error')77 if not (new_label):78 log ('new_label not provided in data xml file', 'error')...
Automation Testing Tutorials
Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.
LambdaTest Learning Hubs:
- JUnit Tutorial
- TestNG Tutorial
- Webdriver Tutorial
- WebDriverIO Tutorial
- Protractor Tutorial
- Selenium 4 Tutorial
- Jenkins Tutorial
- NUnit Tutorial
- Jest Tutorial
- Playwright Tutorial
- Cypress Tutorial
- PyTest Tutorial
YouTube
You could also refer to video tutorials over LambdaTest YouTube channel to get step by step demonstration from industry experts.
Run pyatom automation tests on LambdaTest cloud grid
Perform automation testing on 3000+ real desktop and mobile devices online.
Try LambdaTest Now !!
Get 100 minutes of automation test minutes FREE!!
Was this article helpful?
