How to use links method in Kiwi

Best Python code snippet using Kiwi_python

test_NewTabUtils.js

Source:test_NewTabUtils.js Github

copy

Full Screen

...29 );30 // isTopSiteGivenProvider() and getProviderLinks() should still return results31 // even when cache is empty or being populated.32 Assert.ok(!NewTabUtils.isTopSiteGivenProvider("example1.com", provider));33 do_check_links(NewTabUtils.getProviderLinks(provider), []);34 await promise;35 // Once the cache is populated, we get the expected results36 Assert.ok(NewTabUtils.isTopSiteGivenProvider("example1.com", provider));37 do_check_links(NewTabUtils.getProviderLinks(provider), expectedLinks);38 NewTabUtils.links.removeProvider(provider);39});40add_task(async function notifyLinkDelete() {41 let expectedLinks = makeLinks(0, 3, 1);42 let provider = new TestProvider(done => done(expectedLinks));43 provider.maxNumLinks = expectedLinks.length;44 NewTabUtils.initWithoutProviders();45 NewTabUtils.links.addProvider(provider);46 await new Promise(resolve => NewTabUtils.links.populateCache(resolve));47 do_check_links(NewTabUtils.links.getLinks(), expectedLinks);48 // Remove a link.49 let removedLink = expectedLinks[2];50 provider.notifyLinkChanged(removedLink, 2, true);51 let links = NewTabUtils.links._providers.get(provider);52 // Check that sortedLinks is correctly updated.53 do_check_links(NewTabUtils.links.getLinks(), expectedLinks.slice(0, 2));54 // Check that linkMap is accurately updated.55 Assert.equal(links.linkMap.size, 2);56 Assert.ok(links.linkMap.get(expectedLinks[0].url));57 Assert.ok(links.linkMap.get(expectedLinks[1].url));58 Assert.ok(!links.linkMap.get(removedLink.url));59 // Check that siteMap is correctly updated.60 Assert.equal(links.siteMap.size, 2);61 Assert.ok(links.siteMap.has(NewTabUtils.extractSite(expectedLinks[0].url)));62 Assert.ok(links.siteMap.has(NewTabUtils.extractSite(expectedLinks[1].url)));63 Assert.ok(!links.siteMap.has(NewTabUtils.extractSite(removedLink.url)));64 NewTabUtils.links.removeProvider(provider);65});66add_task(async function populatePromise() {67 let count = 0;68 let expectedLinks = makeLinks(0, 10, 2);69 let getLinksFcn = async function(callback) {70 // Should not be calling getLinksFcn twice71 count++;72 Assert.equal(count, 1);73 await Promise.resolve();74 callback(expectedLinks);75 };76 let provider = new TestProvider(getLinksFcn);77 NewTabUtils.initWithoutProviders();78 NewTabUtils.links.addProvider(provider);79 NewTabUtils.links.populateProviderCache(provider, () => {});80 NewTabUtils.links.populateProviderCache(provider, () => {81 do_check_links(NewTabUtils.links.getLinks(), expectedLinks);82 NewTabUtils.links.removeProvider(provider);83 });84});85add_task(async function isTopSiteGivenProvider() {86 let expectedLinks = makeLinks(0, 10, 2);87 // The lowest 2 frecencies have the same base domain.88 expectedLinks[expectedLinks.length - 2].url =89 expectedLinks[expectedLinks.length - 1].url + "Test";90 let provider = new TestProvider(done => done(expectedLinks));91 provider.maxNumLinks = expectedLinks.length;92 NewTabUtils.initWithoutProviders();93 NewTabUtils.links.addProvider(provider);94 await new Promise(resolve => NewTabUtils.links.populateCache(resolve));95 Assert.equal(96 NewTabUtils.isTopSiteGivenProvider("example2.com", provider),97 true98 );99 Assert.equal(100 NewTabUtils.isTopSiteGivenProvider("example1.com", provider),101 false102 );103 // Push out frecency 2 because the maxNumLinks is reached when adding frecency 3104 let newLink = makeLink(3);105 provider.notifyLinkChanged(newLink);106 // There is still a frecent url with example2 domain, so it's still frecent.107 Assert.equal(108 NewTabUtils.isTopSiteGivenProvider("example3.com", provider),109 true110 );111 Assert.equal(112 NewTabUtils.isTopSiteGivenProvider("example2.com", provider),113 true114 );115 // Push out frecency 3116 newLink = makeLink(5);117 provider.notifyLinkChanged(newLink);118 // Push out frecency 4119 newLink = makeLink(9);120 provider.notifyLinkChanged(newLink);121 // Our count reached 0 for the example2.com domain so it's no longer a frecent site.122 Assert.equal(123 NewTabUtils.isTopSiteGivenProvider("example5.com", provider),124 true125 );126 Assert.equal(127 NewTabUtils.isTopSiteGivenProvider("example2.com", provider),128 false129 );130 NewTabUtils.links.removeProvider(provider);131});132add_task(async function multipleProviders() {133 // Make each provider generate NewTabUtils.links.maxNumLinks links to check134 // that no more than maxNumLinks are actually returned in the merged list.135 let evenLinks = makeLinks(0, 2 * NewTabUtils.links.maxNumLinks, 2);136 let evenProvider = new TestProvider(done => done(evenLinks));137 let oddLinks = makeLinks(0, 2 * NewTabUtils.links.maxNumLinks - 1, 2);138 let oddProvider = new TestProvider(done => done(oddLinks));139 NewTabUtils.initWithoutProviders();140 NewTabUtils.links.addProvider(evenProvider);141 NewTabUtils.links.addProvider(oddProvider);142 await new Promise(resolve => NewTabUtils.links.populateCache(resolve));143 let links = NewTabUtils.links.getLinks();144 let expectedLinks = makeLinks(145 NewTabUtils.links.maxNumLinks,146 2 * NewTabUtils.links.maxNumLinks,147 1148 );149 Assert.equal(links.length, NewTabUtils.links.maxNumLinks);150 do_check_links(links, expectedLinks);151 NewTabUtils.links.removeProvider(evenProvider);152 NewTabUtils.links.removeProvider(oddProvider);153});154add_task(async function changeLinks() {155 let expectedLinks = makeLinks(0, 20, 2);156 let provider = new TestProvider(done => done(expectedLinks));157 NewTabUtils.initWithoutProviders();158 NewTabUtils.links.addProvider(provider);159 await new Promise(resolve => NewTabUtils.links.populateCache(resolve));160 do_check_links(NewTabUtils.links.getLinks(), expectedLinks);161 // Notify of a new link.162 let newLink = makeLink(19);163 expectedLinks.splice(1, 0, newLink);164 provider.notifyLinkChanged(newLink);165 do_check_links(NewTabUtils.links.getLinks(), expectedLinks);166 // Notify of a link that's changed sort criteria.167 newLink.frecency = 17;168 expectedLinks.splice(1, 1);169 expectedLinks.splice(2, 0, newLink);170 provider.notifyLinkChanged({171 url: newLink.url,172 frecency: 17,173 });174 do_check_links(NewTabUtils.links.getLinks(), expectedLinks);175 // Notify of a link that's changed title.176 newLink.title = "My frecency is now 17";177 provider.notifyLinkChanged({178 url: newLink.url,179 title: newLink.title,180 });181 do_check_links(NewTabUtils.links.getLinks(), expectedLinks);182 // Notify of a new link again, but this time make it overflow maxNumLinks.183 provider.maxNumLinks = expectedLinks.length;184 newLink = makeLink(21);185 expectedLinks.unshift(newLink);186 expectedLinks.pop();187 Assert.equal(expectedLinks.length, provider.maxNumLinks); // Sanity check.188 provider.notifyLinkChanged(newLink);189 do_check_links(NewTabUtils.links.getLinks(), expectedLinks);190 // Notify of many links changed.191 expectedLinks = makeLinks(0, 3, 1);192 provider.notifyManyLinksChanged();193 // Since _populateProviderCache() is async, we must wait until the provider's194 // populate promise has been resolved.195 await NewTabUtils.links._providers.get(provider).populatePromise;196 // NewTabUtils.links will now repopulate its cache197 do_check_links(NewTabUtils.links.getLinks(), expectedLinks);198 NewTabUtils.links.removeProvider(provider);199});200add_task(async function oneProviderAlreadyCached() {201 let links1 = makeLinks(0, 10, 1);202 let provider1 = new TestProvider(done => done(links1));203 NewTabUtils.initWithoutProviders();204 NewTabUtils.links.addProvider(provider1);205 await new Promise(resolve => NewTabUtils.links.populateCache(resolve));206 do_check_links(NewTabUtils.links.getLinks(), links1);207 let links2 = makeLinks(10, 20, 1);208 let provider2 = new TestProvider(done => done(links2));209 NewTabUtils.links.addProvider(provider2);210 await new Promise(resolve => NewTabUtils.links.populateCache(resolve));211 do_check_links(NewTabUtils.links.getLinks(), links2.concat(links1));212 NewTabUtils.links.removeProvider(provider1);213 NewTabUtils.links.removeProvider(provider2);214});215add_task(async function newLowRankedLink() {216 // Init a provider with 10 links and make its maximum number also 10.217 let links = makeLinks(0, 10, 1);218 let provider = new TestProvider(done => done(links));219 provider.maxNumLinks = links.length;220 NewTabUtils.initWithoutProviders();221 NewTabUtils.links.addProvider(provider);222 await new Promise(resolve => NewTabUtils.links.populateCache(resolve));223 do_check_links(NewTabUtils.links.getLinks(), links);224 // Notify of a new link that's low-ranked enough not to make the list.225 let newLink = makeLink(0);226 provider.notifyLinkChanged(newLink);227 do_check_links(NewTabUtils.links.getLinks(), links);228 // Notify about the new link's title change.229 provider.notifyLinkChanged({230 url: newLink.url,231 title: "a new title",232 });233 do_check_links(NewTabUtils.links.getLinks(), links);234 NewTabUtils.links.removeProvider(provider);235});236add_task(async function extractSite() {237 // All these should extract to the same site238 [239 "mozilla.org",240 "m.mozilla.org",241 "mobile.mozilla.org",242 "www.mozilla.org",243 "www3.mozilla.org",244 ].forEach(host => {245 let url = "http://" + host;246 Assert.equal(247 NewTabUtils.extractSite(url),...

Full Screen

Full Screen

cleanfailed.js

Source:cleanfailed.js Github

copy

Full Screen

1const fs = require('fs')2const storage = require('../../util/storage.js')3const removeRss = require('../../util/removeRss.js')4const channelTracker = require('../../util/channelTracker.js')5const fileOps = require('../../util/fileOps.js')6exports.normal = function (bot, message) {7 const currentGuilds = storage.currentGuilds8 const failedLinks = storage.failedLinks9 let content = message.content.split(' ')10 content.shift()11 const reason = content.join(' ').trim()12 const affectedGuilds = []13 const links = []14 currentGuilds.forEach(function (guildRss, guildId) {15 const rssList = guildRss.sources16 for (var failedLink in failedLinks) {17 if (typeof failedLinks[failedLink] !== 'string' || failedLinks[failedLink] < 100) continue18 for (var rssName in rssList) {19 if (rssList[rssName].link === failedLink) {20 if (!links.includes(rssList[rssName].link)) links.push(rssList[rssName].link)21 if (!affectedGuilds.includes(guildId)) affectedGuilds.push(guildId)22 }23 }24 }25 })26 if (links.length === 0) {27 let cleaned = false28 if (!bot.shard) {29 for (var j in failedLinks) {30 if (typeof failedLinks[j] === 'string' || failedLinks[j] >= 100) {31 cleaned = true32 delete failedLinks[j]33 }34 }35 }36 fs.writeFileSync('./settings/failedLinks.json', JSON.stringify(failedLinks, null, 2))37 return message.channel.send(cleaned ? `No links were eligible to be removed from guilds, but outdated links have been deleted from failedLinks.json.` : `No links are eligible to be cleaned out from failedLinks.json.`).catch(err => console.log(`Promise Warning: forceremove 1.`, err.message || err))38 }39 let msg = '```'40 for (var x in links) {41 msg += `\n${links[x]}`42 }43 msg += '```'44 const collector = message.channel.createMessageCollector(m => m.author.id === message.author.id, {time: 240000})45 channelTracker.add(message.channel.id)46 message.channel.send(`The list of links (${links.length}) to be cleaned out from failedLinks.json ${reason ? 'and the reason for removal ' : ''}is shown below.\n\n${reason ? '```Reason: ' + reason + '```\n' : ''}${msg.length > 1950 ? '```Unable to print links to discord - exceeds 1950 chars. Please see console.```' : msg}\n\nDo you want to continue? Type **Yes** to confirm, or **No** to cancel.`)47 .then(function (prompt) {48 if (msg.length > 1950) {49 console.log(`Bot Controller: Links that are about to be forcibly removed for cleanup as requested by (${message.author.id}, ${message.author.username}): `)50 for (var a in links) {51 console.info(`\n${links[a]}`)52 }53 }54 collector.on('collect', function (m) {55 if (m.content !== 'Yes' && m.content !== 'No') return message.channel.send('That is not a valid Option. Please type either **Yes** or **No**.')56 collector.stop()57 if (m.content === 'No') return message.channel.send(`Force removal canceled.`)58 let removedLinks = []59 for (var i in affectedGuilds) {60 const guildRss = currentGuilds.get(affectedGuilds[i])61 const rssList = guildRss.sources62 let names = []63 for (var name in rssList) {64 for (var e in links) {65 if (rssList[name].link === links[e]) {66 removedLinks.push(links[e])67 names.push(name)68 }69 }70 }71 for (var l in names) {72 const rssName = names[l]73 const link = rssList[rssName].link74 const channel = bot.channels.get(rssList[rssName].channel)75 if (reason && channel) channel.send(`**ATTENTION:** Feeds with link <${link}> have been forcibly removed from all servers. Reason: ${reason}`).catch(err => console.log(`Could not send force removal notification to server ${channel.guild.id}. `, err.message || err))76 removeRss(channel.guild.id, rssName)77 delete failedLinks[link]78 delete rssList[rssName]79 }80 fileOps.updateFile(affectedGuilds[i], guildRss)81 }82 if (removedLinks.length === 0) return message.channel.send('Unable to remove any links.').catch(err => console.log(`Promise Warning: forceremove 2. `, err.message || err))83 msg = '```'84 for (var p in removedLinks) {85 msg += `\n${removedLinks[p]}`86 }87 message.channel.send(`Successfully removed \`${removedLinks.length}\` source(s) from guilds. ${msg.length > 1950 ? '' : 'Links:```\n' + removedLinks + '```'}`)88 console.log(`Bot Controller: The following links have been forcibly removed by (${message.author.id}, ${message.author.username}): \n`, removedLinks)89 for (var j in failedLinks) {90 if (typeof failedLinks[j] === 'string' || failedLinks[j] === 100) delete failedLinks[j]91 }92 try { fs.writeFileSync('./settings/failedLinks.json', JSON.stringify(failedLinks, null, 2)) } catch (e) { console.log(`Bot Controller: Unable to write to failedLinks.json after cleanfailed. `, e.message || e) }93 })94 collector.on('end', function (collected, reason) {95 channelTracker.remove(message.channel.id)96 if (reason === 'time') return message.channel.send(`I have closed the menu due to inactivity.`).catch(err => console.log(`Promise Warning: Unable to send expired menu message.`, err.message || err))97 else if (reason !== 'user') return message.channel.send(reason)98 })99 }).catch(err => console.log(`Bot Controller: Could not send a list of links that are to be affected. `, err.message || err))100}101exports.sharded = function (bot, message, Manager) {102 const failedLinks = storage.failedLinks103 let content = message.content.split(' ')104 content.shift()105 const reason = content.join(' ').trim()106 const affectedGuilds = []107 const links = []108 bot.shard.broadcastEval(`109 const appDir = require('path').dirname(require.main.filename);110 const storage = require(appDir + '/util/storage.js');111 const currentGuilds = storage.currentGuilds;112 const failedLinks = storage.failedLinks;113 const obj = {affectedGuilds: [], links: []};114 const affectedGuilds = obj.affectedGuilds;115 const links = obj.links;116 currentGuilds.forEach(function (guildRss, guildId) {117 const rssList = guildRss.sources118 for (var failedLink in failedLinks) {119 if (typeof failedLinks[failedLink] !== 'string' || failedLinks[failedLink] < 100) continue120 for (var rssName in rssList) {121 if (rssList[rssName].link === failedLink) {122 if (!links.includes(rssList[rssName].link)) links.push(rssList[rssName].link);123 if (!affectedGuilds.includes(guildId)) affectedGuilds.push(guildId);124 }125 }126 }127 })128 obj;129 `).then(results => {130 for (var x in results) {131 const shardLinks = results[x].links132 const shardAffectedGuilds = results[x].affectedGuilds133 for (var a in shardLinks) if (!links.includes(shardLinks[a])) links.push(shardLinks[a])134 for (var b in shardAffectedGuilds) if (!affectedGuilds.includes(shardAffectedGuilds[b])) affectedGuilds.push(shardAffectedGuilds[b])135 }136 if (links.length === 0) {137 let cleaned = false138 for (var j in failedLinks) {139 if (typeof failedLinks[j] === 'string' || failedLinks[j] >= 100) {140 cleaned = true141 delete failedLinks[j]142 }143 }144 if (!cleaned) return message.channel.send(`No links are eligible to be cleaned out from failedLinks.json.`)145 try { fs.writeFileSync('./settings/failedLinks.json', JSON.stringify(failedLinks, null, 2)) } catch (e) { console.log(`Bot Controller: Unable to write to failedLinks after eval cleanfailed resulting in 0 eligible links. `, e.message || e) }146 return bot.shard.broadcastEval(`147 require(require('path').dirname(require.main.filename) + '/util/storage.js').failedLinks = JSON.parse('${JSON.stringify(failedLinks)}');148 `).then(() => {149 return message.channel.send(`No links were eligible to be removed from guilds, but outdated links have been deleted from failedLinks.json.`).catch(err => console.log(`Promise Warning: forceremove 1.`, err.message || err))150 }).catch(err => {151 console.log(`Bot Controller: Unable to broadcast failedLinks unification after eval cleanfailed resulting in 0 eligible links. `, err.message || err)152 message.channel.send(`Unable to broadcast failedLinks unification after eval cleanfailed resulting in 0 eligible links. `, err.message || err)153 })154 }155 let msg = '```'156 for (var u in links) msg += `\n${links[u]}`157 msg += '```'158 const collector = message.channel.createMessageCollector(m => m.author.id === message.author.id, {time: 240000})159 channelTracker.add(message.channel.id)160 message.channel.send(`The list of links (${links.length}) to be cleaned out from failedLinks.json ${reason ? 'and the reason for removal ' : ''}is shown below.\n\n${reason ? '```Reason: ' + reason + '```\n' : ''}${msg.length > 1950 ? '```Unable to print links to discord - exceeds 1950 chars. Please see console.```' : msg}\n\nDo you want to continue? Type **Yes** to confirm, or **No** to cancel.`)161 .then(function (prompt) {162 if (msg.length > 1950) {163 console.log(`Bot Controller: Links that are about to be forcibly removed for cleanup as requested by (${message.author.id}, ${message.author.username}): `)164 for (var a in links) {165 console.info(`\n${links[a]}`)166 }167 }168 collector.on('collect', function (m) {169 if (m.content !== 'Yes' && m.content !== 'No') return message.channel.send('That is not a valid Option. Please type either **Yes** or **No**.')170 collector.stop()171 if (m.content === 'No') return message.channel.send(`Force removal canceled.`)172 bot.shard.broadcastEval(`173 const appDir = require('path').dirname(require.main.filename);174 const storage = require(appDir + '/util/storage.js');175 const currentGuilds = storage.currentGuilds;176 const failedLinks = storage.failedLinks;177 const removeRss = require(appDir + '/util/removeRss.js');178 const fileOps = require(appDir + '/util/fileOps.js');179 const removedLinks = [];180 const affectedGuilds = JSON.parse('${JSON.stringify(affectedGuilds)}');181 const links = JSON.parse('${JSON.stringify(links)}')182 for (var i in affectedGuilds) {183 const guildRss = currentGuilds.get(affectedGuilds[i]);184 if (!guildRss) continue;185 const rssList = guildRss.sources;186 let names = [];187 for (var name in rssList) {188 for (var e in links) {189 if (rssList[name].link === links[e]) {190 removedLinks.push(links[e]);191 names.push(name);192 }193 }194 }195 for (var l in names) {196 const rssName = names[l];197 const link = rssList[rssName].link;198 const channel = this.channels.get(rssList[rssName].channel);199 if (${reason ? '"' + reason + '"' : undefined} && channel) channel.send('**ATTENTION:** Feeds with link <' + link + '> have been forcibly removed from all servers. Reason: ${reason ? '"' + reason + '"' : undefined}').catch(err => console.log('Could not send force removal notification to server ' + channel.guild.id + '. ', err.message || err))200 removeRss(channel.guild.id, rssName);201 delete failedLinks[link];202 delete rssList[rssName];203 }204 fileOps.updateFile(affectedGuilds[i], guildRss);205 }206 removedLinks;207 `).then(results => {208 let removed = []209 let msg = '```'210 let total = 0211 for (var a in results) {212 const removedLinks = results[a]213 if (!removedLinks) continue214 removed = removed.concat(removedLinks)215 total += removedLinks.length216 }217 for (var b in removed) {218 const link = removed[b]219 msg += '\n' + link220 delete failedLinks[link]221 }222 if (removed.length === 0) return message.channel.send('Unable to remove any links.').catch(err => console.log(`Promise Warning: forceremove 2s. `, err.message || err))223 for (var j in failedLinks) {224 if (typeof failedLinks[j] === 'string' || failedLinks[j] === 100) delete failedLinks[j]225 }226 bot.shard.broadcastEval(`227 require(require('path').dirname(require.main.filename) + '/util/storage.js').failedLinks = JSON.parse('${JSON.stringify(failedLinks)}');228 `).then(() => {229 message.channel.send(`Successfully removed \`${total}\` source(s) from guilds. ${msg.length > 1950 ? '' : 'Links:```\n' + removed + '```'}`)230 console.log(`Bot Controller: The following links have been forcibly removed by (${message.author.id}, ${message.author.username}): \n`, removed)231 try { fs.writeFileSync('./settings/failedLinks.json', JSON.stringify(failedLinks, null, 2)) } catch (e) { console.log(`Bot Controller: Unable to write to failedLinks.json after cleanfailed. `, e.message || e) }232 }).catch(err => console.log(`Bot Controller: Unable to broadcast failedLinks cache update after successful eval cleanfailed and removal action. `, err.message || err))233 }).catch(err => console.log(`Bot Controller: Unable to broadcast removal actions after successful eval cleanfailed. `, err.message || err))234 })235 collector.on('end', function (collected, reason) {236 channelTracker.remove(message.channel.id)237 if (reason === 'time') return message.channel.send(`I have closed the menu due to inactivity.`).catch(err => console.log(`Promise Warning: Unable to send expired menu message.`, err.message || err))238 else if (reason !== 'user') return message.channel.send(reason)239 })240 })241 }).catch(err => console.log(`Bot Controller: Unable to broadcast eval cleanfailed. `, err.message || err))...

Full Screen

Full Screen

LinkSaveIn.py

Source:LinkSaveIn.py Github

copy

Full Screen

1# -*- coding: utf-8 -*-23#4# v2.01 - hagg5# * cnl2 and web links are skipped if JS is not available (instead of failing the package)6# * only best available link source is used (priority: cnl2>rsdf>ccf>dlc>web7#89from Crypto.Cipher import AES10from module.plugins.Crypter import Crypter11from module.unescape import unescape12import base6413import binascii14import re1516class LinkSaveIn(Crypter):17 __name__ = "LinkSaveIn"18 __type__ = "crypter"19 __pattern__ = r"http://(www\.)?linksave.in/(?P<id>\w+)$"20 __version__ = "2.01"21 __description__ = """LinkSave.in Crypter Plugin"""22 __author_name__ = ("fragonib")23 __author_mail__ = ("fragonib[AT]yahoo[DOT]es")2425 # Constants26 _JK_KEY_ = "jk"27 _CRYPTED_KEY_ = "crypted"28 HOSTER_DOMAIN = "linksave.in"29 30 def setup(self):31 self.html = None32 self.fileid = None33 self.captcha = False34 self.package = None35 self.preferred_sources = ['cnl2', 'rsdf', 'ccf', 'dlc', 'web']36 37 def decrypt(self, pyfile):3839 # Init40 self.package = pyfile.package()41 self.fileid = re.match(self.__pattern__, pyfile.url).group('id')42 self.req.cj.setCookie(self.HOSTER_DOMAIN, "Linksave_Language", "english")43 44 # Request package45 self.html = self.load(self.pyfile.url)46 if not self.isOnline():47 self.offline()48 49 # Check for protection 50 if self.isPasswordProtected():51 self.unlockPasswordProtection()52 self.handleErrors()53 54 if self.isCaptchaProtected():55 self.captcha = True56 self.unlockCaptchaProtection()57 self.handleErrors()5859 # Get package name and folder60 (package_name, folder_name) = self.getPackageInfo()6162 # Extract package links63 package_links = []64 for type_ in self.preferred_sources:65 package_links.extend(self.handleLinkSource(type_))66 if package_links: # use only first source which provides links67 break68 package_links = set(package_links)6970 # Pack71 if package_links:72 self.packages = [(package_name, package_links, folder_name)]73 else:74 self.fail('Could not extract any links')7576 def isOnline(self):77 if "<big>Error 404 - Folder not found!</big>" in self.html:78 self.logDebug("File not found")79 return False80 return True81 82 def isPasswordProtected(self):83 if re.search(r'''<input.*?type="password"''', self.html):84 self.logDebug("Links are password protected")85 return True86 87 def isCaptchaProtected(self):88 if "<b>Captcha:</b>" in self.html:89 self.logDebug("Links are captcha protected")90 return True91 return False92 93 def unlockPasswordProtection(self):94 password = self.getPassword()95 self.logDebug("Submitting password [%s] for protected links" % password)96 post = {"id": self.fileid, "besucherpasswort": password, 'login': 'submit'}97 self.html = self.load(self.pyfile.url, post=post)98 99 def unlockCaptchaProtection(self):100 hash = re.search(r'name="hash" value="([^"]+)', self.html).group(1)101 captchaUrl = re.search(r'src=".(/captcha/cap.php\?hsh=[^"]+)', self.html).group(1)102 code = self.decryptCaptcha("http://linksave.in" + captchaUrl, forceUser=True)103 self.html = self.load(self.pyfile.url, post={"id": self.fileid, "hash": hash, "code": code}) 104105 def getPackageInfo(self):106 name = self.pyfile.package().name107 folder = self.pyfile.package().folder108 self.logDebug("Defaulting to pyfile name [%s] and folder [%s] for package" % (name, folder))109 return name, folder110 111 def handleErrors(self): 112 if "The visitorpassword you have entered is wrong" in self.html:113 self.logDebug("Incorrect password, please set right password on 'Edit package' form and retry")114 self.fail("Incorrect password, please set right password on 'Edit package' form and retry") 115116 if self.captcha: 117 if "Wrong code. Please retry" in self.html:118 self.logDebug("Invalid captcha, retrying")119 self.invalidCaptcha()120 self.retry()121 else:122 self.correctCaptcha()123 124 def handleLinkSource(self, type_):125 if type_ == 'cnl2':126 return self.handleCNL2()127 elif type_ in ('rsdf', 'ccf', 'dlc'):128 return self.handleContainer(type_)129 elif type_ == 'web':130 return self.handleWebLinks()131 else:132 self.fail('unknown source type "%s" (this is probably a bug)' % type_)133134 def handleWebLinks(self):135 package_links = []136 self.logDebug("Search for Web links")137 if not self.js:138 self.logDebug("no JS -> skip Web links")139 else: 140 #@TODO: Gather paginated web links 141 pattern = r'<a href="http://linksave\.in/(\w{43})"'142 ids = re.findall(pattern, self.html)143 self.logDebug("Decrypting %d Web links" % len(ids))144 for i, id in enumerate(ids):145 try:146 webLink = "http://linksave.in/%s" % id147 self.logDebug("Decrypting Web link %d, %s" % (i+1, webLink))148 fwLink = "http://linksave.in/fw-%s" % id149 response = self.load(fwLink)150 jscode = re.findall(r'<script type="text/javascript">(.*)</script>', response)[-1]151 jseval = self.js.eval("document = { write: function(e) { return e; } }; %s" % jscode)152 dlLink = re.search(r'http://linksave\.in/dl-\w+', jseval).group(0)153 self.logDebug("JsEngine returns value [%s] for redirection link" % dlLink)154 response = self.load(dlLink)155 link = unescape(re.search(r'<iframe src="(.+?)"', response).group(1))156 package_links.append(link)157 except Exception, detail:158 self.logDebug("Error decrypting Web link %s, %s" % (webLink, detail)) 159 return package_links160 161 def handleContainer(self, type_):162 package_links = []163 type_ = type_.lower()164 self.logDebug('Seach for %s Container links' % type_.upper())165 if not type_.isalnum(): # check to prevent broken re-pattern (cnl2,rsdf,ccf,dlc,web are all alpha-numeric)166 self.fail('unknown container type "%s" (this is probably a bug)' % type_)167 pattern = r"\('%s_link'\).href=unescape\('(.*?\.%s)'\)" % (type_, type_)168 containersLinks = re.findall(pattern, self.html)169 self.logDebug("Found %d %s Container links" % (len(containersLinks), type_.upper()))170 for containerLink in containersLinks:171 link = "http://linksave.in/%s" % unescape(containerLink)172 package_links.append(link)173 return package_links174175 def handleCNL2(self):176 package_links = []177 self.logDebug("Search for CNL2 links")178 if not self.js:179 self.logDebug("no JS -> skip CNL2 links")180 elif 'cnl2_load' in self.html:181 try:182 (vcrypted, vjk) = self._getCipherParams()183 for (crypted, jk) in zip(vcrypted, vjk):184 package_links.extend(self._getLinks(crypted, jk))185 except:186 self.fail("Unable to decrypt CNL2 links") 187 return package_links188 189 def _getCipherParams(self):190 191 # Get jk192 jk_re = r'<INPUT.*?NAME="%s".*?VALUE="(.*?)"' % LinkSaveIn._JK_KEY_ 193 vjk = re.findall(jk_re, self.html)194 195 # Get crypted196 crypted_re = r'<INPUT.*?NAME="%s".*?VALUE="(.*?)"' % LinkSaveIn._CRYPTED_KEY_ 197 vcrypted = re.findall(crypted_re, self.html)198199 # Log and return200 self.logDebug("Detected %d crypted blocks" % len(vcrypted))201 return vcrypted, vjk202203 def _getLinks(self, crypted, jk):204205 # Get key206 jreturn = self.js.eval("%s f()" % jk)207 self.logDebug("JsEngine returns value [%s]" % jreturn)208 key = binascii.unhexlify(jreturn)209210 # Decode crypted211 crypted = base64.standard_b64decode(crypted)212213 # Decrypt214 Key = key215 IV = key216 obj = AES.new(Key, AES.MODE_CBC, IV)217 text = obj.decrypt(crypted)218219 # Extract links220 text = text.replace("\x00", "").replace("\r", "")221 links = text.split("\n")222 links = filter(lambda x: x != "", links)223224 # Log and return225 self.logDebug("Package has %d links" % len(links))226 return links ...

Full Screen

Full Screen

NCryptIn.py

Source:NCryptIn.py Github

copy

Full Screen

1# -*- coding: utf-8 -*-23from Crypto.Cipher import AES4from module.plugins.Crypter import Crypter5from module.plugins.ReCaptcha import ReCaptcha6import base647import binascii8import re910class NCryptIn(Crypter):11 __name__ = "NCryptIn"12 __type__ = "crypter"13 __pattern__ = r"http://(?:www\.)?ncrypt.in/folder-([^/\?]+)"14 __version__ = "1.21"15 __description__ = """NCrypt.in Crypter Plugin"""16 __author_name__ = ("fragonib")17 __author_mail__ = ("fragonib[AT]yahoo[DOT]es")1819 # Constants20 _JK_KEY_ = "jk"21 _CRYPTED_KEY_ = "crypted"22 23 def setup(self):24 self.html = None25 self.cleanedHtml = None26 self.captcha = False27 self.package = None2829 def decrypt(self, pyfile):30 31 # Init32 self.package = pyfile.package()33 34 # Request package35 self.html = self.load(self.pyfile.url)36 self.cleanedHtml = self.removeCrap(self.html)37 if not self.isOnline():38 self.offline()39 40 # Check for protection 41 if self.isProtected():42 self.html = self.unlockProtection()43 self.cleanedHtml = self.removeCrap(self.html)44 self.handleErrors()4546 # Get package name and folder47 (package_name, folder_name) = self.getPackageInfo()4849 # Extract package links50 package_links = []51 package_links.extend(self.handleWebLinks())52 package_links.extend(self.handleContainers())53 package_links.extend(self.handleCNL2())54 package_links = set(package_links)5556 # Pack57 self.packages = [(package_name, package_links, folder_name)]58 59 def removeCrap(self, content):60 patterns = (r'(type="hidden".*?(name=".*?")?.*?value=".*?")',61 r'display:none;">(.*?)</(div|span)>',62 r'<div\s+class="jdownloader"(.*?)</div>',63 r'<iframe\s+style="display:none(.*?)</iframe>')64 for pattern in patterns:65 rexpr = re.compile(pattern, re.DOTALL)66 content = re.sub(rexpr, "", content)67 return content68 69 def isOnline(self): 70 if "Your folder does not exist" in self.cleanedHtml:71 self.logDebug("File not found")72 return False73 return True74 75 def isProtected(self):76 if re.search(r'''<form.*?name.*?protected.*?>''', self.cleanedHtml):77 self.logDebug("Links are protected")78 return True79 return False80 81 def getPackageInfo(self):82 title_re = r'<h2><span.*?class="arrow".*?>(?P<title>[^<]+).*?</span>.*?</h2>'83 m = re.findall(title_re, self.html, re.DOTALL)84 if m is not None:85 title = m[-1].strip()86 name = folder = title87 self.logDebug("Found name [%s] and folder [%s] in package info" % (name, folder))88 else:89 name = self.package.name90 folder = self.package.folder91 self.logDebug("Package info not found, defaulting to pyfile name [%s] and folder [%s]" % (name, folder))92 return name, folder93 94 def unlockProtection(self):95 96 postData = {}97 98 form = re.search(r'''<form\ name="protected"(.*?)</form>''', self.cleanedHtml, re.DOTALL).group(1)99 100 # Submit package password101 if "password" in form:102 password = self.getPassword()103 self.logDebug("Submitting password [%s] for protected links" % password)104 postData['password'] = password105 106 # Resolve anicaptcha107 if "anicaptcha" in form:108 self.captcha = True109 self.logDebug("Captcha protected, resolving captcha")110 captchaUri = re.search(r'src="(/temp/anicaptcha/[^"]+)', form).group(1)111 captcha = self.decryptCaptcha("http://ncrypt.in" + captchaUri)112 self.logDebug("Captcha resolved [%s]" % captcha)113 postData['captcha'] = captcha114 115 # Resolve recaptcha 116 if "recaptcha" in form:117 self.captcha = True 118 id = re.search(r'\?k=(.*?)"', form).group(1)119 self.logDebug("Resolving ReCaptcha with key [%s]" % id)120 recaptcha = ReCaptcha(self)121 challenge, code = recaptcha.challenge(id)122 postData['recaptcha_challenge_field'] = challenge123 postData['recaptcha_response_field'] = code124 125 # Unlock protection126 postData['submit_protected'] = 'Continue to folder '127 return self.load(self.pyfile.url, post=postData)128 129 def handleErrors(self):130 131 if "This password is invalid!" in self.cleanedHtml:132 self.logDebug("Incorrect password, please set right password on 'Edit package' form and retry")133 self.fail("Incorrect password, please set right password on 'Edit package' form and retry") 134135 if self.captcha: 136 if "The securitycheck was wrong!" in self.cleanedHtml:137 self.logDebug("Invalid captcha, retrying")138 self.invalidCaptcha()139 self.retry()140 else:141 self.correctCaptcha()142143 def handleWebLinks(self):144 package_links = []145 self.logDebug("Handling Web links")146 147 pattern = r"(http://ncrypt\.in/link-.*?=)"148 links = re.findall(pattern, self.html)149 self.logDebug("Decrypting %d Web links" % len(links))150 for i, link in enumerate(links):151 self.logDebug("Decrypting Web link %d, %s" % (i+1, link))152 try:153 url = link.replace("link-", "frame-")154 link = self.load(url, just_header=True)['location']155 package_links.append(link)156 except Exception, detail:157 self.logDebug("Error decrypting Web link %s, %s" % (link, detail)) 158 return package_links159 160 def handleContainers(self):161 package_links = []162 self.logDebug("Handling Container links")163 164 pattern = r"/container/(rsdf|dlc|ccf)/([a-z0-9]+)"165 containersLinks = re.findall(pattern, self.html)166 self.logDebug("Decrypting %d Container links" % len(containersLinks))167 for containerLink in containersLinks:168 link = "http://ncrypt.in/container/%s/%s.%s" % (containerLink[0], containerLink[1], containerLink[0])169 package_links.append(link)170 return package_links171 172 def handleCNL2(self):173 package_links = []174 self.logDebug("Handling CNL2 links")175 176 if 'cnl2_output' in self.cleanedHtml:177 try:178 (vcrypted, vjk) = self._getCipherParams()179 for (crypted, jk) in zip(vcrypted, vjk):180 package_links.extend(self._getLinks(crypted, jk))181 except:182 self.fail("Unable to decrypt CNL2 links") 183 return package_links184 185 def _getCipherParams(self):186 187 pattern = r'<input.*?name="%s".*?value="(.*?)"' 188 189 # Get jk190 jk_re = pattern % NCryptIn._JK_KEY_ 191 vjk = re.findall(jk_re, self.html)192 193 # Get crypted194 crypted_re = pattern % NCryptIn._CRYPTED_KEY_ 195 vcrypted = re.findall(crypted_re, self.html)196197 # Log and return198 self.logDebug("Detected %d crypted blocks" % len(vcrypted))199 return vcrypted, vjk200201 def _getLinks(self, crypted, jk):202203 # Get key204 jreturn = self.js.eval("%s f()" % jk)205 self.logDebug("JsEngine returns value [%s]" % jreturn)206 key = binascii.unhexlify(jreturn)207208 # Decode crypted209 crypted = base64.standard_b64decode(crypted)210211 # Decrypt212 Key = key213 IV = key214 obj = AES.new(Key, AES.MODE_CBC, IV)215 text = obj.decrypt(crypted)216217 # Extract links218 text = text.replace("\x00", "").replace("\r", "")219 links = text.split("\n")220 links = filter(lambda x: x != "", links)221222 # Log and return223 self.logDebug("Block has %d links" % len(links)) ...

Full Screen

Full Screen

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.

LambdaTest Learning Hubs:

YouTube

You could also refer to video tutorials over LambdaTest YouTube channel to get step by step demonstration from industry experts.

Run Kiwi automation tests on LambdaTest cloud grid

Perform automation testing on 3000+ real desktop and mobile devices online.

Try LambdaTest Now !!

Get 100 minutes of automation test minutes FREE!!

Next-Gen App & Browser Testing Cloud

Was this article helpful?

Helpful

NotHelpful