How to use removeNoise method in mountebank

Best JavaScript code snippet using mountebank

clustering.js

Source:clustering.js

1'use strict';2//get the tweets model3var TweetsDB = require('../model/tweetsSchema');4//include async module5var async = require('async');6//include logger config7var logger = require('../logger-config/log-config');8//include output file config9var output = require('../logger-config/finalOutput');10//regex for removing the emojis from the text11var regex = /(?:[\u2700-\u27bf]|(?:\ud83c[\udde6-\uddff]){2}|[\ud800-\udbff][\udc00-\udfff]|[\u0023-\u0039]\ufe0f?\u20e3|\u3299|\u3297|\u303d|\u3030|\u24c2|\ud83c[\udd70-\udd71]|\ud83c[\udd7e-\udd7f]|\ud83c\udd8e|\ud83c[\udd91-\udd9a]|\ud83c[\udde6-\uddff]|\ud83c[\ude01-\ude02]|\ud83c\ude1a|\ud83c\ude2f|\ud83c[\ude32-\ude3a]|\ud83c[\ude50-\ude51]|\u203c|\u2049|[\u25aa-\u25ab]|\u25b6|\u25c0|[\u25fb-\u25fe]|\u00a9|\u00ae|\u2122|\u2139|\ud83c\udc04|[\u2600-\u26FF]|\u2b05|\u2b06|\u2b07|\u2b1b|\u2b1c|\u2b50|\u2b55|\u231a|\u231b|\u2328|\u23cf|[\u23e9-\u23f3]|[\u23f8-\u23fa]|\ud83c\udccf|\u2934|\u2935|[\u2190-\u21ff])/g;12//all tweets array13var totalTweets = [];14//import minhash and lsh modules15var {16    Minhash,17    LshIndex18} = require('minhash');19//create object for lsh20var index = new LshIndex();21//include plotly graph22var plotly = require('plotly')("kinshukb", "o4ZxVuiWSEGM2jAQud16");23//creating a new variable to get more graphs24var plotly2 = require('plotly')("kinshukbhardwaj", "wW4Gfxn5JeMqvbvYmF9K");25//variables to store new cluster information for geo data and user geo information26//variables to store if each entry of the cluster contains geo data and user geo information or not27var newArrayPlace = [],28    newArrayGeoEnabled = [],29    eachNewInternalArrayPlace = [],30    eachNewInternalArrayGeoEnabled = [];31//variable to include stopword module to remove noise from the text32var removeNoise = require('stopword');33//variables to store location of each tweet in the clusters formed34var newArrayLocation = [];35//variable to store country of each tweet in clusters found36var newArrayCountry = [];37//variable to store the total number of glasgow clusters38var totalGlasgowCLusters = 0;39//variable to store the number of cluster which are not glasgow clusters40var totalNotGlasgowClusters = 0;41//variable to store clusters without any location information42var totalNullLocationClusters = 0;43//variable to store all UK clusters44var totalUKClusters = 0;45exports.minhashLshClustering = function () {46    async.waterfall([47        //1. Function to get all no filter tweets from database48        function (callback) {49            TweetsDB.tweetsSTREAMNoFilter.find({}, function (err, nofilterTweets) {50                if (err) {51                    //log the error52                    logger.error('minhashLshClustering: Error in finding geo tweets in location filter stream collection: ' + err);53                    //continue to the next function54                    callback(null, totalTweets);55                } else {56                    async.forEachSeries(nofilterTweets, function (eachNoFilterTweet, callback) {57                            var dataToWrite = {58                                'text': '',59                                'id': '',60                                'place': {},61                                'geoEnabled': '',62                                'countryCode': ''63                            };64                            dataToWrite.text = JSON.parse(JSON.stringify(eachNoFilterTweet)).text.replace(regex, '').toString();65                            dataToWrite.id = JSON.parse(JSON.stringify(eachNoFilterTweet)).id_str;66                            dataToWrite.place = JSON.parse(JSON.stringify(eachNoFilterTweet)).place ? JSON.parse(JSON.stringify(eachNoFilterTweet)).place.name : 'null';67                            dataToWrite.geoEnabled = JSON.parse(JSON.stringify(eachNoFilterTweet)).user.geo_enabled;68                            dataToWrite.countryCode = JSON.parse(JSON.stringify(eachNoFilterTweet)).place ? JSON.parse(JSON.stringify(eachNoFilterTweet)).place.country_code : 'null';69                            totalTweets.push(dataToWrite);70                            callback();71                        },72                        function (errAsyncDuplicate) {73                            //handle error74                            if (errAsyncDuplicate) {75                                //log the error found76                                logger.error('minhashLshClustering: Error in async for minhashLshClustering 1: ' + errAsyncDuplicate);77                                //go to next function78                                callback(null, totalTweets);79                            } else {80                                //log the final output81                                logger.info('minhashLshClustering: Going to function 2 to find rest tweets');82                                //go to next function83                                callback(null, totalTweets);84                            }85                        });86                }87            });88        },89        //2. Function to find all rest tweets from db90        function (totalTweets, callback) {91            TweetsDB.tweetsREST.find({}, function (err, restTweets) {92                if (err) {93                    //log the error94                    logger.error('minhashLshClustering: Error in finding geo tweets in location filter stream collection: ' + err);95                    //continue to the next function96                    callback(null, totalTweets);97                } else {98                    async.forEachSeries(restTweets, function (eachRestTweet, callback) {99                            var dataToWrite = {100                                'text': '',101                                'id': '',102                                'place': {},103                                'geoEnabled': '',104                                'countryCode': ''105                            };106                            dataToWrite.text = JSON.parse(JSON.stringify(eachRestTweet)).text.replace(regex, '').toString();107                            dataToWrite.id = JSON.parse(JSON.stringify(eachRestTweet)).id_str;108                            dataToWrite.place = JSON.parse(JSON.stringify(eachRestTweet)).place ? JSON.parse(JSON.stringify(eachRestTweet)).place : 'null';109                            dataToWrite.geoEnabled = JSON.parse(JSON.stringify(eachRestTweet)).user.geo_enabled;110                            dataToWrite.countryCode = JSON.parse(JSON.stringify(eachRestTweet)).place ? JSON.parse(JSON.stringify(eachRestTweet)).place.country_code : 'null';111                            totalTweets.push(dataToWrite);112                            callback();113                        },114                        function (errAsyncDuplicate) {115                            //handle error116                            if (errAsyncDuplicate) {117                                //log the error found118                                logger.error('minhashLshClustering: Error in async for minhashLshClustering 2: ' + errAsyncDuplicate);119                                //go to next function120                                callback(null, totalTweets);121                            } else {122                                //log the final output123                                logger.info('minhashLshClustering: Going to function 3 to find keyword filtered tweets');124                                //go to next function125                                callback(null, totalTweets);126                            }127                        });128                }129            });130            // callback(null, totalTweets);131        },132        //3. Function to find all keyword filtered tweets from db133        function (totalTweets, callback) {134            TweetsDB.tweetsSTREAMKeywordFilter.find({}, function (err, keywordTweets) {135                if (err) {136                    //log the error137                    logger.error('minhashLshClustering: Error in finding geo tweets in location filter stream collection: ' + err);138                    //continue to the next function139                    callback(null, totalTweets);140                } else {141                    async.forEachSeries(keywordTweets, function (eachKeywordTweet, callback) {142                            var dataToWrite = {143                                'text': '',144                                'id': '',145                                'place': {},146                                'geoEnabled': '',147                                'countryCode': ''148                            };149                            dataToWrite.text = JSON.parse(JSON.stringify(eachKeywordTweet)).text.replace(regex, '').toString();150                            dataToWrite.id = JSON.parse(JSON.stringify(eachKeywordTweet)).id_str;151                            dataToWrite.place = JSON.parse(JSON.stringify(eachKeywordTweet)).place ? JSON.parse(JSON.stringify(eachKeywordTweet)).place : 'null';152                            dataToWrite.geoEnabled = JSON.parse(JSON.stringify(eachKeywordTweet)).user.geo_enabled;153                            dataToWrite.countryCode = JSON.parse(JSON.stringify(eachKeywordTweet)).place ? JSON.parse(JSON.stringify(eachKeywordTweet)).place.country_code : 'null';154                            totalTweets.push(dataToWrite);155                            callback();156                        },157                        function (errAsyncDuplicate) {158                            //handle error159                            if (errAsyncDuplicate) {160                                //log the error found161                                logger.error('minhashLshClustering: Error in async for minhashLshClustering 3: ' + errAsyncDuplicate);162                                //go to next function163                                callback(null, totalTweets);164                            } else {165                                //log the final output166                                logger.info('minhashLshClustering: Going to function 4 to find geo tweets');167                                //go to next function168                                callback(null, totalTweets);169                            }170                        });171                }172            });173            // callback(null, totalTweets);174        },175        //4. Function to find all geo tweets from db176        function (totalTweets, callback) {177            TweetsDB.tweetsSTREAMLocationFilter.find({}, function (err, geoTweets) {178                if (err) {179                    //log the error180                    logger.error('minhashLshClustering: Error in finding geo tweets in location filter stream collection: ' + err);181                    //continue to the next function182                    callback(null, totalTweets);183                } else {184                    async.forEachSeries(geoTweets, function (eachGeoTweet, callback) {185                            var dataToWrite = {186                                'text': '',187                                'id': '',188                                'place': {},189                                'geoEnabled': '',190                                'countryCode': ''191                            };192                            dataToWrite.text = JSON.parse(JSON.stringify(eachGeoTweet)).text.replace(regex, '').toString();193                            dataToWrite.id = JSON.parse(JSON.stringify(eachGeoTweet)).id_str;194                            dataToWrite.place = JSON.parse(JSON.stringify(eachGeoTweet)).place ? JSON.parse(JSON.stringify(eachGeoTweet)).place : 'null';195                            dataToWrite.geoEnabled = JSON.parse(JSON.stringify(eachGeoTweet)).user.geo_enabled;196                            dataToWrite.countryCode = JSON.parse(JSON.stringify(eachGeoTweet)).place ? JSON.parse(JSON.stringify(eachGeoTweet)).place.country_code : 'null';197                            totalTweets.push(dataToWrite);198                            callback();199                        },200                        function (errAsyncDuplicate) {201                            //handle error202                            if (errAsyncDuplicate) {203                                //log the error found204                                logger.error('minhashLshClustering: Error in async for minhashLshClustering 4: ' + errAsyncDuplicate);205                                //go to next function206                                callback(null, totalTweets);207                            } else {208                                //log the final output209                                logger.info('minhashLshClustering: Going to function 5 to form indexes using minhash lsh');210                                //go to next function211                                callback(null, totalTweets);212                            }213                        });214                }215            });216            // callback(null, totalTweets);217        },218        //5. Function to work with minhash lsh and create hashes and indexes219        function (totalTweets, callback) {220            var mArr = [];221            var count = 0;222            //loop through all tweets to get texts of each of them to remove noise like emojis, stopwords223            //from the tweets and them cluster them using Minhash LSH224            async.forEachSeries(totalTweets, function (each, callback) {225                    //create array of words for each tweet text226                    var textSplit = each.text.split(' ');227                    //remove the noise of different languages228                    //remove arabic229                    var updatedText = removeNoise.removeStopwords(textSplit, removeNoise.ar);230                    //remove bengali231                    var updatedText = removeNoise.removeStopwords(updatedText, removeNoise.bn);232                    //remove Brazilian Portuguese233                    var updatedText = removeNoise.removeStopwords(updatedText, removeNoise.br);234                    //remove Danish noise235                    var updatedText = removeNoise.removeStopwords(updatedText, removeNoise.da);236                    //remove German noise237                    var updatedText = removeNoise.removeStopwords(updatedText, removeNoise.de);238                    //remove English noise239                    var updatedText = removeNoise.removeStopwords(updatedText, removeNoise.en);240                    //remove Spanish noise241                    var updatedText = removeNoise.removeStopwords(updatedText, removeNoise.es);242                    //remove Farsi noise243                    var updatedText = removeNoise.removeStopwords(updatedText, removeNoise.fa);244                    //remove French noise245                    var updatedText = removeNoise.removeStopwords(updatedText, removeNoise.fr);246                    //remove Hindi noise247                    var updatedText = removeNoise.removeStopwords(updatedText, removeNoise.hi);248                    //remove Italian noise249                    var updatedText = removeNoise.removeStopwords(updatedText, removeNoise.it);250                    //remove Japanese noise251                    var updatedText = removeNoise.removeStopwords(updatedText, removeNoise.ja);252                    //remove Dutch noise253                    var updatedText = removeNoise.removeStopwords(updatedText, removeNoise.nl);254                    //remove Norwegian noise255                    var updatedText = removeNoise.removeStopwords(updatedText, removeNoise.no);256                    //remove Polish noise257                    var updatedText = removeNoise.removeStopwords(updatedText, removeNoise.pl);258                    //remove Portuguese noise259                    var updatedText = removeNoise.removeStopwords(updatedText, removeNoise.pt);260                    //remove Punjab gurmukhi noise261                    var updatedText = removeNoise.removeStopwords(updatedText, removeNoise.pa_in);262                    //remove Russian noise263                    var updatedText = removeNoise.removeStopwords(updatedText, removeNoise.ru);264                    //remove Swedish noise265                    var updatedText = removeNoise.removeStopwords(updatedText, removeNoise.sv);266                    //remove Chinese simplified noise267                    var updatedText = removeNoise.removeStopwords(updatedText, removeNoise.zh);268                    //new variable for each text hash269                    var m = 'm' + count;270                    //increment count for next variable271                    count++;272                    //new hashing273                    m = new Minhash();274                    //update each sentence275                    updatedText.map(function (w) {276                        m.update(w);277                    });278                    //add hashes to the index after updation279                    index.insert(each.id + "-" + each.place + "-" + each.geoEnabled + "-" + each.countryCode, m);280                    //add to m to late to compute closeness for each sentense281                    mArr.push(m);282                    //go to next iteration283                    callback();284                },285                //final function for async286                function (errFinal) {287                    if (errFinal) {288                        //log error289                        logger.error('Error in minhash calculation asynch final: ' + errFinal);290                        //go to next function291                        callback(null, totalTweets, mArr);292                    } else {293                        logger.info('minhashLshClustering: Going to function 6 to form clusters using minhash lsh');294                        //go to next function295                        callback(null, totalTweets, mArr);296                    }297                });298        },299        //6. Function to perform query of lsh of each sentense300        function (totalTweets, mArr, callback) {301            //store all clusters for each sentence302            var allMatches = [];303            //variable to store unique clusters304            var allMatchesUnique = [];305            //loop through all indexes formed306            async.forEachSeries(mArr, function (eachM, callback) {307                    //find clusters308                    var match = index.query(eachM);309                    //form a cluster of arrays310                    allMatches.push(match);311                    //go to next index312                    callback();313                },314                //final function for async315                function (errFinal) {316                    if (errFinal) {317                        //log error318                        logger.error('Error matches in forming clusters based on minhash LSH: ' + errFinal);319                        //go to next function320                        callback(null, totalTweets, allMatchesUnique);321                    } else {322                        //convert the arrays to JSON strings and use a Set to get unique values323                        var set = new Set(allMatches.map(JSON.stringify));324                        //convert back to array of arrays again325                        allMatchesUnique = Array.from(set).map(JSON.parse);326                        logger.info('minhashLshClustering: Going to function 7 to form the cluster graph');327                        //go to next function328                        callback(null, totalTweets, allMatchesUnique);329                    }330                });331        },332        //7. Function to gather data to plot the graph333        function (totalTweets, allMatchesUnique, callback) {334            //commenting out as this is generally used to find the number of clusters formed335            logger.info('total cluster formed: ' + allMatchesUnique.length);336            var one = [],337                two = [],338                threeTo4 = [],339                fiveTo9 = [],340                tenTo19 = [],341                twentyTo49 = [],342                moreThan50 = [];343            async.forEachSeries(allMatchesUnique, function (eachMatch, callback) {344                    if (eachMatch.length >= 1 && eachMatch.length < 2) {345                        one.push(eachMatch);346                        callback();347                    } else if (eachMatch.length >= 2 && eachMatch.length < 3) {348                        two.push(eachMatch);349                        callback();350                    } else if (eachMatch.length >= 3 && eachMatch.length < 5) {351                        threeTo4.push(eachMatch);352                        callback();353                    } else if (eachMatch.length >= 5 && eachMatch.length < 10) {354                        fiveTo9.push(eachMatch);355                        callback();356                    } else if (eachMatch.length >= 10 && eachMatch.length < 20) {357                        tenTo19.push(eachMatch);358                        callback();359                    } else if (eachMatch.length >= 20 && eachMatch.length < 50) {360                        twentyTo49.push(eachMatch);361                        callback();362                    } else {363                        moreThan50.push(eachMatch);364                        callback();365                    }366                },367                function (err) {368                    var xVal = ['1', '2', '3-4', '5-9', '10-19', '20-49', 'More than equal to 50'];369                    var yVal = [one.length, two.length, threeTo4.length, fiveTo9.length, tenTo19.length, twentyTo49.length, moreThan50.length];370                    var data = [{371                        x: xVal,372                        y: yVal,373                        type: 'bar',374                        text: yVal.map(String),375                        textposition: 'auto',376                        hoverinfo: 'none',377                        marker: {378                            color: 'rgb(158,202,225)',379                            opacity: 0.6,380                            line: {381                                color: 'rgb(8,48,107)',382                                width: 1.5383                            }384                        }385                    }];386                    var layout = {387                        title: "Number of clusters in Ranges",388                        fileopt: "overwrite",389                        filename: "simple-node-example",390                        font: {391                            family: "Raleway, sans-serif"392                        },393                        bargap: 0.05394                    };395                    plotly.plot(data, layout, function (err, msg) {396                        if (err) {397                            logger.error('Error in plotting graph for number of clusters: ' + JSON.stringify(err));398                            //go to the next function399                            callback(null, totalTweets, allMatchesUnique);400                        } else {401                            //log the output to logger and final output file402                            logger.info('Message to check the graph for the size of clusters formed by Minhash LSH: ' + JSON.stringify(msg));403                            output.info('Link to check the graph for the size of clusters formed by Minhash LSH: ' + JSON.stringify(msg.url));404                            //go to the next function405                            callback(null, totalTweets, allMatchesUnique);406                        }407                    });408                    // callback(null, totalTweets, allMatchesUnique);409                });410        },411        //8. Funciton to find geo-tagged and user profile geo information412        function (totalTweets, allMatchesUnique, callback) {413            //loop through all unique clusters414            async.forEachSeries(allMatchesUnique, function (eachMatch, callback) {415                    //loop through each match cluster entry416                    async.forEachSeries(eachMatch, function (singleEntry, callback) {417                            //split the string418                            var splitSingle = singleEntry.split('-');419                            //check if place is null, if not then push '1' to the array420                            if (!(splitSingle[1] == 'null')) {421                                eachNewInternalArrayPlace.push(1);422                            }423                            //check if geo information is true or not, if so, then push '1' to the array424                            if (splitSingle[2] == 'true') {425                                eachNewInternalArrayGeoEnabled.push(1);426                            }427                            //go to the next entry of matched cluster entry (inner loop)428                            callback();429                        },430                        //final function for internal loop431                        function (errSingle) {432                            if (errSingle) {433                                //log the error434                                logger.error('Error found in async of inner loop: ' + JSON.stringify(errSingle));435                                callback();436                            } else {437                                //if any data found438                                if (eachNewInternalArrayPlace.length != 0) {439                                    //create place data440                                    newArrayPlace.push(JSON.stringify(eachNewInternalArrayPlace));441                                }442                                //if any data found443                                if (eachNewInternalArrayGeoEnabled.length != 0) {444                                    //create user geo information data445                                    newArrayGeoEnabled.push(JSON.stringify(eachNewInternalArrayPlace));446                                }447                                //empty contents of array for next iteration448                                eachNewInternalArrayPlace.splice(0, eachNewInternalArrayPlace.length);449                                eachNewInternalArrayGeoEnabled.splice(0, eachNewInternalArrayGeoEnabled.length);450                                //go to the next iteration of outer loop451                                callback();452                            }453                        });454                },455                //final function for outer loop456                function (errFinal) {457                    if (errFinal) {458                        //log the error459                        logger.error('Error in outer async of function 8: ' + errFinal);460                        //going to next function to find geo-tagged and user profile geo information461                        callback(null, totalTweets, allMatchesUnique, newArrayPlace, newArrayGeoEnabled);462                    } else {463                        logger.info('Going to next function to find geo-tagged and user profile geo information');464                        //going to next function to find geo-tagged and user profile geo information465                        callback(null, totalTweets, allMatchesUnique, newArrayPlace, newArrayGeoEnabled);466                    }467                });468        },469        //9. Function to build graphs for geo-tagged data470        function (totalTweets, allMatchesUnique, newArrayPlace, newArrayGeoEnabled, callback) {471            var one = [],472                two = [],473                threeTo4 = [],474                fiveTo9 = [],475                tenTo19 = [],476                twentyTo49 = [],477                moreThan50 = [];478            async.forEachSeries(newArrayPlace, function (eachPlace, callback) {479                    if (eachPlace.length >= 1 && eachPlace.length < 2) {480                        one.push(eachPlace);481                        callback();482                    } else if (eachPlace.length >= 2 && eachPlace.length < 3) {483                        two.push(eachPlace);484                        callback();485                    } else if (eachPlace.length >= 3 && eachPlace.length < 5) {486                        threeTo4.push(eachPlace);487                        callback();488                    } else if (eachPlace.length >= 5 && eachPlace.length < 10) {489                        fiveTo9.push(eachPlace);490                        callback();491                    } else if (eachPlace.length >= 10 && eachPlace.length < 20) {492                        tenTo19.push(eachPlace);493                        callback();494                    } else if (eachPlace.length >= 20 && eachPlace.length < 50) {495                        twentyTo49.push(eachPlace);496                        callback();497                    } else {498                        moreThan50.push(eachPlace);499                        callback();500                    }501                },502                function (err) {503                    var xVal = ['1', '2', '3-4', '5-9', '10-19', '20-49', 'More than equal to 50'];504                    var yVal = [one.length, two.length, threeTo4.length, fiveTo9.length, tenTo19.length, twentyTo49.length, moreThan50.length];505                    var data = [{506                        x: xVal,507                        y: yVal,508                        type: 'bar',509                        text: yVal.map(String),510                        textposition: 'auto',511                        hoverinfo: 'none',512                        marker: {513                            color: 'rgb(158,202,225)',514                            opacity: 0.6,515                            line: {516                                color: 'rgb(8,48,107)',517                                width: 1.5518                            }519                        }520                    }];521                    var layout = {522                        title: "Number of geo-tagged clusters across different ranges in Minhash LSH Clustering",523                        fileopt: "overwrite",524                        filename: "basic-bar",525                        font: {526                            family: "Raleway, sans-serif"527                        },528                        bargap: 0.05529                    };530                    plotly.plot(data, layout, function (err, msg) {531                        if (err) {532                            logger.error('Error in plotting graph for number of geo-tagged clusters in total clusters: ' + JSON.stringify(err));533                            //go to the next function534                            callback(null, totalTweets, allMatchesUnique, newArrayGeoEnabled);535                        } else {536                            //log the output to logger and final output file537                            logger.info('Message to check the graph for the number of geo-tagged clusters in cluster formed by Minhash LSH: ' + JSON.stringify(msg));538                            output.info('Link to check the graph for the number of geo-tagged clusters in cluster formed by Minhash LSH: ' + JSON.stringify(msg.url));539                            //go to the next function540                            callback(null, totalTweets, allMatchesUnique, newArrayGeoEnabled);541                        }542                    });543                    // callback(null, totalTweets, allMatchesUnique, newArrayGeoEnabled);544                });545        },546        //10. Function to build graph for user profile geo information547        function (totalTweets, allMatchesUnique, newArrayGeoEnabled, callback) {548            var one = [],549                two = [],550                threeTo4 = [],551                fiveTo9 = [],552                tenTo19 = [],553                twentyTo49 = [],554                moreThan50 = [];555            async.forEachSeries(newArrayGeoEnabled, function (eachUserGeoInfo, callback) {556                    if (eachUserGeoInfo.length >= 1 && eachUserGeoInfo.length < 2) {557                        one.push(eachUserGeoInfo);558                        callback();559                    } else if (eachUserGeoInfo.length >= 2 && eachUserGeoInfo.length < 3) {560                        two.push(eachUserGeoInfo);561                        callback();562                    } else if (eachUserGeoInfo.length >= 3 && eachUserGeoInfo.length < 5) {563                        threeTo4.push(eachUserGeoInfo);564                        callback();565                    } else if (eachUserGeoInfo.length >= 5 && eachUserGeoInfo.length < 10) {566                        fiveTo9.push(eachUserGeoInfo);567                        callback();568                    } else if (eachUserGeoInfo.length >= 10 && eachUserGeoInfo.length < 20) {569                        tenTo19.push(eachUserGeoInfo);570                        callback();571                    } else if (eachUserGeoInfo.length >= 20 && eachUserGeoInfo.length < 50) {572                        twentyTo49.push(eachUserGeoInfo);573                        callback();574                    } else {575                        moreThan50.push(eachUserGeoInfo);576                        callback();577                    }578                },579                function (err) {580                    var xVal = ['1', '2', '3-4', '5-9', '10-19', '20-49', 'More than equal to 50'];581                    var yVal = [one.length, two.length, threeTo4.length, fiveTo9.length, tenTo19.length, twentyTo49.length, moreThan50.length];582                    var data = [{583                        x: xVal,584                        y: yVal,585                        type: 'bar',586                        text: yVal.map(String),587                        textposition: 'auto',588                        hoverinfo: 'none',589                        marker: {590                            color: 'rgb(158,202,225)',591                            opacity: 0.6,592                            line: {593                                color: 'rgb(8,48,107)',594                                width: 1.5595                            }596                        }597                    }];598                    var layout = {599                        title: "Number of user profile based geo information across different ranges in Minhash LSH Clustering",600                        fileopt: "overwrite",601                        filename: "bar-direct-labels",602                        font: {603                            family: "Raleway, sans-serif"604                        },605                        bargap: 0.05606                    };607                    plotly.plot(data, layout, function (err, msg) {608                        if (err) {609                            logger.error('Error in plotting graph for user profile based geo information in total clusters: ' + JSON.stringify(err));610                            //go to the next function611                            callback(null, totalTweets, allMatchesUnique);612                        } else {613                            //log the output to logger and final output file614                            logger.info('Message to check the graph for user profile based geo information in cluster formed by Minhash LSH: ' + JSON.stringify(msg));615                            output.info('Link to check the graph for user profile based geo information in cluster formed by Minhash LSH: ' + JSON.stringify(msg.url));616                            //go to the next function617                            callback(null, totalTweets, allMatchesUnique);618                        }619                    });620                    // callback(null, totalTweets, allMatchesUnique);621                });622        },623        //11. Function to assign each tweet a location inside the cluster624        function (totalTweets, allMatchesUnique, callback) {625            //loop through all unique clusters626            async.forEachSeries(allMatchesUnique, function (eachMatch, callback) {627                    var eachNewInternalArrayLocation = [];628                    //loop through each match cluster entry629                    async.forEachSeries(eachMatch, function (singleEntry, callback) {630                            //split the string631                            var splitSingle = singleEntry.split('-');632                            //push the location of the tweet to the array633                            eachNewInternalArrayLocation.push(splitSingle[1]);634                            callback();635                        },636                        //final function for internal loop637                        function (errSingle) {638                            if (errSingle) {639                                //log the error640                                logger.error('Error found in async of inner loop: ' + JSON.stringify(errSingle));641                                callback();642                            } else {643                                // logger.info('Location Cluster: ' + JSON.stringify(eachNewInternalArrayLocation));644                                //create tweet location data645                                newArrayLocation.push(eachNewInternalArrayLocation);646                                //go to the next iteration of outer loop647                                callback();648                            }649                        });650                },651                //final function for outer loop652                function (errFinal) {653                    if (errFinal) {654                        //log the error655                        logger.error('Error in outer async of function 11: ' + JSON.stringify(errFinal));656                        //going to next function to assign location to the clusters formed using Minhash LSH657                        callback(null, totalTweets, allMatchesUnique, newArrayLocation);658                    } else {659                        logger.info('Going to next function to assign location to the clusters formed using Minhash LSH');660                        //going to next function to assign location to the clusters formed using Minhash LSH661                        callback(null, totalTweets, allMatchesUnique, newArrayLocation);662                    }663                });664        },665        // 12. Function to assign location to the clusters formed using Minhash LSH666        function (totalTweets, allMatchesUnique, newArrayLocation, callback) {667            //variable to store glasgow cluster count, null cluster count668            var glasgowClusterCount = 0, noLocationClusterCount = 0;669            //for each location cluster array, check if occurence of glasgow is more than 50% of the total elements of the array670            async.forEachSeries(newArrayLocation, function(eachLocationCluster, callback) {671                //loop through each location cluster elements to check if it has glasgow672                async.forEachSeries(eachLocationCluster, function (eachLocation, callback) {673                    //check if the location in current cluster is Glasgow or not674                    if (eachLocation == 'Glasgow') {675                        //if glasgow location increment count by one676                        glasgowClusterCount += 1;677                        //go to next location in the cluster678                        callback();679                    } else if (eachLocation == 'null') { //if no location is present in the cluster680                        noLocationClusterCount += 1;681                        //go to next location in cluster682                        callback();683                    } else {684                        //if location is not glasgow, go to next location685                        callback();686                    }687                },688                //final function for internal loop689                function (errSingle) {690                    if (errSingle) {691                        //log the error692                        logger.error('Error found in async of inner loop in function 12: ' + JSON.stringify(errSingle));693                        callback();694                    } else {695                        //if more than half of the tweets have glasgow as location, then 696                        //assign the cluster location as glasgow697                        if (glasgowClusterCount >= eachLocationCluster.length/2) {698                            //if so, then increment the total glasgow clusters699                            totalGlasgowCLusters = totalGlasgowCLusters + 1;700                            //make the count to zero for next cluster iteration701                            glasgowClusterCount = 0;702                            noLocationClusterCount = 0;703                            //go to next iteration704                            callback();705                        } else if (noLocationClusterCount == eachLocationCluster.length) {706                            //if this condition matches, then cluster has no location information at all707                            //hence increment appropriate count708                            totalNullLocationClusters = totalNullLocationClusters + 1;709                            //make the count to zero for next cluster iteration710                            noLocationClusterCount = 0;711                            glasgowClusterCount = 0;712                            //go to next iteration713                            callback();714                        } else if ((glasgowClusterCount != 0) && (glasgowClusterCount + noLocationClusterCount == eachLocationCluster.length)) {715                            //if this condition matches then cluster has location as glasgow but no other location tweet is present716                            totalGlasgowCLusters = totalGlasgowCLusters + 1717                            //make the count to zero for next cluster iteration718                            glasgowClusterCount = 0;719                            noLocationClusterCount = 0;720                            //go to next iteration721                            callback();722                        } else {723                            //if this condition matches then cluster has location information724                            //but it is not a glasgow cluster725                            totalNotGlasgowClusters = totalNotGlasgowClusters + 1726                            //make the count to zero for next cluster iteration727                            glasgowClusterCount = 0;728                            noLocationClusterCount = 0;729                            //go to next iteration730                            callback();731                        }732                    }733                });734            },735            //final function for 736            function(errFinal) {737                if (errFinal) {738                    //log the error739                    logger.error('Error in outer async of function 12: ' + JSON.stringify(errFinal));740                    //going to next function741                    callback(null, totalTweets, allMatchesUnique, totalGlasgowCLusters, totalNotGlasgowClusters, totalNullLocationClusters);742                } else {743                    logger.info('Going to next function 13');744                    //going to next function745                    callback(null, totalTweets, allMatchesUnique, totalGlasgowCLusters, totalNotGlasgowClusters, totalNullLocationClusters);746                }747            });748        },749        //13. Function to plot the graph for the glasgow clusters, null location clusters and other location clusters750        function(totalTweets, allMatchesUnique, totalGlasgowCLusters, totalNotGlasgowClusters, totalNullLocationClusters, callback) {751            var xVal = ['Total Glasgow Clusters', 'Total Clusters not from Glasgow', 'Total cluster with no location'];752            var yVal = [totalGlasgowCLusters, totalNotGlasgowClusters, totalNullLocationClusters];753            var data = [{754                x: xVal,755                y: yVal,756                type: 'bar',757                text: yVal.map(String),758                textposition: 'auto',759                hoverinfo: 'none',760                marker: {761                    color: 'rgb(158,202,225)',762                    opacity: 0.6,763                    line: {764                        color: 'rgb(8,48,107)',765                        width: 1.5766                    }767                }768            }];769            var layout = {770                title: "Assigning Geo-Location to All Clusters",771                fileopt: "overwrite",772                filename: "simple-node-example",773                font: {774                    family: "Raleway, sans-serif"775                },776                bargap: 0.05777            };778            plotly2.plot(data, layout, function (err, msg) {779                if (err) {780                    logger.error('Error in plotting graph for Assigning Geo-Location to All Clusters: ' + JSON.stringify(err));781                    //go to the next function782                    callback(null, totalTweets, allMatchesUnique, totalGlasgowCLusters);783                } else {784                    //log the output to logger and final output file785                    logger.info('Message to check the graph for Assigning Geo-Location to All Clusters: ' + JSON.stringify(msg));786                    output.info('Link to check the graph for Assigning Geo-Location to All Clusters: ' + JSON.stringify(msg.url));787                    //go to the next function788                    callback(null, totalTweets, allMatchesUnique, totalGlasgowCLusters);789                }790            });791            // callback(null, totalTweets, allMatchesUnique, totalGlasgowCLusters);792        },793        //14. Function to get the country codes of all tweets in the clusters794        function(totalTweets, allMatchesUnique, totalGlasgowCLusters, callback) {795            //loop through all unique clusters796            async.forEachSeries(allMatchesUnique, function (eachMatch, callback) {797                var eachNewInternalArrayCountry = [];798                //loop through each match cluster entry799                async.forEachSeries(eachMatch, function (singleEntry, callback) {800                        //split the string801                        var splitSingle = singleEntry.split('-');802                        //push the country code of the tweet to the array803                        eachNewInternalArrayCountry.push(splitSingle[3]);804                        callback();805                    },806                    //final function for internal loop807                    function (errSingle) {808                        if (errSingle) {809                            //log the error810                            logger.error('Error found in async of inner loop: ' + JSON.stringify(errSingle));811                            callback();812                        } else {813                            //create tweet location cluster814                            newArrayCountry.push(eachNewInternalArrayCountry);815                            //go to the next iteration of outer loop816                            callback();817                        }818                    });819            },820            //final function for outer loop821            function (errFinal) {822                if (errFinal) {823                    //log the error824                    logger.error('Error in outer async of function 14: ' + JSON.stringify(errFinal));825                    //going to next function to assign country code to the clusters formed using Minhash LSH826                    callback(null, totalTweets, allMatchesUnique, newArrayCountry, totalGlasgowCLusters);827                } else {828                    logger.info('Going to next function to assign country code to the clusters formed using Minhash LSH');829                    //going to next function to assign country code to the clusters formed using Minhash LSH830                    callback(null, totalTweets, allMatchesUnique, newArrayCountry, totalGlasgowCLusters);831                }832            });833        },834        //15. Function to get the number of clusters of United Kingdom835        function(totalTweets, allMatchesUnique, newArrayCountry, totalGlasgowCLusters, callback) {836            //variable to store UK cluster count, null cluster count;837            var ukClusterCount = 0, noLocationClusterCount = 0;838            //for each location cluster array, check if occurence of glasgow is more than 50% of the total elements of the array839            async.forEachSeries(newArrayCountry, function(eachCountryCluster, callback) {840                //loop through each location cluster elements to check if it has UK841                async.forEachSeries(eachCountryCluster, function (eachLocation, callback) {842                    //check if the country in current cluster is UK or not843                    if (eachLocation == 'GB') {844                        //if UK is country increment count by one845                        ukClusterCount = ukClusterCount + 1;846                        //go to next location in the cluster847                        callback();848                    } else if (eachLocation == 'null') { //if no location is present in the cluster849                        noLocationClusterCount = noLocationClusterCount + 1;850                        //go to next location in cluster851                        callback();852                    } else {853                        //if country is not UK, go to next location854                        callback();855                    }856                },857                //final function for internal loop858                function (errSingle) {859                    if (errSingle) {860                        //log the error861                        logger.error('Error found in async of inner loop in function 12: ' + JSON.stringify(errSingle));862                        callback();863                    } else {864                        //if more than half of the tweets have UK as country, then 865                        //assign the cluster location as glasgow866                        if (ukClusterCount >= eachCountryCluster.length/2) {867                            //if so, then increment the total glasgow clusters868                            totalUKClusters = totalUKClusters + 1;869                            //make the count to zero for next cluster iteration870                            ukClusterCount = 0;871                            noLocationClusterCount = 0;872                            //go to next iteration873                            callback();874                        } else if (ukClusterCount == eachCountryCluster.length) {875                            //if this condition matches, then cluster has no location information at all876                            //hence increment appropriate count877                            totalUKClusters = totalUKClusters + 1;878                            //make the count to zero for next cluster iteration879                            ukClusterCount = 0;880                            noLocationClusterCount = 0;881                            //go to next iteration882                            callback();883                        } else if ((ukClusterCount != 0) && (ukClusterCount + noLocationClusterCount == eachCountryCluster.length)) {884                            //if this condition matches then cluster has country as UK but no other country tweet is present885                            totalUKClusters = totalUKClusters + 1886                            //make the count to zero for next cluster iteration887                            ukClusterCount = 0;888                            noLocationClusterCount = 0;889                            //go to next iteration890                            callback();891                        } else { //if no above condition is true892                            //go to next iteration893                            callback();894                        }895                    }896                });897            },898            //final function for calculating UK based clusters899            function(errFinal) {900                if (errFinal) {901                    //log the error902                    logger.error('Error in outer async of function 15: ' + JSON.stringify(errFinal));903                    //going to next function904                    callback(null, totalTweets, allMatchesUnique, totalGlasgowCLusters, totalUKClusters);905                } else {906                    logger.info('Going to next function 16');907                    //going to next function908                    callback(null, totalTweets, allMatchesUnique, totalGlasgowCLusters, totalUKClusters);909                }910            });911        },912        //16. Function to plot graph for UK based clusters and Glasgow clusters913        function(totalTweets, allMatchesUnique, totalGlasgowCLusters, totalUKClusters, callback) {914            var xVal = ['Total Glasgow Clusters', 'Total UK cluster'];915            var yVal = [totalGlasgowCLusters, totalUKClusters];916            var data = [{917                x: xVal,918                y: yVal,919                type: 'bar',920                text: yVal.map(String),921                textposition: 'auto',922                hoverinfo: 'none',923                marker: {924                    color: 'rgb(158,202,225)',925                    opacity: 0.6,926                    line: {927                        color: 'rgb(8,48,107)',928                        width: 1.5929                    }930                }931            }];932            var layout = {933                title: "Evaluation of Method correctness of Geo Location Assignment",934                fileopt: "overwrite",935                filename: "basic-bar",936                font: {937                    family: "Raleway, sans-serif"938                },939                bargap: 0.05940            };941            plotly2.plot(data, layout, function (err, msg) {942                if (err) {943                    logger.error('Error in plotting graph for Evaluation of Method correctness of Geo Location Assignment: ' + JSON.stringify(err));944                    //go to the next function945                    callback(null, totalTweets, allMatchesUnique);946                } else {947                    //log the output to logger and final output file948                    logger.info('Message to check the graph for Evaluation of Method correctness of Geo Location Assignment: ' + JSON.stringify(msg));949                    output.info('Link to check the graph for Evaluation of Method correctness of Geo Location Assignment: ' + JSON.stringify(msg.url));950                    //go to the next function951                    callback(null, totalTweets, allMatchesUnique);952                }953            });954            // callback(null, totalTweets, allMatchesUnique);955        }956    ], function (err, result) {957        logger.info('End of the clustering analysis for all tweets saved in the database');958        //close the program after 5 seconds959        setTimeout((function() {  960            return process.exit(22);961        }), 5000);962    })...

Remove the noise from the string.js

Source:Remove the noise from the string.js

1// Description:2// We have a broken message machine that introduces noise to our incoming messages. We know that our messages are written using a-zA-Z0-9, white characters (like spaces or tabs) and some punctuation chars. Unfortunately our machine introduces noise, which means that our message arrives with characters like: %$&/#Â·@|Âº\Âª within our original message (noise can have only %$&/#Â·@|Âº\Âª chars).3// Your mission is to write a function which removes this noise from the message.4// For example:5// removeNoise("h%e&Â·%$Â·llo w&%or&$lÂ·$%d")6// // returns hello world7// mine 8function removeNoise(str){9  return str.replace(/[\%\$\&\/\#\Â·\@\|\Âº\\\Âª]*/ig,'')10}11// top vote solution12var str;13function removeNoise(str){14  return str.replace(/[%$&\/#Â·@|Âº\\Âª]/g, "");...

removeNoise.test.ts

Source:removeNoise.test.ts

1import removeNoise from "./removeNoise";2test("Removes semicolons", ()=>{3    expect(removeNoise("from db;")).toBe("from db");4});5test("Removes ending whitespace", ()=>{6    expect(removeNoise("text      ")).toBe("text");7});8test("Removes multiline comments", ()=>{9    expect(removeNoise("text/**/")).toBe("text");10});11test("Removes single line comments", ()=>{12    expect(removeNoise("text-- comment")).toBe("text");13});14test("Removes newlines, replaces with space", ()=>{15    expect(removeNoise("text\nmore text")).toBe("text more text");16});17test("Removes tabs, replaces with space", ()=>{18    expect(removeNoise("text\tmore text")).toBe("text more text");19});20test("Removes doubled whitespace", ()=>{21    expect(removeNoise("text     more      text")).toBe("text more text");...

Using AI Code Generation

1var mb = require('mountebank');2var imposter = {3        {4                {5                    is: {6                        headers: {7                        },8                    }9                }10        }11};12mb.create(imposter).then(function (result) {13    console.log('Imposter created at: ' + result.url);14    return mb.removeNoise();15}).then(function () {16    console.log('Removed noise');17}).catch(function (error) {18    console.error(error);19});20mb.removeNoise({ port: 3000 });21mb.removeNoise({ port: 3000, protocol: 'http' });22var mb = require('mountebank');23var imposter = {24        {25                {26                    is: {27                        headers: {28                        },29                    }30                }31        }

Using AI Code Generation

1const mb = require('mountebank');2const imposter = {3        {4                {5                        {6                            equals: {7                            }8                        }9                }10                {11                    is: {12                        headers: {13                        },14                        body: JSON.stringify({ message: 'Hello World!' })15                    }16                }17        }18};19mb.create(imposter).then(function() {20    console.log('Imposter created');21}, function(error) {22    console.error('Error creating imposter', error);23});24Error creating imposter { Error: connect ECONNREFUSED

Using AI Code Generation

1var request = require('request');2var options = {3  json: {4      {5          {6            "is": {7              "headers": {8              },9              "body": "{\"message\":\"Hello World\"}"10            }11          }12      }13  }14};15request(options, function (error, response, body) {16  if (!error && response.statusCode == 200) {17  }18});19var request = require('request');20var options = {21};22request(options, function (error, response, body) {23  if (!error && response.statusCode == 200) {24  }25});26var request = require('request');27var options = {28};29request(options, function (error, response, body) {30  if (!error && response.statusCode == 200) {31  }32});33var request = require('request');34var options = {35  json: {36      {37          {38            "is": {39              "headers": {40              },41              "body": "{\"message\":\"Hello World\"}"42            }43          }44      }45  }46};47request(options, function (error, response, body) {48  if (!error && response.statusCode == 200) {49  }50});

Using AI Code Generation

1const mb = require('mountebank');2mb.create({port: 2525, pidfile: 'mb.pid', logfile: 'mb.log', ipWhitelist: ['*']}, function() {3    mb.removeImposters(function() {4        console.log('Imposters deleted');5    });6});7const mb = require('mountebank');8mb.create({port: 2525, pidfile: 'mb.pid', logfile: 'mb.log', ipWhitelist: ['*']}, function() {9    mb.removeImposters(function() {10        console.log('Imposters deleted');11    });12});13const mb = require('mountebank');14mb.create({port: 2525, pidfile: 'mb.pid', logfile: 'mb.log', ipWhitelist: ['*']}, function() {15    mb.removeImposters(function() {16        console.log('Imposters deleted');17    });18});19const mb = require('mountebank');20mb.create({port: 2525, pidfile: 'mb.pid', logfile: 'mb.log', ipWhitelist: ['*']}, function() {21    mb.removeImposters(function() {22        console.log('Imposters deleted');23    });24});25const mb = require('mountebank');26mb.create({port: 2525, pidfile: 'mb.pid', logfile: 'mb.log', ipWhitelist: ['*']}, function() {27    mb.removeImposters(function() {28        console.log('Imposters deleted');29    });30});31const mb = require('mountebank');32mb.create({port: 2525, pidfile: 'mb.pid', logfile: 'mb.log', ipWhitelist: ['*']}, function() {33    mb.removeImposters(function() {34        console.log('Imposters deleted');35    });36});37const mb = require('mountebank');38mb.create({port: 2525, pidfile: 'mb.pid', logfile: 'mb.log', ipWhitelist: ['*']}, function() {

Using AI Code Generation

1var mb = require('mountebank');2var options = {3};4mb.create(options, function (error, mb) {5  console.log(mb.port);6  mb.removeImposters(function (error) {7    console.log('imposters removed');8  });9});

Using AI Code Generation

1var mb = require('mountebank');2var removeNoise = mb.create({ port: 2525, pidfile: 'mb.pid' });3removeNoise.removeNoise();4removeNoise.start();5var mb = require('mountebank');6var removeNoise = mb.create({ port: 2525, pidfile: 'mb.pid' });7removeNoise.removeNoise();8removeNoise.start();9var mb = require('mountebank');10var removeNoise = mb.create({ port: 2525, pidfile: 'mb.pid' });11removeNoise.removeNoise();12removeNoise.start();13var mb = require('mountebank');14var removeNoise = mb.create({ port: 2525, pidfile: 'mb.pid' });15removeNoise.removeNoise();16removeNoise.start();17var mb = require('mountebank');18var removeNoise = mb.create({ port: 2525, pidfile: 'mb.pid' });19removeNoise.removeNoise();20removeNoise.start();21var mb = require('mountebank');22var removeNoise = mb.create({ port: 2525, pidfile: 'mb.pid' });23removeNoise.removeNoise();24removeNoise.start();25var mb = require('mountebank');26var removeNoise = mb.create({ port: 2525, pidfile: 'mb.pid' });27removeNoise.removeNoise();28removeNoise.start();29var mb = require('mountebank');30var removeNoise = mb.create({ port: 2525, pidfile: 'mb.pid' });31removeNoise.removeNoise();32removeNoise.start();33var mb = require('mountebank');34var removeNoise = mb.create({ port: 2525, pidfile: 'mb.pid' });35removeNoise.removeNoise();36removeNoise.start();37var mb = require('mountebank');38var removeNoise = mb.create({ port: 2525, pidfile: 'mb.pid' });39removeNoise.removeNoise();40removeNoise.start();

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.