Unlock 30% off on Manual Testing Annual Plans this Holiday Season.

Copied to Clipboard!

How to use moo method in Nose

Best Python code snippet using nose

test.py

Source:test.py

1# -*- encoding: utf-8 -*-2###3# Copyright (c) 2003-2005, Daniel DiPaolo4# All rights reserved.5#6# Redistribution and use in source and binary forms, with or without7# modification, are permitted provided that the following conditions are met:8#9#   * Redistributions of source code must retain the above copyright notice,10#     this list of conditions, and the following disclaimer.11#   * Redistributions in binary form must reproduce the above copyright notice,12#     this list of conditions, and the following disclaimer in the13#     documentation and/or other materials provided with the distribution.14#   * Neither the name of the author of this software nor the name of15#     contributors to this software may be used to endorse or promote products16#     derived from this software without specific prior written consent.17#18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"19# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE20# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE21# ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE22# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR23# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF24# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS25# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN26# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)27# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE28# POSSIBILITY OF SUCH DAMAGE.29###30from supybot.test import *31#import supybot.plugin as plugin32import supybot.ircutils as ircutils33try:34    import sqlite35except ImportError:36    sqlite = None37MF = plugin.loadPluginModule('MoobotFactoids')38MFconf = conf.supybot.plugins.MoobotFactoids39class OptionListTestCase(SupyTestCase):40    maxIterations = 26741    def _testOptions(self, s, L):42        max = self.maxIterations43        original = L[:]44        while max and L:45            max -= 146            option = MF.plugin.pickOptions(s)47            self.failUnless(option in original,48                            'Option %s not in %s' % (option, original))49            if option in L:50                L.remove(option)51        self.failIf(L, 'Some options never seen: %s' % L)52    def testPickOptions(self):53        self._testOptions('(a|b)', ['a', 'b'])54        self._testOptions('a', ['a'])55        self._testOptions('(a|b (c|d))', ['a', 'b c', 'b d'])56        self._testOptions('(a|(b|)c)', ['a', 'bc', 'c'])57        self._testOptions('(a(b|)|(c|)d)', ['a', 'ab', 'cd', 'd'])58        self._testOptions('(a|)', ['a', ''])59        self._testOptions('(|a)', ['a', ''])60        self._testOptions('((a)|(b))', ['(a)', '(b)'])61        self._testOptions('^\\%(\\%(foo\\)\\@<!.\\)*$',62                          ['^\\%(\\%(foo\\)\\@<!.\\)*$'])63class FactoidsTestCase(ChannelPluginTestCase):64    plugins = ('MoobotFactoids', 'User', 'String', 'Utilities', 'Web')65    config = {'reply.whenNotCommand': False}66    def setUp(self):67        ChannelPluginTestCase.setUp(self)68        # Create a valid user to use69        self.prefix = 'mf!bar@baz'70        self.irc.feedMsg(ircmsgs.privmsg(self.nick, 'register tester moo',71                                         prefix=self.prefix))72        m = self.irc.takeMsg() # Response to register.73    def testAddFactoid(self):74        self.assertNotError('moo is foo')75        # Check stripping punctuation76        self.assertError('moo!?    is foo') # 'moo' already exists77        self.assertNotError('foo!?    is foo')78        self.assertResponse('foo', 'foo is foo')79        self.assertNotError('bar is <reply>moo is moo')80        self.assertResponse('bar', 'moo is moo')81        # Check substitution82        self.assertNotError('who is <reply>$who')83        self.assertResponse('who', ircutils.nickFromHostmask(self.prefix))84        # Check that actions ("\x01ACTION...") don't match85        m = ircmsgs.action(self.channel, 'is doing something')86        self.irc.feedMsg(m)87        self.assertNoResponse(' ', 1)88    def testLiteral(self):89        self.assertError('literal moo') # no factoids yet90        self.assertNotError('moo is <reply>foo')91        self.assertResponse('literal moo', '<reply>foo')92        self.assertNotError('moo2 is moo!')93        self.assertResponse('literal moo2', 'moo!')94        self.assertNotError('moo3 is <action>foo')95        self.assertResponse('literal moo3', '<action>foo')96    def testGetFactoid(self):97        self.assertNotError('moo is <reply>foo')98        self.assertResponse('moo', 'foo')99        self.assertNotError('moo2 is moo!')100        self.assertResponse('moo2', 'moo2 is moo!')101        self.assertNotError('moo3 is <action>foo')102        self.assertAction('moo3', 'foo')103        # Test and make sure it's parsing104        self.assertNotError('moo4 is <reply>(1|2|3)')105        self.assertRegexp('moo4', '^(1|2|3)$')106        # Check case-insensitivity107        self.assertResponse('MOO', 'foo')108        self.assertResponse('mOo', 'foo')109        self.assertResponse('MoO', 'foo')110        # Check the "_is_" ability111        self.assertNotError('remove moo')112        self.assertNotError('moo _is_ <reply>foo')113        self.assertResponse('moo', 'foo')114        self.assertNotError('foo is bar _is_ baz')115        self.assertResponse('foo is bar', 'foo is bar is baz')116    def testFactinfo(self):117        self.assertNotError('moo is <reply>foo')118        self.assertRegexp('factinfo moo', '^moo: Created by tester on.*$')119        self.assertNotError('moo')120        self.assertRegexp('factinfo moo', self.prefix + '.*1 time')121        self.assertNotError('moo')122        self.assertRegexp('factinfo moo', self.prefix + '.*2 times')123        self.assertNotError('moo =~ s/foo/bar/')124        self.assertRegexp('factinfo moo',125                          '^moo: Created by tester on'126                          '.*?\. Last modified by tester on .*?\. '127                          'Last requested by %s on .*?, '128                          'requested 2 times.$' % self.prefix)129        self.assertNotError('lock moo')130        self.assertRegexp('factinfo moo',131                          '^moo: Created by tester on'132                          '.*?\. Last modified by tester on .*?\. '133                          'Last requested by %s on .*?, '134                          'requested 2 times. '135                          'Locked by tester on .*\.$' % self.prefix)136        self.assertNotError('unlock moo')137        self.assertRegexp('factinfo moo',138                          '^moo: Created by tester on'139                          '.*?\. Last modified by tester on .*?\. '140                          'Last requested by %s on .*?, '141                          'requested 2 times.$' % self.prefix)142        # Make sure I solved this bug143        # Check and make sure all the other stuff is reset144        self.assertNotError('foo is bar')145        self.assertNotError('foo =~ s/bar/blah/')146        self.assertNotError('foo')147        self.assertNotError('no foo is baz')148        self.assertRegexp('factinfo foo',149                          '^foo: Created by tester on'150                          '(?!(request|modif)).*?\.$')151    def testLockUnlock(self):152        # disable world.testing since we want new users to not153        # magically be endowed with the admin capability154        try:155            world.testing = False156            self.assertNotError('moo is <reply>moo')157            self.assertNotError('lock moo')158            self.assertRegexp('factinfo moo',159                              '^moo: Created by tester on'160                              '.*?\. Locked by tester on .*?\.')161            # switch user162            original = self.prefix163            self.prefix = 'moo!moo@moo'164            self.assertNotError('register nottester moo', private=True)165            self.assertError('unlock moo')166            self.assertRegexp('factinfo moo',167                              '^moo: Created by tester on'168                              '.*?\. Locked by tester on .*?\.')169            # switch back170            self.prefix = original171            self.assertNotError('identify tester moo', private=True)172            self.assertNotError('unlock moo')173            self.assertRegexp('factinfo moo',174                              '^moo: Created by tester on.*?\.')175        finally:176            world.testing = True177    def testChangeFactoid(self):178        self.assertNotError('moo is <reply>moo')179        self.assertNotError('moo =~ s/moo/moos/')180        self.assertResponse('moo', 'moos')181        self.assertNotError('moo =~ s/reply/action/')182        self.assertAction('moo', 'moos')183        self.assertNotError('moo =~ s/moos/(moos|woofs)/')184        self.assertActionRegexp('moo', '^(moos|woofs)$')185        self.assertError('moo =~ s/moo/')186    def testMost(self):187        userPrefix1 = 'moo!bar@baz'; userNick1 = 'moo'188        userPrefix2 = 'boo!bar@baz'; userNick2 = 'boo'189        self.assertNotError('register %s bar' % userNick1,190                            frm=userPrefix1, private=True)191        self.assertNotError('register %s bar' % userNick2,192                            frm=userPrefix2, private=True)193        # Check an empty database194        self.assertError('most popular')195        self.assertError('most authored')196        self.assertError('most recent')197        # Check singularity response198        self.prefix = userPrefix1199        self.assertNotError('moogle is <reply>moo')200        self.assertError('most popular')201        self.assertResponse('most authored',202                            'Most prolific author: moo (1)')203        self.assertRegexp('most recent', "1 latest factoid:.*moogle")204        self.assertResponse('moogle', 'moo')205        self.assertRegexp('most popular',206                            "Top 1 requested factoid:.*moogle.*(1)")207        # Check plural response208        self.prefix = userPrefix2209        self.assertNotError('mogle is <reply>mo')210        self.assertRegexp('most authored',211                            'Most prolific authors:.*moo.*(1).*boo.*(1)')212        self.assertRegexp('most recent',213                            "2 latest factoids:.*mogle.*moogle.*")214        self.assertResponse('moogle', 'moo')215        self.assertRegexp('most popular',216                            "Top 1 requested factoid:.*moogle.*(2)")217        self.assertResponse('mogle', 'mo')218        self.assertRegexp('most popular',219                            "Top 2 requested factoids:.*"220                            "moogle.*(2).*mogle.*(1)")221        # Check most author ordering222        self.assertNotError('moo is <reply>oom')223        self.assertRegexp('most authored',224                            'Most prolific authors:.*boo.*(2).*moo.*(1)')225    def testListkeys(self):226        self.assertResponse('listkeys %', 'No keys matching "%" found.')227        self.assertNotError('moo is <reply>moo')228        # With this set, if only one key matches, it should respond with229        # the factoid230        orig = MFconf.showFactoidIfOnlyOneMatch()231        try:232            MFconf.showFactoidIfOnlyOneMatch.setValue(True)233            self.assertResponse('listkeys moo', 'moo')234            self.assertResponse('listkeys foo', 'No keys matching "foo" '235                                'found.')236            # Throw in a bunch more237            for i in range(10):238                self.assertNotError('moo%s is <reply>moo' % i)239            self.assertRegexp('listkeys moo',240                              '^Key search for "moo" '241                              '\(11 found\): ("moo\d*", )+and "moo9"$')242            self.assertNotError('foo is bar')243            self.assertRegexp('listkeys %',244                              '^Key search for "\%" '245                              '\(12 found\): "foo", ("moo\d*", )+and '246                              '"moo9"$')247            # Check quoting248            self.assertNotError('foo\' is bar')249            self.assertResponse('listkeys foo',250                                'Key search for "foo" '251                                '(2 found): "foo" and "foo\'"')252            # Check unicode stuff253            self.assertResponse('listkeys Ð', 'No keys matching "Ð" found.')254            self.assertNotError('ÐÐÐÐÐÐÐ is foo')255            self.assertNotError('ÐÐÐÐÐÐÐÐÐ is foo')256            self.assertResponse('listkeys Ð',257                                'Key search for "Ð" '258                                '(2 found): "ÐÐÐÐÐÐÐ" and "ÐÐÐÐÐÐÐÐÐ"')259        finally:260            MFconf.showFactoidIfOnlyOneMatch.setValue(orig)261    def testListvalues(self):262        self.assertNotError('moo is moo')263        self.assertResponse('listvalues moo',264                            'Value search for "moo" (1 found): "moo"')265    def testListauth(self):266        self.assertNotError('moo is <reply>moo')267        self.assertRegexp('listauth tester', 'tester.*\(1 found\):.*moo')268        self.assertError('listauth moo')269    def testRemove(self):270        self.assertNotError('moo is <reply>moo')271        self.assertNotError('lock moo')272        self.assertError('remove moo')273        self.assertNotError('unlock moo')274        self.assertNotError('remove moo')275    def testAugmentFactoid(self):276        self.assertNotError('moo is foo')277        self.assertNotError('moo is also bar')278        self.assertResponse('moo', 'moo is foo, or bar')279        self.assertNotError('moo is bar _is_ foo')280        self.assertNotError('moo is bar is also foo')281        self.assertResponse('moo is bar', 'moo is bar is foo, or foo')282    def testReplaceFactoid(self):283        self.assertNotError('moo is foo')284        self.assertNotError('no moo is bar')285        self.assertResponse('moo', 'moo is bar')286        self.assertNotError('no, moo is baz')287        self.assertResponse('moo', 'moo is baz')288        self.assertNotError('lock moo')289        self.assertError('no moo is qux')290        self.assertNotError('foo is bar _is_ foo')291        self.assertNotError('no foo is bar _is_ baz')292        self.assertResponse('foo is bar', 'foo is bar is baz')293    def testRegexpNotCalledIfAlreadyHandled(self):294        self.assertResponse('echo foo is bar', 'foo is bar')295        self.assertNoResponse(' ', 3)296    def testNoResponseToCtcp(self):297        self.assertNotError('foo is bar')298        self.assertResponse('foo', 'foo is bar')299        self.irc.feedMsg(ircmsgs.privmsg(self.irc.nick, '\x01VERSION\x01'))300        m = self.irc.takeMsg()301        self.failIf(m)302    def testAddFactoidNotCalledWithBadNestingSyntax(self):303        self.assertError('re s/Error:.*/jbm is a tard/ ]')304        self.assertNoResponse(' ', 3)305    def testConfigShowFactoidIfOnlyOneMatch(self):306        # man these are long307        MFconf = conf.supybot.plugins.MoobotFactoids308        self.assertNotError('foo is bar')309        # Default to saying the factoid value310        self.assertResponse('listkeys foo', 'foo is bar')311        # Check the False setting312        MFconf.showFactoidIfOnlyOneMatch.setValue(False)313        self.assertResponse('listkeys foo', 'Key search for "foo" '314                                            '(1 found): "foo"')315    def testRandom(self):316        self.assertNotError('foo is <reply>bar')317        self.assertNotError('bar is <reply>baz')318        self.assertRegexp('random', r'bar|baz')...

load_test_model_wmoo.py

Source:load_test_model_wmoo.py

1import os2import joblib3import numpy as np4import seaborn as sns5import tensorflow as tf6from algorithms.tft2.harness.train_test import compute_moo_q_loss7from timeseries.experiments.market.moo.utils.utils import get_loss_to_obj_function, rank_solutions, aggregate_qcd_qee8from timeseries.experiments.market.plot.plot import plot_2D_moo_results_equal_w9from timeseries.experiments.market.utils.filename import get_result_folder, quantiles_name, termination_name10from timeseries.experiments.market.utils.harness import load_predict_model, get_model_data_config11from timeseries.experiments.market.utils.results import post_process_results12from timeseries.experiments.utils.files import save_vars13from timeseries.plotly.plot import plot_4D14sns.set_theme('poster')15if __name__ == "__main__":16    # %%17    print("Num GPUs Available: ", len(tf.config.list_physical_devices('GPU')))18    print("Device Name: ", tf.test.gpu_device_name())19    print('TF eager execution: {}'.format(tf.executing_eagerly()))20    general_cfg = {'save_forecast': False,21                   'save_plot': False,22                   'use_all_data': True,23                   'use_moo_weights': True}24    results_cfg = {'formatter': 'snp',25                   'experiment_name': '60t_ema_q159',26                   'results': 'TFTModel_ES_ema_r_q159_NSGA2_g250_p100_s1_k2_wmoo'27                   }28    moo_result = joblib.load(os.path.join(get_result_folder(results_cfg), 'moo', results_cfg['results'] + '.z'))29    config, formatter, model_folder = get_model_data_config(moo_result['experiment_cfg'],30                                                            moo_result['model_cfg'],31                                                            moo_result['fixed_cfg'])32    experiment_cfg = moo_result['experiment_cfg']33    weights, quantiles_loss, eq_quantiles_loss = moo_result['X'], moo_result['F'], moo_result['eq_F']34    basename = '{}_{}_q{}_{}_{}_p{}_s{}_k{}_'.format(experiment_cfg['architecture'],35                                                     experiment_cfg['vars_definition'],36                                                     quantiles_name(moo_result['quantiles']),37                                                     moo_result['moo_method'],38                                                     termination_name(moo_result['algo_cfg']['termination']),39                                                     moo_result['algo_cfg']['pop_size'],40                                                     int(moo_result['algo_cfg']['use_sampling']),41                                                     quantiles_loss.shape[1],42                                                     )43    # %%44    selected_ix = None45    original_ix = np.argmin(np.sum(np.abs(quantiles_loss - moo_result['original_losses']), axis=1))46    filter_thold = 1.47    total_selected_error = 1.25  # for 2k48    sort_weights = [0.2, 0.2, 0.6, 0.2]  # for 4k49    camera_position = np.array([0.3, 1.25, .6]) * 1.550    xaxis_limit = filter_thold51    if quantiles_loss.shape[1] == 2:52        Fs_x_plot_masks = quantiles_loss[:, 0] < xaxis_limit53        selected_ix = np.argmin(np.abs(np.sum(eq_quantiles_loss[Fs_x_plot_masks, :], axis=1) - total_selected_error))54        quantiles_loss_2k = quantiles_loss55        eq_quantiles_loss_2k = eq_quantiles_loss56    elif quantiles_loss.shape[1] == 4:57        tholds = [filter_thold] * 458        mask = np.all(np.vstack([quantiles_loss[:, i] < thold for i, thold in enumerate(tholds)]).T, axis=1)59        filtered_quantiles = quantiles_loss[mask, :]60        ranked_ix = rank_solutions(filtered_quantiles, sort_weights)61        ranked_losses = filtered_quantiles[ranked_ix, :]62        selected_filtered_ix = ranked_ix[0]63        selected_ix = np.argmax(np.all(quantiles_loss == filtered_quantiles[selected_filtered_ix, :], axis=1))64        plot_ranked_losses = [x.reshape(1, -1) if len(x.shape) == 1 else x for x in ranked_losses]65        quantiles_loss_2k = aggregate_qcd_qee(quantiles_loss)66        eq_quantiles_loss_2k = aggregate_qcd_qee(eq_quantiles_loss)67    img_path = os.path.join(config.results_folder,68                            experiment_cfg['experiment_name'],69                            'img',70                            '{}{}_pf'.format(basename,71                                             '_ix_{}'.format(selected_ix) if general_cfg['use_moo_weights'] else '')),72    if quantiles_loss.shape[1] == 4:73        axis_labels = ['QCR lower bound', 'QER lower bound',74                       'QCR upper bound', 'QER upper bound']75        plot_4D(filtered_quantiles,76                color_col=3,77                ranked_F=plot_ranked_losses,78                original_point=moo_result['original_losses'],79                selected_point=quantiles_loss[selected_ix, :],80                save=general_cfg['save_plot'],81                axis_labels=axis_labels,82                file_path=img_path,83                label_scale=1,84                size=(1980, 1080),85                save_png=False,86                title='',87                camera_position=camera_position88                )89    plot_2D_moo_results_equal_w(quantiles_loss_2k, eq_quantiles_loss_2k,90                                selected_ixs=selected_ix if general_cfg['use_moo_weights'] else None,91                                save=general_cfg['save_plot'],92                                file_path=img_path,93                                original_ixs=original_ix,94                                figsize=(20, 15),95                                xaxis_limit=xaxis_limit,96                                title='MOO using {} for quantiles: {}'.format(moo_result['moo_method'],97                                                                      moo_result['quantiles']))98    print('original quantiles loss: {}'.format(moo_result['original_losses']))99    print('selected quantiles loss: {}'.format(quantiles_loss[selected_ix, :]))100    # %%101    # selected_weights = weights[selected_ix, :] if general_cfg['use_moo_weights'] else None102    # results, data = load_predict_model(use_gpu=True,103    #                                    architecture=experiment_cfg['architecture'],104    #                                    model_folder=model_folder,105    #                                    data_config=config.data_config,106    #                                    data_formatter=formatter,107    #                                    use_all_data=general_cfg['use_all_data'],108    #                                    last_layer_weights=selected_weights,109    #                                    exclude_p50=True)110    #111    # post_process_results(results, formatter, experiment_cfg, plot_=False)112    #113    # q_losses = compute_moo_q_loss(results['quantiles'], results['forecasts'])114    # obj = get_loss_to_obj_function(moo_result['loss_to_obj_type'])(q_losses)115    # print('objective space: {}'.format(obj))116    #117    # results['data'] = data118    # results['objective_space'] = obj119    # if general_cfg['save_forecast']:120    #     save_vars(results, os.path.join(config.results_folder,121    #                                     experiment_cfg['experiment_name'],122    #                                     'img',123    #                                     '{}{}{}pf'.format(basename,124    #                                                       'ix{}_'.format(selected_ix) if general_cfg[125    #                                                           'use_moo_weights'] else '',...

load_test_model_dual_wmoo.py

Source:load_test_model_dual_wmoo.py

1import os2import joblib3import numpy as np4import seaborn as sns5import tensorflow as tf6from algorithms.tft2.harness.train_test import compute_moo_q_loss7from timeseries.experiments.market.moo.utils.model import get_new_weights8from timeseries.experiments.market.moo.utils.utils import get_selected_ix9from timeseries.experiments.market.plot.plot import plot_2D_moo_dual_results, get_rect_risk, get_ixs_risk10from timeseries.experiments.market.utils.filename import get_result_folder, quantiles_name, termination_name, risk_name11from timeseries.experiments.market.utils.harness import load_predict_model, get_model_data_config12from timeseries.experiments.market.utils.results import post_process_results13from timeseries.experiments.utils.files import save_vars14sns.set_theme('poster')15if __name__ == "__main__":16    # %%17    print("Num GPUs Available: ", len(tf.config.list_physical_devices('GPU')))18    print("Device Name: ", tf.test.gpu_device_name())19    print('TF eager execution: {}'.format(tf.executing_eagerly()))20    general_cfg = {'save_forecast': False,21                   'save_plot': False,22                   'plot_title': False,23                   'use_all_data': True,24                   'use_moo_weights': False,25                   'plot_tolerance': False,26                   'manual_selection': True}27    results_cfg = {'formatter': 'snp',28                   'experiment_name': '60t_ema_q258',29                   'results': 'TFTModel_ES_ema_r_q258_NSGA2_g100_p100_s0_c1_eq0_dual_wmoo'30                   }31    moo_results = joblib.load(os.path.join(get_result_folder(results_cfg), 'moo', results_cfg['results'] + '.z'))32    config, formatter, model_folder = get_model_data_config(moo_results['lq']['experiment_cfg'],33                                                            moo_results['lq']['model_cfg'],34                                                            moo_results['lq']['fixed_cfg'])35    experiment_cfg = moo_results['lq']['experiment_cfg']36    # %%37    add_risk = 0.0538    xaxis_limit = 1.39    risk_selected = {'qcrl': 0.4,40                     'qcru': 0.15}41    limits_ixs = [get_ixs_risk(moo_result['F'], add_risk) for _, moo_result in moo_results.items()]42    manual_selected_ixs = {'lq': limits_ixs[0][1],43                           'uq': limits_ixs[1][0]}44    # manual_selected_ixs = {'lq': 68,45    #                        'uq': 76}46    labels, quantiles_losses, original_ixs, selected_ixs = [], [], [], []47    selected_weights, original_weights = {}, {}48    original_losses = []49    for bound, moo_result in moo_results.items():50        weights, quantiles_loss, eq_quantiles_loss = moo_result['X'], moo_result['F'], moo_result['eq_F']51        original_ix = np.argmin(np.sum(np.abs(quantiles_loss - moo_result['original_losses']), axis=1))52        original_losses.append(moo_result['original_losses'])53        if general_cfg['manual_selection']:54            selected_ix = manual_selected_ixs[bound]55        else:56            selected_ix = get_selected_ix(quantiles_loss, risk_selected, upper=bound == 'uq')57        moo_results[bound]['original_ix'] = original_ix58        moo_results[bound]['selected_ix'] = selected_ix59        labels.append('upper quantile' if bound == 'uq' else 'lower quantile')60        quantiles_losses.append(quantiles_loss)61        original_ixs.append(original_ix)62        selected_ixs.append(selected_ix)63        selected_weights[bound] = weights[selected_ix, :]64        original_weights[bound] = weights[original_ix, :]65    # Overwrite selected ixs if needed66    risk_lbl = ''67    if general_cfg['use_moo_weights']:68        if general_cfg['manual_selection']:69            risk_lbl = 'lix{}_uix{}'.format(selected_ixs[0], selected_ixs[1])70        else:71            risk_lbl = risk_name(risk_selected)72    basename = '{}_{}_q{}_{}_{}_p{}_s{}_{}_tol{}_'.format(experiment_cfg['architecture'],73                                                          experiment_cfg['vars_definition'],74                                                          quantiles_name(moo_results['lq']['quantiles']),75                                                          moo_results['lq']['moo_method'],76                                                          termination_name(77                                                              moo_results['lq']['algo_cfg']['termination']),78                                                          moo_results['lq']['algo_cfg']['pop_size'],79                                                          int(moo_results['lq']['algo_cfg']['use_sampling']),80                                                          risk_lbl,81                                                          int(add_risk * 100)82                                                          )83    img_path = os.path.join(config.results_folder,84                            experiment_cfg['experiment_name'],85                            'img',86                            'moo',87                            '{}pf'.format(basename))88    plot_2D_moo_dual_results(quantiles_losses,89                             selected_ixs=selected_ixs if general_cfg['use_moo_weights'] else None,90                             save=general_cfg['save_plot'],91                             file_path=img_path,92                             original_losses=original_losses,93                             # original_ixs=original_ixs,94                             figsize=(20, 15),95                             xaxis_limit=xaxis_limit,96                             col_titles=labels,97                             legend_labels=None,98                             add_risk=add_risk if general_cfg['plot_tolerance'] else None,99                             markersize=8,100                             plot_title=general_cfg['plot_title'],101                             title='MOO using {} for quantiles: {}'.format(moo_result['moo_method'],102                                                                           moo_result['quantiles']))103    # %%104    if general_cfg['save_forecast']:105        new_weights = get_new_weights(moo_results['lq']['original_weights'], selected_weights) if \106            general_cfg['use_moo_weights'] else None107        results, data = load_predict_model(use_gpu=True,108                                           architecture=experiment_cfg['architecture'],109                                           model_folder=model_folder,110                                           data_config=config.data_config,111                                           data_formatter=formatter,112                                           use_all_data=general_cfg['use_all_data'],113                                           last_layer_weights=new_weights)114        post_process_results(results, formatter, experiment_cfg, plot_=False)115        q_losses = compute_moo_q_loss(results['quantiles'], results['forecasts'])116        print('lower quantile risk: {} \nupper quantile risk: {}'.format(q_losses[0, :], q_losses[2, :]))117        results['data'] = data118        results['objective_space'] = q_losses119        save_vars(results, os.path.join(config.results_folder,120                                        experiment_cfg['experiment_name'],121                                        'moo',122                                        'selec_sols',123                                        '{}{}pred'.format(basename,...

compare_dual_fronts.py

Source:compare_dual_fronts.py

1import os2import joblib3import numpy as np4import seaborn as sns5from timeseries.experiments.market.plot.plot import plot_2D_moo_dual_results, plot_2D_moo_results_equal_w, \6    plot_2D_dual_moo_results_equal_w7from timeseries.experiments.market.utils.filename import get_result_folder8from timeseries.experiments.market.utils.results import compile_multiple_results_q9sns.set_theme('poster')10if __name__ == "__main__":11    # %%12    general_cfg = {'test_name': 'vary_quantiles',13                   'save_plot': False,14                   'plot_selected': False,15                   'plot_title': False,16                   'plot_tolerance': True17                   }18    results_cfg = {'formatter': 'snp'}19    weights_files = [20        ('60t_ema_q159', 'TFTModel_ES_ema_r_q159_NSGA2_g100_p100_s0_c1_eq0_dual_wmoo'),21        ('60t_ema_q258', 'TFTModel_ES_ema_r_q258_NSGA2_g100_p100_s0_c1_eq0_dual_wmoo'),22        ('60t_ema_q357', 'TFTModel_ES_ema_r_q357_NSGA2_g100_p100_s0_c1_eq0_dual_wmoo'),23    ]24    bounds = ['lq', 'uq']25    results_folder = get_result_folder(results_cfg)26    moo_results = [joblib.load(os.path.join(results_folder, file[0], 'moo', file[1]) + '.z') for file in weights_files]27    # legend_labels_suffix = ['s: {}, g:{}, p:{}'.format(int(moo_result['algo_cfg']['use_sampling']),28    #                                                    moo_result['algo_cfg']['termination'][1],29    #                                                    moo_result['algo_cfg']['pop_size']) for moo_result in moo_results]30    # legend_labels_suffix = ['q: {}'.format(moo_result['quantiles']) for moo_result in moo_results]31    # legend_labels = ['Algorithm: {}'.format(moo_result['lq']['moo_method']) for moo_result in moo_results]32    # experiment_labels = [experiment['lq']['moo_method'] for experiment in moo_results]33    experiment_labels = [34        'q: {}'.format('-'.join((np.array(experiment['lq']['quantiles']) * 10).astype(int).astype(str))) for experiment35        in moo_results]36    results = compile_multiple_results_q(moo_results, experiment_labels=experiment_labels)37    # %%38    xaxis_limit = 1.39    add_risk = 0.0540    qcr_risk_selected = [0.2, 0.2]41    risk_selected = [1.1, 1.1]42    quantiles_losses, eq_quantiles_losses, original_ixs, selected_ixs = [], [], [], []43    for i, bound in enumerate(bounds):44        x_masks = [moo_result[bound]['F'][:, 0] < xaxis_limit for moo_result in moo_results]45        quantiles_losses.append([moo_result[bound]['F'] for moo_result in moo_results])46        eq_quantiles_losses.append([moo_result[bound]['eq_F'] for moo_result in moo_results])47        original_ixs.append([np.argmin(np.sum(np.abs(moo_result[bound]['F'] - moo_result[bound]['original_losses']),48                                              axis=1)) for moo_result in moo_results])49        selected_ixs.append([np.argmin(np.abs(moo_result[bound]['eq_F'][:, 0] - qcr_risk_selected[i]))50                             for x_mask, moo_result in zip(x_masks, moo_results)])51        # selected_ixs.append([np.argmin(np.abs(np.sum(moo_result[bound]['eq_F'], axis=1)[x_mask] - risk_selected[i]))52        #                      for x_mask, moo_result in zip(x_masks, moo_results)])53    filename = '{}_{}_moo_results'.format(general_cfg['test_name'],54                                          moo_results[0]['lq']['experiment_cfg']['vars_definition'])55    # %%56    plot_2D_moo_dual_results(quantiles_losses,57                             save=general_cfg['save_plot'],58                             file_path=os.path.join(os.path.dirname(results_folder),59                                                    'img',60                                                    filename),61                             original_ixs=original_ixs,62                             selected_ixs=selected_ixs if general_cfg['plot_selected'] else None,63                             col_titles=['lower quantile', 'upper quantile'],64                             legend_labels=experiment_labels,65                             figsize=(15, 15),66                             add_risk=add_risk if general_cfg['plot_tolerance'] else None,67                             plot_title=general_cfg['plot_title'],68                             title='Multi objective optimization results',69                             xaxis_limit=xaxis_limit,70                             markersize=5)71    for i, bound in enumerate(['lower quantile', 'upper quantile']):72        risk_lbl = '_eqcr{}'.format(int(qcr_risk_selected[i] * 10)) if general_cfg['plot_selected'] else ''73        plot_2D_moo_results_equal_w(quantiles_losses[i], eq_quantiles_losses[i],74                                    save=general_cfg['save_plot'],75                                    file_path=os.path.join(os.path.dirname(results_folder),76                                                           'img',77                                                           '{}_{}{}'.format(filename,78                                                                            bounds[i],79                                                                            risk_lbl)),80                                    original_ixs=original_ixs[i],81                                    selected_ixs=selected_ixs[i] if general_cfg['plot_selected'] else None,82                                    legend_labels=experiment_labels,83                                    figsize=(15, 15),84                                    add_risk=add_risk if general_cfg['plot_tolerance'] else None,85                                    plot_title=general_cfg['plot_title'],86                                    title='Multi objective optimization results for {}'.format(bound),87                                    xaxis_limit=xaxis_limit,88                                    markersize=5)89    #%%90    plot_2D_dual_moo_results_equal_w(quantiles_losses, eq_quantiles_losses,91                                     save=True, #general_cfg['save_plot'],92                                     file_path=os.path.join(os.path.dirname(results_folder),93                                                            'img',94                                                            '{}_{}_eqW'.format(filename,95                                                                             risk_lbl)),96                                     original_ixs=original_ixs,97                                     selected_ixs=selected_ixs if general_cfg['plot_selected'] else None,98                                     legend_labels=experiment_labels,99                                     figsize=(15, 7),100                                     add_risk=add_risk if general_cfg['plot_tolerance'] else None,101                                     plot_title=general_cfg['plot_title'],102                                     title='Multi objective optimization results',103                                     xaxis_limit=0.8,104                                     markersize=5)...

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.