Unlock 30% off on Manual Testing Annual Plans this Holiday Season.

Copied to Clipboard!

How to use install_components method in localstack

Best Python code snippet using localstack_python

flex_setup.py

Source:flex_setup.py

1#!/usr/bin/python32import sys3import os4import zipfile5import argparse6import time7import glob8import code9import configparser10import shlex11import shutil12from pathlib import Path13from urllib import request14from urllib.parse import urljoin15def get_flex_setup_parser():16    parser = argparse.ArgumentParser(description="This script downloads Gluu Admin UI components and installs")17    parser.add_argument('--jans-setup-branch', help="Jannsen setup github branch", default='main')18    parser.add_argument('--flex-branch', help="Jannsen flex setup github branch", default='main')19    parser.add_argument('--jans-branch', help="Jannsen github branch", default='main')20    parser.add_argument('--flex-non-interactive', help="Non interactive mode", action='store_true')21    parser.add_argument('--install-admin-ui', help="Installs admin-ui", action='store_true')22    parser.add_argument('--install-casa', help="Installs casa", action='store_true')23    return parser24__STATIC_SETUP_DIR__ = '/opt/jans/jans-setup/'25installed_components = {'admin_ui': False, 'casa': False}26argsp = None27try:28    import jans_setup29    path_ = list(jans_setup.__path__)30    sys.path.append(path_[0])31except ModuleNotFoundError:32    if os.path.exists(os.path.join(__STATIC_SETUP_DIR__, 'setup_app')):33        sys.path.append(__STATIC_SETUP_DIR__)34    else:35        argsp, nargs = get_flex_setup_parser().parse_known_args()36        print("Unable to locate jans-setup, installing ...")37        setup_branch = argsp.jans_setup_branch or 'main'38        install_url = 'https://raw.githubusercontent.com/JanssenProject/jans/{}/jans-linux-setup/jans_setup/install.py'.format(setup_branch)39        request.urlretrieve(install_url, 'install.py')40        install_cmd = 'python3 install.py --setup-branch={}'.format(setup_branch)41        if nargs:42            install_cmd += ' --args="{}"'.format(shlex.join(nargs))43        print("Executing", install_cmd)44        os.system(install_cmd)45        sys.path.append(__STATIC_SETUP_DIR__)46if not argsp:47    argsp, nargs = get_flex_setup_parser().parse_known_args()48install_components = {49        'admin_ui': argsp.install_admin_ui,50        'casa': argsp.install_casa51    }52logs_dir = os.path.join(__STATIC_SETUP_DIR__, 'logs')53if not os.path.exists(logs_dir):54    os.makedirs(logs_dir)55from setup_app import paths56paths.LOG_FILE = os.path.join(logs_dir, 'flex-setup.log')57paths.LOG_ERROR_FILE = os.path.join(logs_dir, 'flex-setup-error.log')58print()59print("Log Files:")60print('\033[1m')61print(paths.LOG_FILE)62print(paths.LOG_ERROR_FILE)63print('\033[0m')64parser = get_flex_setup_parser()65from setup_app.utils import arg_parser66arg_parser.add_to_me(parser)67installed = False68if not os.path.exists('/etc/jans/conf/jans.properties'):69    installed = True70    try:71        from jans_setup import jans_setup72    except ImportError:73        import jans_setup74    jans_setup.main()75argsp = arg_parser.get_parser()76from setup_app import static77from setup_app.utils import base78profile_fn = os.path.join(base.par_dir, 'profile')79if os.path.exists(profile_fn):80    with open(profile_fn) as f:81        profile = f.read().strip()82else:83    profile = 'jans'84os.environ['JANS_PROFILE'] = profile85base.current_app.profile = profile86if 'SETUP_BRANCH' not in base.current_app.app_info:87    base.current_app.app_info['SETUP_BRANCH'] = argsp.jans_setup_branch88base.current_app.app_info['ox_version'] = base.current_app.app_info['JANS_APP_VERSION'] + base.current_app.app_info['JANS_BUILD']89sys.path.insert(0, base.pylib_dir)90sys.path.insert(0, os.path.join(base.pylib_dir, 'gcs'))91from setup_app.pylib.jproperties import Properties92from setup_app.utils.package_utils import packageUtils93from setup_app.config import Config94from setup_app.utils.collect_properties import CollectProperties95from setup_app.installers.node import NodeInstaller96from setup_app.installers.httpd import HttpdInstaller97from setup_app.installers.config_api import ConfigApiInstaller98from setup_app.installers.jetty import JettyInstaller99from setup_app.installers.jans_auth import JansAuthInstaller100from setup_app.installers.jans_cli import JansCliInstaller101from setup_app.utils.properties_utils import propertiesUtils102Config.outputFolder = os.path.join(__STATIC_SETUP_DIR__, 'output')103if not os.path.join(Config.outputFolder):104    os.makedirs(Config.outputFolder)105if not installed:106    # initialize config object107    Config.init(paths.INSTALL_DIR)108    collectProperties = CollectProperties()109    collectProperties.collect()110maven_base_url = 'https://jenkins.jans.io/maven/io/jans/'111app_versions = {112  "SETUP_BRANCH": argsp.jans_setup_branch,113  "FLEX_BRANCH": argsp.flex_branch,114  "JANS_BRANCH": argsp.jans_branch,115  "JANS_APP_VERSION": "1.0.1",116  "JANS_BUILD": "-SNAPSHOT", 117  "NODE_VERSION": "v14.18.2",118  "CASA_VERSION": "5.0.0-SNAPSHOT",119  "TWILIO_VERSION": "7.17.0",120}121node_installer = NodeInstaller()122httpd_installer = HttpdInstaller()123config_api_installer = ConfigApiInstaller()124jansAuthInstaller = JansAuthInstaller()125jans_cli_installer = JansCliInstaller()126setup_properties = base.read_properties_file(argsp.f) if argsp.f else {}127class flex_installer(JettyInstaller):128    def __init__(self):129        self.jans_auth_dir = os.path.join(Config.jetty_base, jansAuthInstaller.service_name)130        self.jans_auth_custom_lib_dir = os.path.join(self.jans_auth_dir, 'custom/libs')131        self.gluu_admin_ui_source_path = os.path.join(Config.dist_jans_dir, 'gluu-admin-ui.zip')132        self.log4j2_adminui_path = os.path.join(Config.dist_jans_dir, 'log4j2-adminui.xml')133        self.log4j2_path = os.path.join(Config.dist_jans_dir, 'log4j2.xml')134        self.admin_ui_plugin_source_path = os.path.join(Config.dist_jans_dir, 'admin-ui-plugin.jar')135        self.flex_path = os.path.join(Config.dist_jans_dir, 'flex.zip')136        self.source_dir = os.path.join(Config.install_dir, 'flex')137        self.flex_setup_dir = os.path.join(self.source_dir, 'flex-linux-setup')138        self.templates_dir = os.path.join(self.flex_setup_dir, 'templates')139        self.admin_ui_config_properties_path = os.path.join(self.templates_dir, 'auiConfiguration.properties')140        self.casa_dist_dir = os.path.join(Config.dist_jans_dir, 'gluu-casa')141        self.casa_web_resources_fn = os.path.join(self.casa_dist_dir, 'casa_web_resources.xml')142        self.casa_war_fn = os.path.join(self.casa_dist_dir, 'casa.war')143        self.casa_config_fn = os.path.join(self.casa_dist_dir,'casa-config.jar')144        self.casa_script_fn = os.path.join(self.casa_dist_dir,'Casa.py')145        self.twillo_fn = os.path.join(self.casa_dist_dir,'twilio.jar')146        self.py_lib_dir = os.path.join(Config.jansOptPythonFolder, 'libs')147        self.fido2_client_jar_fn = os.path.join(Config.dist_jans_dir, 'jans-fido2-client.jar')148        self.dbUtils.bind(force=True)149        if os.path.exists(self.source_dir):150            os.rename(self.source_dir, self.source_dir+'-'+time.ctime().replace(' ', '_'))151    def download_files(self):152        print("Downloading components")153        base.download('https://github.com/GluuFederation/flex/archive/refs/heads/{}.zip'.format(app_versions['FLEX_BRANCH']), self.flex_path, verbose=True)154        print("Extracting", self.flex_path)155        base.extract_from_zip(self.flex_path, 'flex-linux-setup/flex_linux_setup', self.flex_setup_dir)156        if install_components['admin_ui']:157            base.download(urljoin(maven_base_url, 'jans-config-api/plugins/admin-ui-plugin/{0}{1}/admin-ui-plugin-{0}{1}-distribution.jar'.format(app_versions['JANS_APP_VERSION'], app_versions['JANS_BUILD'])), self.admin_ui_plugin_source_path, verbose=True)158            base.download('https://raw.githubusercontent.com/JanssenProject/jans/{}/jans-config-api/server/src/main/resources/log4j2.xml'.format(app_versions['JANS_BRANCH']), self.log4j2_path, verbose=True)159            base.download('https://raw.githubusercontent.com/JanssenProject/jans/{}/jans-config-api/plugins/admin-ui-plugin/config/log4j2-adminui.xml'.format(app_versions['JANS_BRANCH']), self.log4j2_adminui_path, verbose=True)160        161        if install_components['casa']:162            base.download('https://raw.githubusercontent.com/GluuFederation/flex/main/casa/extras/casa_web_resources.xml', self.casa_web_resources_fn, verbose=True)163            base.download('https://maven.gluu.org/maven/org/gluu/casa/{0}/casa-{0}.war'.format(app_versions['CASA_VERSION']), self.casa_war_fn, verbose=True)164            base.download('https://maven.gluu.org/maven/org/gluu/casa-config/{0}/casa-config-{0}.jar'.format(app_versions['CASA_VERSION']), self.casa_config_fn, verbose=True)165            base.download('https://repo1.maven.org/maven2/com/twilio/sdk/twilio/{0}/twilio-{0}.jar'.format(app_versions['TWILIO_VERSION']), self.twillo_fn, verbose=True)166            base.download('https://raw.githubusercontent.com/GluuFederation/flex/main/casa/extras/Casa.py', self.casa_script_fn, verbose=True)167            base.download('https://raw.githubusercontent.com/GluuFederation/flex/main/casa/extras/casa-external_fido2.py', os.path.join(self.casa_dist_dir, 'pylib/casa-external_fido2.py'), verbose=True)168            base.download('https://raw.githubusercontent.com/GluuFederation/flex/main/casa/extras/casa-external_otp.py', os.path.join(self.casa_dist_dir, 'pylib/casa-external_otp.py'), verbose=True)169            base.download('https://raw.githubusercontent.com/GluuFederation/flex/main/casa/extras/casa-external_super_gluu.py', os.path.join(self.casa_dist_dir, 'pylib/casa-external_super_gluu.py'), verbose=True)170            base.download('https://raw.githubusercontent.com/GluuFederation/flex/main/casa/extras/casa-external_twilio_sms.py', os.path.join(self.casa_dist_dir, 'pylib/casa-external_twilio_sms.py'), verbose=True)171            base.download('https://maven.jans.io/maven/io/jans/jans-fido2-client/{0}{1}/jans-fido2-client-{0}{1}.jar'.format(app_versions['JANS_APP_VERSION'], app_versions['JANS_BUILD']), self.fido2_client_jar_fn, verbose=True)172    def add_apache_directive(self, check_str, template):173        print("Updating apache configuration")174        apache_directive_template_text = self.readFile(os.path.join(self.templates_dir, template))175        apache_directive_text = self.fomatWithDict(apache_directive_template_text, Config.templateRenderingDict)176        https_jans_text = self.readFile(httpd_installer.https_jans_fn)177        if check_str not in https_jans_text:178            https_jans_list = https_jans_text.splitlines()179            n = 0180            for i, l in enumerate(https_jans_list):181                if l.strip() == '</LocationMatch>':182                    n = i183            https_jans_list.insert(n+1, '\n' + apache_directive_text + '\n')184            self.writeFile(httpd_installer.https_jans_fn, '\n'.join(https_jans_list))185        self.enable_apache_mod_dir()186    def enable_apache_mod_dir(self):187        # Enable mod_dir for apache188        cmd_a2enmod = shutil.which('a2enmod')189        if base.clone_type == 'deb':190            httpd_installer.run([cmd_a2enmod, 'dir'])191        elif base.os_type == 'suse':192            httpd_installer.run([cmd_a2enmod, 'dir'])193            cmd_a2enflag = shutil.which('a2enflag')194            httpd_installer.run([cmd_a2enflag, 'SSL'])195        else:196            base_mod_path = Path('/etc/httpd/conf.modules.d/00-base.conf')197            mod_load_content = base_mod_path.read_text().splitlines()198            modified = False199            for i, l in enumerate(mod_load_content[:]):200                ls = l.strip()201                if ls.startswith('#') and ls.endswith('mod_dir.so'):202                    mod_load_content[i] = ls.lstrip('#')203                    modified = True204            if modified:205                base_mod_path.write_text('\n'.join(mod_load_content))206    def install_gluu_admin_ui(self):207        print("Installing Gluu Admin UI Frontend")208        print("Extracting admin-ui from", self.flex_path)209        base.extract_from_zip(self.flex_path, 'admin-ui', self.source_dir)210        print("Source directory:", self.source_dir)211        env_tmp = os.path.join(self.source_dir, '.env.tmp')212        print("env_tmp", env_tmp)213        config_api_installer.renderTemplateInOut(env_tmp, self.source_dir, self.source_dir)214        config_api_installer.copyFile(os.path.join(self.source_dir, '.env.tmp'), os.path.join(self.source_dir, '.env'))215        config_api_installer.run([paths.cmd_chown, '-R', 'node:node', self.source_dir])216        cmd_path = 'PATH=$PATH:{}/bin:{}/bin'.format(Config.jre_home, Config.node_home)217        for cmd in ('npm install @openapitools/openapi-generator-cli', 'npm run api', 'npm install', 'npm run build:prod'):218            print("Executing `{}`".format(cmd))219            run_cmd = '{} {}'.format(cmd_path, cmd)220            config_api_installer.run(['/bin/su', 'node','-c', run_cmd], self.source_dir)221        Config.templateRenderingDict['admin_ui_apache_root'] = os.path.join(httpd_installer.server_root, 'admin')222        self.add_apache_directive(Config.templateRenderingDict['admin_ui_apache_root'], 'admin_ui_apache_directive')223        print("Copying files to",  Config.templateRenderingDict['admin_ui_apache_root'])224        config_api_installer.copy_tree(os.path.join(self.source_dir, 'dist'),  Config.templateRenderingDict['admin_ui_apache_root'])225        config_api_installer.check_clients([('role_based_client_id', '2000.')])226        config_api_installer.renderTemplateInOut(self.admin_ui_config_properties_path, os.path.join(self.flex_setup_dir, 'templates'), config_api_installer.custom_config_dir)227        admin_ui_plugin_path = os.path.join(config_api_installer.libDir, os.path.basename(self.admin_ui_plugin_source_path))228        config_api_installer.copyFile(self.admin_ui_plugin_source_path, config_api_installer.libDir)229        config_api_installer.add_extra_class(admin_ui_plugin_path)230        for logfn in (self.log4j2_adminui_path, self.log4j2_path):231            config_api_installer.copyFile(logfn, config_api_installer.custom_config_dir)232        cli_config = Path(jans_cli_installer.config_ini_fn)233        current_plugins = []234        if cli_config.exists():235            config = configparser.ConfigParser()236            config.read_file(cli_config.open())237            current_plugins = config['DEFAULT'].get('jca_plugins', '').split(',')238        plugins = config_api_installer.get_plugins()239        for plugin in plugins:240            if plugin not in current_plugins:241                current_plugins.append(plugin)242        config['DEFAULT']['jca_plugins'] = ','.join(current_plugins)243        config.write(cli_config.open('w'))244        cli_config.chmod(0o600)245    def install_casa(self):246        print("Adding twillo and casa config to jans-auth")247        self.copyFile(self.casa_config_fn, self.jans_auth_custom_lib_dir)248        self.copyFile(self.twillo_fn, self.jans_auth_custom_lib_dir)249        class_path = '{},{}'.format(250            os.path.join(self.jans_auth_custom_lib_dir, os.path.basename(self.casa_config_fn)),251            os.path.join(self.jans_auth_custom_lib_dir, os.path.basename(self.twillo_fn)),252            )253        jansAuthInstaller.add_extra_class(class_path)254        print("Adding Fido2 Client lib to jans-auth")255        self.copyFile(self.fido2_client_jar_fn, self.jans_auth_custom_lib_dir)256        class_path = os.path.join(self.jans_auth_custom_lib_dir, os.path.basename(self.fido2_client_jar_fn))257        jansAuthInstaller.add_extra_class(class_path)258        simple_auth_scr_inum = 'A51E-76DA'259        print("Enabling script", simple_auth_scr_inum)260        self.dbUtils.enable_script(simple_auth_scr_inum)261        # copy casa scripts262        if not os.path.exists(self.py_lib_dir):263            os.makedirs(self.py_lib_dir)264        for fn in glob.glob(os.path.join(self.casa_dist_dir, 'pylib/*.py')):265            print("Copying", fn, "to", self.py_lib_dir)266            self.copyFile(fn, self.py_lib_dir)267        self.run([paths.cmd_chown, '-R', '{0}:{0}'.format(Config.jetty_user), self.py_lib_dir])268        # prepare casa scipt ldif269        casa_auth_script_fn = os.path.join(self.templates_dir, 'casa_person_authentication_script.ldif')270        base64_script_file = self.generate_base64_file(self.casa_script_fn, 1)271        Config.templateRenderingDict['casa_person_authentication_script'] = base64_script_file272        self.renderTemplateInOut(casa_auth_script_fn, self.templates_dir, self.source_dir)273        Config.templateRenderingDict['casa_redirect_uri'] = 'https://{}/casa'.format(Config.hostname)274        Config.templateRenderingDict['casa_redirect_logout_uri'] = 'https://{}/casa/bye.zul'.format(Config.hostname)275        Config.templateRenderingDict['casa_frontchannel_logout_uri'] = 'https://{}/casa/autologout'.format(Config.hostname)276        Config.templateRenderingDict['casa_web_port'] = '8080'277        self.casa_client_fn = os.path.join(self.source_dir, 'templates/casa_client.ldif')278        self.casa_config_fn = os.path.join(self.source_dir, 'templates/casa_config.ldif')279        self.service_name = 'casa'280        for casa_prop in ('casa_client_id', 'casa_client_pw'):281            if casa_prop in setup_properties:282                setattr(Config, casa_prop, setup_properties[casa_prop])283        self.check_clients([('casa_client_id', '3000.')])284        if not Config.get('casa_client_encoded_pw'):285            Config.casa_client_encoded_pw = jansAuthInstaller.obscure(Config.casa_client_pw)286        print("Casa client id", Config.casa_client_id)287        print("Casa client encoded password", Config.casa_client_encoded_pw)288        print("Importing LDIF Files")289        self.renderTemplateInOut(self.casa_client_fn, self.templates_dir, self.source_dir)290        self.renderTemplateInOut(self.casa_config_fn, self.templates_dir, self.source_dir)291        self.dbUtils.import_ldif([292                os.path.join(self.source_dir, os.path.basename(self.casa_client_fn)),293                os.path.join(self.source_dir, os.path.basename(self.casa_config_fn)),294                os.path.join(self.source_dir, os.path.basename(casa_auth_script_fn)),295                ])296        Config.installCasa = True297        self.copyFile(os.path.join(self.templates_dir, 'casa.default'), os.path.join(Config.templateFolder, 'jetty/casa'))298        self.jetty_app_configuration[self.service_name] = {299                    "memory": {300                        "max_allowed_mb": 1024,301                        "metaspace_mb": 128,302                        "jvm_heap_ration": 0.7,303                        "ratio": 0.1304                        },305                    "jetty": {306                        "modules": "server,deploy,resources,http,http-forwarded,console-capture,jsp,cdi-decorate"307                    },308                    "installed": False,309                    "name": self.service_name310                }311        print("Calculating application memory")312        installedComponents = []313        # Jetty apps314        for config_var, service in [('installOxAuth', 'jans-auth'),315                                    ('installScimServer', 'jans-scim'),316                                    ('installFido2', 'jans-fido2'),317                                    ('installConfigApi', 'jans-config-api'),318                                    ('installEleven', 'jans-eleven'),319                                    ('installCasa', self.service_name),320                                    ]:321            if Config.get(config_var) and service in self.jetty_app_configuration:322                installedComponents.append(self.jetty_app_configuration[service])323        self.calculate_aplications_memory(Config.application_max_ram, self.jetty_app_configuration, installedComponents)324        print("Deploying casa as Jetty application")325        self.installJettyService(self.jetty_app_configuration[self.service_name], True)326        self.copyFile(os.path.join(self.templates_dir, 'casa.service'), '/etc/systemd/system')327        jetty_service_dir = os.path.join(Config.jetty_base, self.service_name)328        jetty_service_webapps_dir = os.path.join(jetty_service_dir, 'webapps')329        self.run([paths.cmd_mkdir, '-p', os.path.join(jetty_service_dir, 'static')])330        self.run([paths.cmd_mkdir, '-p', os.path.join(jetty_service_dir, 'plugins')])331        self.copyFile(self.casa_war_fn, jetty_service_webapps_dir)332        self.copyFile(self.casa_web_resources_fn, jetty_service_webapps_dir)333        jansAuthInstaller.chown(jetty_service_dir, Config.jetty_user, Config.jetty_group, recursive=True)334        jansAuthInstaller.chown(self.jans_auth_custom_lib_dir, Config.jetty_user, Config.jetty_group, recursive=True)335        self.add_apache_directive('<Location /casa>', 'casa_apache_directive')336        self.enable()337    def save_properties(self):338        fn = Config.savedProperties339        print("Saving properties", fn)340        if os.path.exists(fn):341            p = Properties()342            with open(fn, 'rb') as f:343                p.load(f, 'utf-8')344            for prop in ('casa_client_id', 'casa_client_pw', 'casa_client_encoded_pw'):345                if Config.get(prop):346                    p[prop] = Config.get(prop)347            with open(fn, 'wb') as f:348                p.store(f, encoding="utf-8")349        else:350            propertiesUtils.save_properties()351def prompt_for_installation():352    if not os.path.exists(os.path.join(httpd_installer.server_root, 'admin')):353        prompt_admin_ui_install = input("Install Admin UI [Y/n]: ")354        if prompt_admin_ui_install and prompt_admin_ui_install.lower().startswith('y'):355            install_components['admin_ui'] = True356    else:357        print("Admin UI is allready installed on this system")358        install_components['admin_ui'] = False359    if not os.path.exists(os.path.join(Config.jetty_base, 'casa')):360        prompt_casa_install = input("Install Casa [Y/n]: ")361        if prompt_casa_install and prompt_casa_install.lower().startswith('y'):362            install_components['casa'] = True363    else:364        print("Casa is allready installed on this system")365        install_components['casa'] = False366    if not (install_components['casa'] or install_components['admin_ui']):367        print("Nothing to install. Exiting ...")368        sys.exit()369def main():370    if not argsp.flex_non_interactive:371        prompt_for_installation()372    if install_components['admin_ui'] and not node_installer.installed():373        node_fn = 'node-{0}-linux-x64.tar.xz'.format(app_versions['NODE_VERSION'])374        node_path = os.path.join(Config.dist_app_dir, node_fn)375        if not os.path.exists(node_path):376            base.download('https://nodejs.org/dist/{0}/node-{0}-linux-x64.tar.xz'.format(app_versions['NODE_VERSION']), node_path, verbose=True)377        print("Installing node")378        node_installer.install()379    installer_obj = flex_installer()380    installer_obj.download_files()381    if install_components['admin_ui']:382        installer_obj.install_gluu_admin_ui()383    if install_components['casa']:384        installer_obj.install_casa()385        installer_obj.save_properties()386    print("Restarting Apache")387    httpd_installer.restart()388    print("Restarting Jans Auth")389    config_api_installer.restart('jans-auth')390    print("Restarting Janssen Config Api")391    config_api_installer.restart()392    if install_components['casa']:393        print("Starting Casa")394        config_api_installer.start('casa')395    print("Installation was completed.")396    if install_components['admin_ui']:397        print("Browse https://{}/admin".format(Config.hostname))398    if install_components['casa']:399        print("Browse https://{}/casa".format(Config.hostname))400if __name__ == "__main__":401    if argsp.shell:402        code.interact(local=locals())403        sys.exit()404    else:...

intelbase.py

Source:intelbase.py

1# #2# Copyright 2009-2019 Ghent University3#4# This file is part of EasyBuild,5# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),6# with support of Ghent University (http://ugent.be/hpc),7# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),8# Flemish Research Foundation (FWO) (http://www.fwo.be/en)9# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).10#11# https://github.com/easybuilders/easybuild12#13# EasyBuild is free software: you can redistribute it and/or modify14# it under the terms of the GNU General Public License as published by15# the Free Software Foundation v2.16#17# EasyBuild is distributed in the hope that it will be useful,18# but WITHOUT ANY WARRANTY; without even the implied warranty of19# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the20# GNU General Public License for more details.21#22# You should have received a copy of the GNU General Public License23# along with EasyBuild.  If not, see <http://www.gnu.org/licenses/>.24# #25"""26Generic EasyBuild support for installing Intel tools, implemented as an easyblock27@author: Stijn De Weirdt (Ghent University)28@author: Dries Verdegem (Ghent University)29@author: Kenneth Hoste (Ghent University)30@author: Pieter De Baets (Ghent University)31@author: Jens Timmerman (Ghent University)32@author: Ward Poelmans (Ghent University)33@author: Lumir Jasiok (IT4Innovations)34@author: Damian Alvarez (Forschungszentrum Juelich GmbH)35"""36import os37import re38import shutil39import tempfile40import glob41from distutils.version import LooseVersion42import easybuild.tools.environment as env43from easybuild.framework.easyblock import EasyBlock44from easybuild.framework.easyconfig import CUSTOM45from easybuild.framework.easyconfig.types import ensure_iterable_license_specs46from easybuild.tools.build_log import EasyBuildError47from easybuild.tools.filetools import find_flexlm_license, read_file, remove_file48from easybuild.tools.run import run_cmd49from vsc.utils import fancylogger50_log = fancylogger.getLogger('generic.intelbase')51# different supported activation types (cfr. Intel documentation)52ACTIVATION_EXIST_LIC = 'exist_lic'  # use a license which exists on the system53ACTIVATION_LIC_FILE = 'license_file'  # use a license file54ACTIVATION_LIC_SERVER = 'license_server'  # use a license server55ACTIVATION_SERIAL = 'serial_number'  # use a serial number56ACTIVATION_TRIAL = 'trial_lic'  # use trial activation57ACTIVATION_TYPES = [58    ACTIVATION_EXIST_LIC,59    ACTIVATION_LIC_FILE,60    ACTIVATION_LIC_SERVER,61    ACTIVATION_SERIAL,62    ACTIVATION_TRIAL,63]64# silent.cfg parameter name for type of license activation (cfr. options listed above)65ACTIVATION_NAME = 'ACTIVATION_TYPE'  # since icc/ifort v2013_sp1, impi v4.1.1, imkl v11.166ACTIVATION_NAME_2012 = 'ACTIVATION'  # previous activation type parameter used in older versions67# silent.cfg parameter name for install prefix68INSTALL_DIR_NAME = 'PSET_INSTALL_DIR'69# silent.cfg parameter name for install mode70INSTALL_MODE_NAME = 'PSET_MODE'71# Older (2015 and previous) silent.cfg parameter name for install mode72INSTALL_MODE_NAME_2015 = 'INSTALL_MODE'73# Install mode for 2016 version74INSTALL_MODE = 'install'75# Install mode for 2015 and older versions76INSTALL_MODE_2015 = 'NONRPM'77# silent.cfg parameter name for license file/server specification78LICENSE_FILE_NAME = 'ACTIVATION_LICENSE_FILE'  # since icc/ifort v2013_sp1, impi v4.1.1, imkl v11.179LICENSE_FILE_NAME_2012 = 'PSET_LICENSE_FILE'  # previous license file parameter used in older versions80LICENSE_SERIAL_NUMBER = 'ACTIVATION_SERIAL_NUMBER'81COMP_ALL = 'ALL'82COMP_DEFAULTS = 'DEFAULTS'83class IntelBase(EasyBlock):84    """85    Base class for Intel software86    - no configure/make : binary release87    - add license_file variable88    """89    def __init__(self, *args, **kwargs):90        """Constructor, adds extra config options"""91        super(IntelBase, self).__init__(*args, **kwargs)92        self.license_file = 'UNKNOWN'93        self.license_env_var = 'UNKNOWN'94        # Initialise whether we need a runtime licence or not95        self.requires_runtime_license = True96        self.home_subdir = os.path.join(os.getenv('HOME'), 'intel')97        common_tmp_dir = os.path.dirname(tempfile.gettempdir())  # common tmp directory, same across nodes98        self.home_subdir_local = os.path.join(common_tmp_dir, os.getenv('USER'), 'easybuild_intel')99        self.install_components = None100    def get_guesses_tools(self):101        """Find reasonable paths for a subset of Intel tools, ignoring CPATH, LD_LIBRARY_PATH and LIBRARY_PATH"""102        guesses = super(IntelBase, self).make_module_req_guess()103        if self.cfg['m32']:104            guesses['PATH'] = [os.path.join(self.subdir, 'bin32')]105        else:106            guesses['PATH'] = [os.path.join(self.subdir, 'bin64')]107        guesses['MANPATH'] = [os.path.join(self.subdir, 'man')]108        # make sure $CPATH, $LD_LIBRARY_PATH and $LIBRARY_PATH are not updated in generated module file,109        # because that leads to problem when the libraries included with VTune/Advisor/Inspector are being picked up110        for key in ['CPATH', 'LD_LIBRARY_PATH', 'LIBRARY_PATH']:111            if key in guesses:112                self.log.debug("Purposely not updating $%s in %s module file", key, self.name)113                del guesses[key]114        return guesses115    def get_custom_paths_tools(self, binaries):116        """Custom sanity check paths for certain Intel tools."""117        if self.cfg['m32']:118            files = [os.path.join('bin32', b) for b in binaries]119            dirs = ['lib32', 'include']120        else:121            files = [os.path.join('bin64', b) for b in binaries]122            dirs = ['lib64', 'include']123        custom_paths = {124            'files': [os.path.join(self.subdir, f) for f in files],125            'dirs': [os.path.join(self.subdir, d) for d in dirs],126        }127        return custom_paths128    @staticmethod129    def extra_options(extra_vars=None):130        extra_vars = EasyBlock.extra_options(extra_vars)131        extra_vars.update({132            'license_activation': [ACTIVATION_LIC_SERVER, "License activation type", CUSTOM],133            'serial_number': [None, "Serial number for the product", CUSTOM],134            'requires_runtime_license': [True, "Boolean indicating whether or not a runtime license is required",135                                         CUSTOM],136            # 'usetmppath':137            # workaround for older SL5 version (5.5 and earlier)138            # used to be True, but False since SL5.6/SL6139            # disables TMP_PATH env and command line option140            'usetmppath': [False, "Use temporary path for installation", CUSTOM],141            'm32': [False, "Enable 32-bit toolchain", CUSTOM],142            'components': [None, "List of components to install", CUSTOM],143        })144        return extra_vars145    def parse_components_list(self):146        """parse the regex in the components extra_options and select the matching components147        from the mediaconfig.xml file in the install dir"""148        mediaconfigpath = os.path.join(self.cfg['start_dir'], 'pset', 'mediaconfig.xml')149        if not os.path.isfile(mediaconfigpath):150            raise EasyBuildError("Could not find %s to find list of components." % mediaconfigpath)151        mediaconfig = read_file(mediaconfigpath)152        available_components = re.findall("<Abbr>(?P<component>[^<]+)</Abbr>", mediaconfig, re.M)153        self.log.debug("Intel components found: %s" % available_components)154        self.log.debug("Using regex list: %s" % self.cfg['components'])155        if COMP_ALL in self.cfg['components'] or COMP_DEFAULTS in self.cfg['components']:156            if len(self.cfg['components']) == 1:157                self.install_components = self.cfg['components']158            else:159                raise EasyBuildError("If you specify %s as components, you cannot specify anything else: %s",160                                     ' or '.join([COMP_ALL, COMP_DEFAULTS]), self.cfg['components'])161        else:162            self.install_components = []163            for comp_regex in self.cfg['components']:164                comps = [comp for comp in available_components if re.match(comp_regex, comp)]165                self.install_components.extend(comps)166        self.log.debug("Components to install: %s" % self.install_components)167    def clean_home_subdir(self):168        """Remove contents of (local) 'intel' directory home subdir, where stuff is cached."""169        self.log.debug("Cleaning up %s..." % self.home_subdir_local)170        try:171            for tree in os.listdir(self.home_subdir_local):172                self.log.debug("... removing %s subtree" % tree)173                path = os.path.join(self.home_subdir_local, tree)174                if os.path.isfile(path) or os.path.islink(path):175                    remove_file(path)176                else:177                    shutil.rmtree(path)178        except OSError as err:179            raise EasyBuildError("Cleaning up intel dir %s failed: %s", self.home_subdir_local, err)180    def setup_local_home_subdir(self):181        """182        Intel script use $HOME/intel to cache stuff.183        To enable parallel builds, we symlink $HOME/intel to a temporary dir on the local disk."""184        try:185            # make sure local directory exists186            if not os.path.exists(self.home_subdir_local):187                os.makedirs(self.home_subdir_local)188                self.log.debug("Created local dir %s" % self.home_subdir_local)189            if os.path.exists(self.home_subdir):190                # if 'intel' dir in $HOME already exists, make sure it's the right symlink191                symlink_ok = os.path.islink(self.home_subdir) and os.path.samefile(self.home_subdir,192                                                                                   self.home_subdir_local)193                if not symlink_ok:194                    # rename current 'intel' dir195                    home_intel_bk = tempfile.mkdtemp(dir=os.path.dirname(self.home_subdir),196                                                     prefix='%s.bk.' % os.path.basename(self.home_subdir))197                    self.log.info("Moving %(ih)s to %(ihl)s, I need %(ih)s myself..." % {'ih': self.home_subdir,198                                                                                         'ihl': home_intel_bk})199                    shutil.move(self.home_subdir, home_intel_bk)200                    # set symlink in place201                    os.symlink(self.home_subdir_local, self.home_subdir)202                    self.log.debug("Created symlink (1) %s to %s" % (self.home_subdir, self.home_subdir_local))203            else:204                # if a broken symlink is present, remove it first205                if os.path.islink(self.home_subdir):206                    remove_file(self.home_subdir)207                os.symlink(self.home_subdir_local, self.home_subdir)208                self.log.debug("Created symlink (2) %s to %s" % (self.home_subdir, self.home_subdir_local))209        except OSError as err:210            raise EasyBuildError("Failed to symlink %s to %s: %s", self.home_subdir_local, self.home_subdir, err)211    def prepare_step(self, *args, **kwargs):212        """Custom prepare step for IntelBase. Set up the license"""213        requires_runtime_license = kwargs.pop('requires_runtime_license', True)214        super(IntelBase, self).prepare_step(*args, **kwargs)215        # Decide if we need a license or not (default is True because of defaults of individual Booleans)216        self.requires_runtime_license = self.cfg['requires_runtime_license'] and requires_runtime_license217        self.serial_number = self.cfg['serial_number']218        if self.serial_number:219            self.log.info("Using provided serial number (%s) and ignoring other licenses", self.serial_number)220        elif self.requires_runtime_license:221            default_lic_env_var = 'INTEL_LICENSE_FILE'222            license_specs = ensure_iterable_license_specs(self.cfg['license_file'])223            lic_specs, self.license_env_var = find_flexlm_license(custom_env_vars=[default_lic_env_var],224                                                                  lic_specs=license_specs)225            if lic_specs:226                if self.license_env_var is None:227                    self.log.info("Using Intel license specifications from 'license_file': %s", lic_specs)228                    self.license_env_var = default_lic_env_var229                else:230                    self.log.info("Using Intel license specifications from $%s: %s", self.license_env_var, lic_specs)231                self.license_file = os.pathsep.join(lic_specs)232                env.setvar(self.license_env_var, self.license_file)233                # if we have multiple retained lic specs, specify to 'use a license which exists on the system'234                if len(lic_specs) > 1:235                    self.log.debug("More than one license specs found, using '%s' license activation instead of "236                                   "'%s'", ACTIVATION_EXIST_LIC, self.cfg['license_activation'])237                    self.cfg['license_activation'] = ACTIVATION_EXIST_LIC238                    # $INTEL_LICENSE_FILE should always be set during installation with existing license239                    env.setvar(default_lic_env_var, self.license_file)240            else:241                msg = "No viable license specifications found; "242                msg += "specify 'license_file', or define $INTEL_LICENSE_FILE or $LM_LICENSE_FILE"243                raise EasyBuildError(msg)244    def configure_step(self):245        """Configure: handle license file and clean home dir."""246        # prepare (local) 'intel' home subdir247        self.setup_local_home_subdir()248        self.clean_home_subdir()249        # determine list of components, based on 'components' easyconfig parameter (if specified)250        if self.cfg['components']:251            self.parse_components_list()252        else:253            self.log.debug("No components specified")254    def build_step(self):255        """Binary installation files, so no building."""256        pass257    def install_step(self, silent_cfg_names_map=None, silent_cfg_extras=None):258        """Actual installation259        - create silent cfg file260        - set environment parameters261        - execute command262        """263        if silent_cfg_names_map is None:264            silent_cfg_names_map = {}265        if self.serial_number or self.requires_runtime_license:266            lic_entry = ""267            if self.serial_number:268                lic_entry = "%(license_serial_number)s=%(serial_number)s"269                self.cfg['license_activation'] = ACTIVATION_SERIAL270            else:271                # license file entry is only applicable with license file or server type of activation272                # also check whether specified activation type makes sense273                lic_file_server_activations = [ACTIVATION_LIC_FILE, ACTIVATION_LIC_SERVER]274                other_activations = [act for act in ACTIVATION_TYPES if act not in lic_file_server_activations]275                if self.cfg['license_activation'] in lic_file_server_activations:276                    lic_entry = "%(license_file_name)s=%(license_file)s"277                elif not self.cfg['license_activation'] in other_activations:278                    raise EasyBuildError("Unknown type of activation specified: %s (known :%s)",279                                         self.cfg['license_activation'], ACTIVATION_TYPES)280            silent = '\n'.join([281                "%(activation_name)s=%(activation)s",282                lic_entry,283                ""  # Add a newline at the end, so we can easily append if needed284            ]) % {285                'activation_name': silent_cfg_names_map.get('activation_name', ACTIVATION_NAME),286                'activation': self.cfg['license_activation'],287                'license_file_name': silent_cfg_names_map.get('license_file_name', LICENSE_FILE_NAME),288                'license_file': self.license_file,289                'license_serial_number': silent_cfg_names_map.get('license_serial_number', LICENSE_SERIAL_NUMBER),290                'serial_number': self.serial_number,291            }292        else:293            self.log.debug("No license required, so not including license specifications in silent.cfg")294            silent = ''295        silent += '\n'.join([296            "%(install_dir_name)s=%(install_dir)s",297            "ACCEPT_EULA=accept",298            "%(install_mode_name)s=%(install_mode)s",299            "CONTINUE_WITH_OPTIONAL_ERROR=yes",300            ""  # Add a newline at the end, so we can easily append if needed301        ]) % {302            'install_dir_name': silent_cfg_names_map.get('install_dir_name', INSTALL_DIR_NAME),303            'install_dir': silent_cfg_names_map.get('install_dir', self.installdir),304            'install_mode': silent_cfg_names_map.get('install_mode', INSTALL_MODE_2015),305            'install_mode_name': silent_cfg_names_map.get('install_mode_name', INSTALL_MODE_NAME_2015),306        }307        if self.install_components is not None:308            if len(self.install_components) == 1 and self.install_components[0] in [COMP_ALL, COMP_DEFAULTS]:309                # no quotes should be used for ALL or DEFAULTS310                silent += 'COMPONENTS=%s\n' % self.install_components[0]311            elif self.install_components:312                # a list of components is specified (needs quotes)313                components = ';'.join(self.install_components)314                if LooseVersion(self.version) >= LooseVersion('2017'):315                    # for versions 2017.x and newer, double quotes should not be there...316                    silent += 'COMPONENTS=%s\n' % components317                else:318                    silent += 'COMPONENTS="%s"\n' % components319            else:320                raise EasyBuildError("Empty list of matching components obtained via %s", self.cfg['components'])321        if silent_cfg_extras is not None:322            if isinstance(silent_cfg_extras, dict):323                silent += '\n'.join("%s=%s" % (key, value) for (key, value) in silent_cfg_extras.iteritems())324            else:325                raise EasyBuildError("silent_cfg_extras needs to be a dict")326        # we should be already in the correct directory327        silentcfg = os.path.join(os.getcwd(), "silent.cfg")328        try:329            f = open(silentcfg, 'w')330            f.write(silent)331            f.close()332        except:333            raise EasyBuildError("Writing silent cfg, failed", silent)334        self.log.debug("Contents of %s:\n%s" % (silentcfg, silent))335        # workaround for mktmp: create tmp dir and use it336        tmpdir = os.path.join(self.cfg['start_dir'], 'mytmpdir')337        try:338            os.makedirs(tmpdir)339        except:340            raise EasyBuildError("Directory %s can't be created", tmpdir)341        tmppathopt = ''342        if self.cfg['usetmppath']:343            env.setvar('TMP_PATH', tmpdir)344            tmppathopt = "-t %s" % tmpdir345        # set some extra env variables346        env.setvar('LOCAL_INSTALL_VERBOSE', '1')347        env.setvar('VERBOSE_MODE', '1')348        env.setvar('INSTALL_PATH', self.installdir)349        # perform installation350        cmd = ' '.join([351            self.cfg['preinstallopts'],352            './install.sh',353            tmppathopt,354            '-s ' + silentcfg,355            self.cfg['installopts'],356        ])357        return run_cmd(cmd, log_all=True, simple=True, log_output=True)358    def move_after_install(self):359        """Move installed files to correct location after installation."""360        subdir = os.path.join(self.installdir, self.name, self.version)361        self.log.debug("Moving contents of %s to %s" % (subdir, self.installdir))362        try:363            # remove senseless symlinks, e.g. impi_5.0.1 and impi_latest364            majver = '.'.join(self.version.split('.')[:-1])365            for symlink in ['%s_%s' % (self.name, majver), '%s_latest' % self.name]:366                symlink_fp = os.path.join(self.installdir, symlink)367                if os.path.exists(symlink_fp):368                    remove_file(symlink_fp)369            # move contents of 'impi/<version>' dir to installdir370            for fil in os.listdir(subdir):371                source = os.path.join(subdir, fil)372                target = os.path.join(self.installdir, fil)373                self.log.debug("Moving %s to %s" % (source, target))374                shutil.move(source, target)375            shutil.rmtree(os.path.join(self.installdir, self.name))376        except OSError as err:377            raise EasyBuildError("Failed to move contents of %s to %s: %s", subdir, self.installdir, err)378    def sanity_check_rpath(self):379        """Skip the rpath sanity check, this is binary software"""380        self.log.info("RPATH sanity check is skipped when using %s easyblock (derived from IntelBase)",381                      self.__class__.__name__)382    def make_module_extra(self, *args, **kwargs):383        """Custom variable definitions in module file."""384        txt = super(IntelBase, self).make_module_extra(*args, **kwargs)385        if self.requires_runtime_license:386            txt += self.module_generator.prepend_paths(self.license_env_var, [self.license_file],387                                                       allow_abs=True, expand_relpaths=False)388        return txt389    def cleanup_step(self):390        """Cleanup leftover mess391        - clean home dir392        - generic cleanup (get rid of build dir)393        """394        self.clean_home_subdir()395        super(IntelBase, self).cleanup_step()...

fabfile.py

Source:fabfile.py

...43    44    # å®è£æææ¬å°çä¾èµå45    if local('/bin/bash install.sh').failed:46        abort('å®è£æ¬å°python packageså¤±è´¥ã')47def install_components(pattern, install_path):48    """49    ä»Download_Path æ¥æ¾ç¬¦åpatternçç¬¬ä¸ä¸ªåç¼©å, è§£åä¹åç§»å¨å°install_path50    """51    tarfile_path = find_tarfile(opath.join(Download_Path, pattern)) # ./pip-cache/$pattern-x.x.x.tar.gz ,find ./pip-cache -name pattern-*.tar.gz52    extract_path = find_tarfile_extract_path(tarfile_path) # pattern-x.x.x, tar xzf x.tar.gz | head -n153    54    if extract_path:55        extract_result = local("tar xzf %s" % (tarfile_path, ))56        57        if extract_result.failed:58            abort('cant not extract %s' % tarfile_path)59        60        if local("mv %s %s" % (extract_path, install_path)).failed:61            abort('can not move %s from %s to %s' % (pattern, extract_path, install_path))62def extract_swagger():63    install_components('swagger-*.tar.gz', 'nebula/middleware/tornado_rest_swagger/assets')64def extract_grafana():65    install_components('grafana-*.tar.gz', 'nebula/grafana_app')66def dep_web_front():67    pattern = 'nebula_web_frontend-*.tar.gz'68    tarfile_path = find_tarfile(opath.join(Download_Path, pattern)) # ./pip-cache/nebula_web_frontend-x.x.x.tar.gz69    extract_path = find_tarfile_extract_path(tarfile_path) # nebula_web_frontend-x.x.x70    71    FrontEnd_Install_Path = 'target/nebula/' # @todo nebula72    move_dirs = ['statics', 'templates']73    if extract_path:74        extract_result = local("tar xzf %s" % (tarfile_path, ))75        76        if extract_result.failed:77            abort('cant not extract %s' % tarfile_path)78        for _ in move_dirs:79            if local("mv %s %s" % ( opath.join(extract_path, _),...

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.