How to use install_components method in localstack

Best Python code snippet using localstack_python

flex_setup.py

Source:flex_setup.py Github

copy

Full Screen

1#!/​usr/​bin/​python32import sys3import os4import zipfile5import argparse6import time7import glob8import code9import configparser10import shlex11import shutil12from pathlib import Path13from urllib import request14from urllib.parse import urljoin15def get_flex_setup_parser():16 parser = argparse.ArgumentParser(description="This script downloads Gluu Admin UI components and installs")17 parser.add_argument('--jans-setup-branch', help="Jannsen setup github branch", default='main')18 parser.add_argument('--flex-branch', help="Jannsen flex setup github branch", default='main')19 parser.add_argument('--jans-branch', help="Jannsen github branch", default='main')20 parser.add_argument('--flex-non-interactive', help="Non interactive mode", action='store_true')21 parser.add_argument('--install-admin-ui', help="Installs admin-ui", action='store_true')22 parser.add_argument('--install-casa', help="Installs casa", action='store_true')23 return parser24__STATIC_SETUP_DIR__ = '/​opt/​jans/​jans-setup/​'25installed_components = {'admin_ui': False, 'casa': False}26argsp = None27try:28 import jans_setup29 path_ = list(jans_setup.__path__)30 sys.path.append(path_[0])31except ModuleNotFoundError:32 if os.path.exists(os.path.join(__STATIC_SETUP_DIR__, 'setup_app')):33 sys.path.append(__STATIC_SETUP_DIR__)34 else:35 argsp, nargs = get_flex_setup_parser().parse_known_args()36 print("Unable to locate jans-setup, installing ...")37 setup_branch = argsp.jans_setup_branch or 'main'38 install_url = 'https:/​/​raw.githubusercontent.com/​JanssenProject/​jans/​{}/​jans-linux-setup/​jans_setup/​install.py'.format(setup_branch)39 request.urlretrieve(install_url, 'install.py')40 install_cmd = 'python3 install.py --setup-branch={}'.format(setup_branch)41 if nargs:42 install_cmd += ' --args="{}"'.format(shlex.join(nargs))43 print("Executing", install_cmd)44 os.system(install_cmd)45 sys.path.append(__STATIC_SETUP_DIR__)46if not argsp:47 argsp, nargs = get_flex_setup_parser().parse_known_args()48install_components = {49 'admin_ui': argsp.install_admin_ui,50 'casa': argsp.install_casa51 }52logs_dir = os.path.join(__STATIC_SETUP_DIR__, 'logs')53if not os.path.exists(logs_dir):54 os.makedirs(logs_dir)55from setup_app import paths56paths.LOG_FILE = os.path.join(logs_dir, 'flex-setup.log')57paths.LOG_ERROR_FILE = os.path.join(logs_dir, 'flex-setup-error.log')58print()59print("Log Files:")60print('\033[1m')61print(paths.LOG_FILE)62print(paths.LOG_ERROR_FILE)63print('\033[0m')64parser = get_flex_setup_parser()65from setup_app.utils import arg_parser66arg_parser.add_to_me(parser)67installed = False68if not os.path.exists('/​etc/​jans/​conf/​jans.properties'):69 installed = True70 try:71 from jans_setup import jans_setup72 except ImportError:73 import jans_setup74 jans_setup.main()75argsp = arg_parser.get_parser()76from setup_app import static77from setup_app.utils import base78profile_fn = os.path.join(base.par_dir, 'profile')79if os.path.exists(profile_fn):80 with open(profile_fn) as f:81 profile = f.read().strip()82else:83 profile = 'jans'84os.environ['JANS_PROFILE'] = profile85base.current_app.profile = profile86if 'SETUP_BRANCH' not in base.current_app.app_info:87 base.current_app.app_info['SETUP_BRANCH'] = argsp.jans_setup_branch88base.current_app.app_info['ox_version'] = base.current_app.app_info['JANS_APP_VERSION'] + base.current_app.app_info['JANS_BUILD']89sys.path.insert(0, base.pylib_dir)90sys.path.insert(0, os.path.join(base.pylib_dir, 'gcs'))91from setup_app.pylib.jproperties import Properties92from setup_app.utils.package_utils import packageUtils93from setup_app.config import Config94from setup_app.utils.collect_properties import CollectProperties95from setup_app.installers.node import NodeInstaller96from setup_app.installers.httpd import HttpdInstaller97from setup_app.installers.config_api import ConfigApiInstaller98from setup_app.installers.jetty import JettyInstaller99from setup_app.installers.jans_auth import JansAuthInstaller100from setup_app.installers.jans_cli import JansCliInstaller101from setup_app.utils.properties_utils import propertiesUtils102Config.outputFolder = os.path.join(__STATIC_SETUP_DIR__, 'output')103if not os.path.join(Config.outputFolder):104 os.makedirs(Config.outputFolder)105if not installed:106 # initialize config object107 Config.init(paths.INSTALL_DIR)108 collectProperties = CollectProperties()109 collectProperties.collect()110maven_base_url = 'https:/​/​jenkins.jans.io/​maven/​io/​jans/​'111app_versions = {112 "SETUP_BRANCH": argsp.jans_setup_branch,113 "FLEX_BRANCH": argsp.flex_branch,114 "JANS_BRANCH": argsp.jans_branch,115 "JANS_APP_VERSION": "1.0.1",116 "JANS_BUILD": "-SNAPSHOT", 117 "NODE_VERSION": "v14.18.2",118 "CASA_VERSION": "5.0.0-SNAPSHOT",119 "TWILIO_VERSION": "7.17.0",120}121node_installer = NodeInstaller()122httpd_installer = HttpdInstaller()123config_api_installer = ConfigApiInstaller()124jansAuthInstaller = JansAuthInstaller()125jans_cli_installer = JansCliInstaller()126setup_properties = base.read_properties_file(argsp.f) if argsp.f else {}127class flex_installer(JettyInstaller):128 def __init__(self):129 self.jans_auth_dir = os.path.join(Config.jetty_base, jansAuthInstaller.service_name)130 self.jans_auth_custom_lib_dir = os.path.join(self.jans_auth_dir, 'custom/​libs')131 self.gluu_admin_ui_source_path = os.path.join(Config.dist_jans_dir, 'gluu-admin-ui.zip')132 self.log4j2_adminui_path = os.path.join(Config.dist_jans_dir, 'log4j2-adminui.xml')133 self.log4j2_path = os.path.join(Config.dist_jans_dir, 'log4j2.xml')134 self.admin_ui_plugin_source_path = os.path.join(Config.dist_jans_dir, 'admin-ui-plugin.jar')135 self.flex_path = os.path.join(Config.dist_jans_dir, 'flex.zip')136 self.source_dir = os.path.join(Config.install_dir, 'flex')137 self.flex_setup_dir = os.path.join(self.source_dir, 'flex-linux-setup')138 self.templates_dir = os.path.join(self.flex_setup_dir, 'templates')139 self.admin_ui_config_properties_path = os.path.join(self.templates_dir, 'auiConfiguration.properties')140 self.casa_dist_dir = os.path.join(Config.dist_jans_dir, 'gluu-casa')141 self.casa_web_resources_fn = os.path.join(self.casa_dist_dir, 'casa_web_resources.xml')142 self.casa_war_fn = os.path.join(self.casa_dist_dir, 'casa.war')143 self.casa_config_fn = os.path.join(self.casa_dist_dir,'casa-config.jar')144 self.casa_script_fn = os.path.join(self.casa_dist_dir,'Casa.py')145 self.twillo_fn = os.path.join(self.casa_dist_dir,'twilio.jar')146 self.py_lib_dir = os.path.join(Config.jansOptPythonFolder, 'libs')147 self.fido2_client_jar_fn = os.path.join(Config.dist_jans_dir, 'jans-fido2-client.jar')148 self.dbUtils.bind(force=True)149 if os.path.exists(self.source_dir):150 os.rename(self.source_dir, self.source_dir+'-'+time.ctime().replace(' ', '_'))151 def download_files(self):152 print("Downloading components")153 base.download('https:/​/​github.com/​GluuFederation/​flex/​archive/​refs/​heads/​{}.zip'.format(app_versions['FLEX_BRANCH']), self.flex_path, verbose=True)154 print("Extracting", self.flex_path)155 base.extract_from_zip(self.flex_path, 'flex-linux-setup/​flex_linux_setup', self.flex_setup_dir)156 if install_components['admin_ui']:157 base.download(urljoin(maven_base_url, 'jans-config-api/​plugins/​admin-ui-plugin/​{0}{1}/​admin-ui-plugin-{0}{1}-distribution.jar'.format(app_versions['JANS_APP_VERSION'], app_versions['JANS_BUILD'])), self.admin_ui_plugin_source_path, verbose=True)158 base.download('https:/​/​raw.githubusercontent.com/​JanssenProject/​jans/​{}/​jans-config-api/​server/​src/​main/​resources/​log4j2.xml'.format(app_versions['JANS_BRANCH']), self.log4j2_path, verbose=True)159 base.download('https:/​/​raw.githubusercontent.com/​JanssenProject/​jans/​{}/​jans-config-api/​plugins/​admin-ui-plugin/​config/​log4j2-adminui.xml'.format(app_versions['JANS_BRANCH']), self.log4j2_adminui_path, verbose=True)160 161 if install_components['casa']:162 base.download('https:/​/​raw.githubusercontent.com/​GluuFederation/​flex/​main/​casa/​extras/​casa_web_resources.xml', self.casa_web_resources_fn, verbose=True)163 base.download('https:/​/​maven.gluu.org/​maven/​org/​gluu/​casa/​{0}/​casa-{0}.war'.format(app_versions['CASA_VERSION']), self.casa_war_fn, verbose=True)164 base.download('https:/​/​maven.gluu.org/​maven/​org/​gluu/​casa-config/​{0}/​casa-config-{0}.jar'.format(app_versions['CASA_VERSION']), self.casa_config_fn, verbose=True)165 base.download('https:/​/​repo1.maven.org/​maven2/​com/​twilio/​sdk/​twilio/​{0}/​twilio-{0}.jar'.format(app_versions['TWILIO_VERSION']), self.twillo_fn, verbose=True)166 base.download('https:/​/​raw.githubusercontent.com/​GluuFederation/​flex/​main/​casa/​extras/​Casa.py', self.casa_script_fn, verbose=True)167 base.download('https:/​/​raw.githubusercontent.com/​GluuFederation/​flex/​main/​casa/​extras/​casa-external_fido2.py', os.path.join(self.casa_dist_dir, 'pylib/​casa-external_fido2.py'), verbose=True)168 base.download('https:/​/​raw.githubusercontent.com/​GluuFederation/​flex/​main/​casa/​extras/​casa-external_otp.py', os.path.join(self.casa_dist_dir, 'pylib/​casa-external_otp.py'), verbose=True)169 base.download('https:/​/​raw.githubusercontent.com/​GluuFederation/​flex/​main/​casa/​extras/​casa-external_super_gluu.py', os.path.join(self.casa_dist_dir, 'pylib/​casa-external_super_gluu.py'), verbose=True)170 base.download('https:/​/​raw.githubusercontent.com/​GluuFederation/​flex/​main/​casa/​extras/​casa-external_twilio_sms.py', os.path.join(self.casa_dist_dir, 'pylib/​casa-external_twilio_sms.py'), verbose=True)171 base.download('https:/​/​maven.jans.io/​maven/​io/​jans/​jans-fido2-client/​{0}{1}/​jans-fido2-client-{0}{1}.jar'.format(app_versions['JANS_APP_VERSION'], app_versions['JANS_BUILD']), self.fido2_client_jar_fn, verbose=True)172 def add_apache_directive(self, check_str, template):173 print("Updating apache configuration")174 apache_directive_template_text = self.readFile(os.path.join(self.templates_dir, template))175 apache_directive_text = self.fomatWithDict(apache_directive_template_text, Config.templateRenderingDict)176 https_jans_text = self.readFile(httpd_installer.https_jans_fn)177 if check_str not in https_jans_text:178 https_jans_list = https_jans_text.splitlines()179 n = 0180 for i, l in enumerate(https_jans_list):181 if l.strip() == '</​LocationMatch>':182 n = i183 https_jans_list.insert(n+1, '\n' + apache_directive_text + '\n')184 self.writeFile(httpd_installer.https_jans_fn, '\n'.join(https_jans_list))185 self.enable_apache_mod_dir()186 def enable_apache_mod_dir(self):187 # Enable mod_dir for apache188 cmd_a2enmod = shutil.which('a2enmod')189 if base.clone_type == 'deb':190 httpd_installer.run([cmd_a2enmod, 'dir'])191 elif base.os_type == 'suse':192 httpd_installer.run([cmd_a2enmod, 'dir'])193 cmd_a2enflag = shutil.which('a2enflag')194 httpd_installer.run([cmd_a2enflag, 'SSL'])195 else:196 base_mod_path = Path('/​etc/​httpd/​conf.modules.d/​00-base.conf')197 mod_load_content = base_mod_path.read_text().splitlines()198 modified = False199 for i, l in enumerate(mod_load_content[:]):200 ls = l.strip()201 if ls.startswith('#') and ls.endswith('mod_dir.so'):202 mod_load_content[i] = ls.lstrip('#')203 modified = True204 if modified:205 base_mod_path.write_text('\n'.join(mod_load_content))206 def install_gluu_admin_ui(self):207 print("Installing Gluu Admin UI Frontend")208 print("Extracting admin-ui from", self.flex_path)209 base.extract_from_zip(self.flex_path, 'admin-ui', self.source_dir)210 print("Source directory:", self.source_dir)211 env_tmp = os.path.join(self.source_dir, '.env.tmp')212 print("env_tmp", env_tmp)213 config_api_installer.renderTemplateInOut(env_tmp, self.source_dir, self.source_dir)214 config_api_installer.copyFile(os.path.join(self.source_dir, '.env.tmp'), os.path.join(self.source_dir, '.env'))215 config_api_installer.run([paths.cmd_chown, '-R', 'node:node', self.source_dir])216 cmd_path = 'PATH=$PATH:{}/​bin:{}/​bin'.format(Config.jre_home, Config.node_home)217 for cmd in ('npm install @openapitools/​openapi-generator-cli', 'npm run api', 'npm install', 'npm run build:prod'):218 print("Executing `{}`".format(cmd))219 run_cmd = '{} {}'.format(cmd_path, cmd)220 config_api_installer.run(['/​bin/​su', 'node','-c', run_cmd], self.source_dir)221 Config.templateRenderingDict['admin_ui_apache_root'] = os.path.join(httpd_installer.server_root, 'admin')222 self.add_apache_directive(Config.templateRenderingDict['admin_ui_apache_root'], 'admin_ui_apache_directive')223 print("Copying files to", Config.templateRenderingDict['admin_ui_apache_root'])224 config_api_installer.copy_tree(os.path.join(self.source_dir, 'dist'), Config.templateRenderingDict['admin_ui_apache_root'])225 config_api_installer.check_clients([('role_based_client_id', '2000.')])226 config_api_installer.renderTemplateInOut(self.admin_ui_config_properties_path, os.path.join(self.flex_setup_dir, 'templates'), config_api_installer.custom_config_dir)227 admin_ui_plugin_path = os.path.join(config_api_installer.libDir, os.path.basename(self.admin_ui_plugin_source_path))228 config_api_installer.copyFile(self.admin_ui_plugin_source_path, config_api_installer.libDir)229 config_api_installer.add_extra_class(admin_ui_plugin_path)230 for logfn in (self.log4j2_adminui_path, self.log4j2_path):231 config_api_installer.copyFile(logfn, config_api_installer.custom_config_dir)232 cli_config = Path(jans_cli_installer.config_ini_fn)233 current_plugins = []234 if cli_config.exists():235 config = configparser.ConfigParser()236 config.read_file(cli_config.open())237 current_plugins = config['DEFAULT'].get('jca_plugins', '').split(',')238 plugins = config_api_installer.get_plugins()239 for plugin in plugins:240 if plugin not in current_plugins:241 current_plugins.append(plugin)242 config['DEFAULT']['jca_plugins'] = ','.join(current_plugins)243 config.write(cli_config.open('w'))244 cli_config.chmod(0o600)245 def install_casa(self):246 print("Adding twillo and casa config to jans-auth")247 self.copyFile(self.casa_config_fn, self.jans_auth_custom_lib_dir)248 self.copyFile(self.twillo_fn, self.jans_auth_custom_lib_dir)249 class_path = '{},{}'.format(250 os.path.join(self.jans_auth_custom_lib_dir, os.path.basename(self.casa_config_fn)),251 os.path.join(self.jans_auth_custom_lib_dir, os.path.basename(self.twillo_fn)),252 )253 jansAuthInstaller.add_extra_class(class_path)254 print("Adding Fido2 Client lib to jans-auth")255 self.copyFile(self.fido2_client_jar_fn, self.jans_auth_custom_lib_dir)256 class_path = os.path.join(self.jans_auth_custom_lib_dir, os.path.basename(self.fido2_client_jar_fn))257 jansAuthInstaller.add_extra_class(class_path)258 simple_auth_scr_inum = 'A51E-76DA'259 print("Enabling script", simple_auth_scr_inum)260 self.dbUtils.enable_script(simple_auth_scr_inum)261 # copy casa scripts262 if not os.path.exists(self.py_lib_dir):263 os.makedirs(self.py_lib_dir)264 for fn in glob.glob(os.path.join(self.casa_dist_dir, 'pylib/​*.py')):265 print("Copying", fn, "to", self.py_lib_dir)266 self.copyFile(fn, self.py_lib_dir)267 self.run([paths.cmd_chown, '-R', '{0}:{0}'.format(Config.jetty_user), self.py_lib_dir])268 # prepare casa scipt ldif269 casa_auth_script_fn = os.path.join(self.templates_dir, 'casa_person_authentication_script.ldif')270 base64_script_file = self.generate_base64_file(self.casa_script_fn, 1)271 Config.templateRenderingDict['casa_person_authentication_script'] = base64_script_file272 self.renderTemplateInOut(casa_auth_script_fn, self.templates_dir, self.source_dir)273 Config.templateRenderingDict['casa_redirect_uri'] = 'https:/​/​{}/​casa'.format(Config.hostname)274 Config.templateRenderingDict['casa_redirect_logout_uri'] = 'https:/​/​{}/​casa/​bye.zul'.format(Config.hostname)275 Config.templateRenderingDict['casa_frontchannel_logout_uri'] = 'https:/​/​{}/​casa/​autologout'.format(Config.hostname)276 Config.templateRenderingDict['casa_web_port'] = '8080'277 self.casa_client_fn = os.path.join(self.source_dir, 'templates/​casa_client.ldif')278 self.casa_config_fn = os.path.join(self.source_dir, 'templates/​casa_config.ldif')279 self.service_name = 'casa'280 for casa_prop in ('casa_client_id', 'casa_client_pw'):281 if casa_prop in setup_properties:282 setattr(Config, casa_prop, setup_properties[casa_prop])283 self.check_clients([('casa_client_id', '3000.')])284 if not Config.get('casa_client_encoded_pw'):285 Config.casa_client_encoded_pw = jansAuthInstaller.obscure(Config.casa_client_pw)286 print("Casa client id", Config.casa_client_id)287 print("Casa client encoded password", Config.casa_client_encoded_pw)288 print("Importing LDIF Files")289 self.renderTemplateInOut(self.casa_client_fn, self.templates_dir, self.source_dir)290 self.renderTemplateInOut(self.casa_config_fn, self.templates_dir, self.source_dir)291 self.dbUtils.import_ldif([292 os.path.join(self.source_dir, os.path.basename(self.casa_client_fn)),293 os.path.join(self.source_dir, os.path.basename(self.casa_config_fn)),294 os.path.join(self.source_dir, os.path.basename(casa_auth_script_fn)),295 ])296 Config.installCasa = True297 self.copyFile(os.path.join(self.templates_dir, 'casa.default'), os.path.join(Config.templateFolder, 'jetty/​casa'))298 self.jetty_app_configuration[self.service_name] = {299 "memory": {300 "max_allowed_mb": 1024,301 "metaspace_mb": 128,302 "jvm_heap_ration": 0.7,303 "ratio": 0.1304 },305 "jetty": {306 "modules": "server,deploy,resources,http,http-forwarded,console-capture,jsp,cdi-decorate"307 },308 "installed": False,309 "name": self.service_name310 }311 print("Calculating application memory")312 installedComponents = []313 # Jetty apps314 for config_var, service in [('installOxAuth', 'jans-auth'),315 ('installScimServer', 'jans-scim'),316 ('installFido2', 'jans-fido2'),317 ('installConfigApi', 'jans-config-api'),318 ('installEleven', 'jans-eleven'),319 ('installCasa', self.service_name),320 ]:321 if Config.get(config_var) and service in self.jetty_app_configuration:322 installedComponents.append(self.jetty_app_configuration[service])323 self.calculate_aplications_memory(Config.application_max_ram, self.jetty_app_configuration, installedComponents)324 print("Deploying casa as Jetty application")325 self.installJettyService(self.jetty_app_configuration[self.service_name], True)326 self.copyFile(os.path.join(self.templates_dir, 'casa.service'), '/​etc/​systemd/​system')327 jetty_service_dir = os.path.join(Config.jetty_base, self.service_name)328 jetty_service_webapps_dir = os.path.join(jetty_service_dir, 'webapps')329 self.run([paths.cmd_mkdir, '-p', os.path.join(jetty_service_dir, 'static')])330 self.run([paths.cmd_mkdir, '-p', os.path.join(jetty_service_dir, 'plugins')])331 self.copyFile(self.casa_war_fn, jetty_service_webapps_dir)332 self.copyFile(self.casa_web_resources_fn, jetty_service_webapps_dir)333 jansAuthInstaller.chown(jetty_service_dir, Config.jetty_user, Config.jetty_group, recursive=True)334 jansAuthInstaller.chown(self.jans_auth_custom_lib_dir, Config.jetty_user, Config.jetty_group, recursive=True)335 self.add_apache_directive('<Location /​casa>', 'casa_apache_directive')336 self.enable()337 def save_properties(self):338 fn = Config.savedProperties339 print("Saving properties", fn)340 if os.path.exists(fn):341 p = Properties()342 with open(fn, 'rb') as f:343 p.load(f, 'utf-8')344 for prop in ('casa_client_id', 'casa_client_pw', 'casa_client_encoded_pw'):345 if Config.get(prop):346 p[prop] = Config.get(prop)347 with open(fn, 'wb') as f:348 p.store(f, encoding="utf-8")349 else:350 propertiesUtils.save_properties()351def prompt_for_installation():352 if not os.path.exists(os.path.join(httpd_installer.server_root, 'admin')):353 prompt_admin_ui_install = input("Install Admin UI [Y/​n]: ")354 if prompt_admin_ui_install and prompt_admin_ui_install.lower().startswith('y'):355 install_components['admin_ui'] = True356 else:357 print("Admin UI is allready installed on this system")358 install_components['admin_ui'] = False359 if not os.path.exists(os.path.join(Config.jetty_base, 'casa')):360 prompt_casa_install = input("Install Casa [Y/​n]: ")361 if prompt_casa_install and prompt_casa_install.lower().startswith('y'):362 install_components['casa'] = True363 else:364 print("Casa is allready installed on this system")365 install_components['casa'] = False366 if not (install_components['casa'] or install_components['admin_ui']):367 print("Nothing to install. Exiting ...")368 sys.exit()369def main():370 if not argsp.flex_non_interactive:371 prompt_for_installation()372 if install_components['admin_ui'] and not node_installer.installed():373 node_fn = 'node-{0}-linux-x64.tar.xz'.format(app_versions['NODE_VERSION'])374 node_path = os.path.join(Config.dist_app_dir, node_fn)375 if not os.path.exists(node_path):376 base.download('https:/​/​nodejs.org/​dist/​{0}/​node-{0}-linux-x64.tar.xz'.format(app_versions['NODE_VERSION']), node_path, verbose=True)377 print("Installing node")378 node_installer.install()379 installer_obj = flex_installer()380 installer_obj.download_files()381 if install_components['admin_ui']:382 installer_obj.install_gluu_admin_ui()383 if install_components['casa']:384 installer_obj.install_casa()385 installer_obj.save_properties()386 print("Restarting Apache")387 httpd_installer.restart()388 print("Restarting Jans Auth")389 config_api_installer.restart('jans-auth')390 print("Restarting Janssen Config Api")391 config_api_installer.restart()392 if install_components['casa']:393 print("Starting Casa")394 config_api_installer.start('casa')395 print("Installation was completed.")396 if install_components['admin_ui']:397 print("Browse https:/​/​{}/​admin".format(Config.hostname))398 if install_components['casa']:399 print("Browse https:/​/​{}/​casa".format(Config.hostname))400if __name__ == "__main__":401 if argsp.shell:402 code.interact(local=locals())403 sys.exit()404 else:...

Full Screen

Full Screen

intelbase.py

Source:intelbase.py Github

copy

Full Screen

1# #2# Copyright 2009-2019 Ghent University3#4# This file is part of EasyBuild,5# originally created by the HPC team of Ghent University (http:/​/​ugent.be/​hpc/​en),6# with support of Ghent University (http:/​/​ugent.be/​hpc),7# the Flemish Supercomputer Centre (VSC) (https:/​/​www.vscentrum.be),8# Flemish Research Foundation (FWO) (http:/​/​www.fwo.be/​en)9# and the Department of Economy, Science and Innovation (EWI) (http:/​/​www.ewi-vlaanderen.be/​en).10#11# https:/​/​github.com/​easybuilders/​easybuild12#13# EasyBuild is free software: you can redistribute it and/​or modify14# it under the terms of the GNU General Public License as published by15# the Free Software Foundation v2.16#17# EasyBuild is distributed in the hope that it will be useful,18# but WITHOUT ANY WARRANTY; without even the implied warranty of19# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the20# GNU General Public License for more details.21#22# You should have received a copy of the GNU General Public License23# along with EasyBuild. If not, see <http:/​/​www.gnu.org/​licenses/​>.24# #25"""26Generic EasyBuild support for installing Intel tools, implemented as an easyblock27@author: Stijn De Weirdt (Ghent University)28@author: Dries Verdegem (Ghent University)29@author: Kenneth Hoste (Ghent University)30@author: Pieter De Baets (Ghent University)31@author: Jens Timmerman (Ghent University)32@author: Ward Poelmans (Ghent University)33@author: Lumir Jasiok (IT4Innovations)34@author: Damian Alvarez (Forschungszentrum Juelich GmbH)35"""36import os37import re38import shutil39import tempfile40import glob41from distutils.version import LooseVersion42import easybuild.tools.environment as env43from easybuild.framework.easyblock import EasyBlock44from easybuild.framework.easyconfig import CUSTOM45from easybuild.framework.easyconfig.types import ensure_iterable_license_specs46from easybuild.tools.build_log import EasyBuildError47from easybuild.tools.filetools import find_flexlm_license, read_file, remove_file48from easybuild.tools.run import run_cmd49from vsc.utils import fancylogger50_log = fancylogger.getLogger('generic.intelbase')51# different supported activation types (cfr. Intel documentation)52ACTIVATION_EXIST_LIC = 'exist_lic' # use a license which exists on the system53ACTIVATION_LIC_FILE = 'license_file' # use a license file54ACTIVATION_LIC_SERVER = 'license_server' # use a license server55ACTIVATION_SERIAL = 'serial_number' # use a serial number56ACTIVATION_TRIAL = 'trial_lic' # use trial activation57ACTIVATION_TYPES = [58 ACTIVATION_EXIST_LIC,59 ACTIVATION_LIC_FILE,60 ACTIVATION_LIC_SERVER,61 ACTIVATION_SERIAL,62 ACTIVATION_TRIAL,63]64# silent.cfg parameter name for type of license activation (cfr. options listed above)65ACTIVATION_NAME = 'ACTIVATION_TYPE' # since icc/​ifort v2013_sp1, impi v4.1.1, imkl v11.166ACTIVATION_NAME_2012 = 'ACTIVATION' # previous activation type parameter used in older versions67# silent.cfg parameter name for install prefix68INSTALL_DIR_NAME = 'PSET_INSTALL_DIR'69# silent.cfg parameter name for install mode70INSTALL_MODE_NAME = 'PSET_MODE'71# Older (2015 and previous) silent.cfg parameter name for install mode72INSTALL_MODE_NAME_2015 = 'INSTALL_MODE'73# Install mode for 2016 version74INSTALL_MODE = 'install'75# Install mode for 2015 and older versions76INSTALL_MODE_2015 = 'NONRPM'77# silent.cfg parameter name for license file/​server specification78LICENSE_FILE_NAME = 'ACTIVATION_LICENSE_FILE' # since icc/​ifort v2013_sp1, impi v4.1.1, imkl v11.179LICENSE_FILE_NAME_2012 = 'PSET_LICENSE_FILE' # previous license file parameter used in older versions80LICENSE_SERIAL_NUMBER = 'ACTIVATION_SERIAL_NUMBER'81COMP_ALL = 'ALL'82COMP_DEFAULTS = 'DEFAULTS'83class IntelBase(EasyBlock):84 """85 Base class for Intel software86 - no configure/​make : binary release87 - add license_file variable88 """89 def __init__(self, *args, **kwargs):90 """Constructor, adds extra config options"""91 super(IntelBase, self).__init__(*args, **kwargs)92 self.license_file = 'UNKNOWN'93 self.license_env_var = 'UNKNOWN'94 # Initialise whether we need a runtime licence or not95 self.requires_runtime_license = True96 self.home_subdir = os.path.join(os.getenv('HOME'), 'intel')97 common_tmp_dir = os.path.dirname(tempfile.gettempdir()) # common tmp directory, same across nodes98 self.home_subdir_local = os.path.join(common_tmp_dir, os.getenv('USER'), 'easybuild_intel')99 self.install_components = None100 def get_guesses_tools(self):101 """Find reasonable paths for a subset of Intel tools, ignoring CPATH, LD_LIBRARY_PATH and LIBRARY_PATH"""102 guesses = super(IntelBase, self).make_module_req_guess()103 if self.cfg['m32']:104 guesses['PATH'] = [os.path.join(self.subdir, 'bin32')]105 else:106 guesses['PATH'] = [os.path.join(self.subdir, 'bin64')]107 guesses['MANPATH'] = [os.path.join(self.subdir, 'man')]108 # make sure $CPATH, $LD_LIBRARY_PATH and $LIBRARY_PATH are not updated in generated module file,109 # because that leads to problem when the libraries included with VTune/​Advisor/​Inspector are being picked up110 for key in ['CPATH', 'LD_LIBRARY_PATH', 'LIBRARY_PATH']:111 if key in guesses:112 self.log.debug("Purposely not updating $%s in %s module file", key, self.name)113 del guesses[key]114 return guesses115 def get_custom_paths_tools(self, binaries):116 """Custom sanity check paths for certain Intel tools."""117 if self.cfg['m32']:118 files = [os.path.join('bin32', b) for b in binaries]119 dirs = ['lib32', 'include']120 else:121 files = [os.path.join('bin64', b) for b in binaries]122 dirs = ['lib64', 'include']123 custom_paths = {124 'files': [os.path.join(self.subdir, f) for f in files],125 'dirs': [os.path.join(self.subdir, d) for d in dirs],126 }127 return custom_paths128 @staticmethod129 def extra_options(extra_vars=None):130 extra_vars = EasyBlock.extra_options(extra_vars)131 extra_vars.update({132 'license_activation': [ACTIVATION_LIC_SERVER, "License activation type", CUSTOM],133 'serial_number': [None, "Serial number for the product", CUSTOM],134 'requires_runtime_license': [True, "Boolean indicating whether or not a runtime license is required",135 CUSTOM],136 # 'usetmppath':137 # workaround for older SL5 version (5.5 and earlier)138 # used to be True, but False since SL5.6/​SL6139 # disables TMP_PATH env and command line option140 'usetmppath': [False, "Use temporary path for installation", CUSTOM],141 'm32': [False, "Enable 32-bit toolchain", CUSTOM],142 'components': [None, "List of components to install", CUSTOM],143 })144 return extra_vars145 def parse_components_list(self):146 """parse the regex in the components extra_options and select the matching components147 from the mediaconfig.xml file in the install dir"""148 mediaconfigpath = os.path.join(self.cfg['start_dir'], 'pset', 'mediaconfig.xml')149 if not os.path.isfile(mediaconfigpath):150 raise EasyBuildError("Could not find %s to find list of components." % mediaconfigpath)151 mediaconfig = read_file(mediaconfigpath)152 available_components = re.findall("<Abbr>(?P<component>[^<]+)</​Abbr>", mediaconfig, re.M)153 self.log.debug("Intel components found: %s" % available_components)154 self.log.debug("Using regex list: %s" % self.cfg['components'])155 if COMP_ALL in self.cfg['components'] or COMP_DEFAULTS in self.cfg['components']:156 if len(self.cfg['components']) == 1:157 self.install_components = self.cfg['components']158 else:159 raise EasyBuildError("If you specify %s as components, you cannot specify anything else: %s",160 ' or '.join([COMP_ALL, COMP_DEFAULTS]), self.cfg['components'])161 else:162 self.install_components = []163 for comp_regex in self.cfg['components']:164 comps = [comp for comp in available_components if re.match(comp_regex, comp)]165 self.install_components.extend(comps)166 self.log.debug("Components to install: %s" % self.install_components)167 def clean_home_subdir(self):168 """Remove contents of (local) 'intel' directory home subdir, where stuff is cached."""169 self.log.debug("Cleaning up %s..." % self.home_subdir_local)170 try:171 for tree in os.listdir(self.home_subdir_local):172 self.log.debug("... removing %s subtree" % tree)173 path = os.path.join(self.home_subdir_local, tree)174 if os.path.isfile(path) or os.path.islink(path):175 remove_file(path)176 else:177 shutil.rmtree(path)178 except OSError as err:179 raise EasyBuildError("Cleaning up intel dir %s failed: %s", self.home_subdir_local, err)180 def setup_local_home_subdir(self):181 """182 Intel script use $HOME/​intel to cache stuff.183 To enable parallel builds, we symlink $HOME/​intel to a temporary dir on the local disk."""184 try:185 # make sure local directory exists186 if not os.path.exists(self.home_subdir_local):187 os.makedirs(self.home_subdir_local)188 self.log.debug("Created local dir %s" % self.home_subdir_local)189 if os.path.exists(self.home_subdir):190 # if 'intel' dir in $HOME already exists, make sure it's the right symlink191 symlink_ok = os.path.islink(self.home_subdir) and os.path.samefile(self.home_subdir,192 self.home_subdir_local)193 if not symlink_ok:194 # rename current 'intel' dir195 home_intel_bk = tempfile.mkdtemp(dir=os.path.dirname(self.home_subdir),196 prefix='%s.bk.' % os.path.basename(self.home_subdir))197 self.log.info("Moving %(ih)s to %(ihl)s, I need %(ih)s myself..." % {'ih': self.home_subdir,198 'ihl': home_intel_bk})199 shutil.move(self.home_subdir, home_intel_bk)200 # set symlink in place201 os.symlink(self.home_subdir_local, self.home_subdir)202 self.log.debug("Created symlink (1) %s to %s" % (self.home_subdir, self.home_subdir_local))203 else:204 # if a broken symlink is present, remove it first205 if os.path.islink(self.home_subdir):206 remove_file(self.home_subdir)207 os.symlink(self.home_subdir_local, self.home_subdir)208 self.log.debug("Created symlink (2) %s to %s" % (self.home_subdir, self.home_subdir_local))209 except OSError as err:210 raise EasyBuildError("Failed to symlink %s to %s: %s", self.home_subdir_local, self.home_subdir, err)211 def prepare_step(self, *args, **kwargs):212 """Custom prepare step for IntelBase. Set up the license"""213 requires_runtime_license = kwargs.pop('requires_runtime_license', True)214 super(IntelBase, self).prepare_step(*args, **kwargs)215 # Decide if we need a license or not (default is True because of defaults of individual Booleans)216 self.requires_runtime_license = self.cfg['requires_runtime_license'] and requires_runtime_license217 self.serial_number = self.cfg['serial_number']218 if self.serial_number:219 self.log.info("Using provided serial number (%s) and ignoring other licenses", self.serial_number)220 elif self.requires_runtime_license:221 default_lic_env_var = 'INTEL_LICENSE_FILE'222 license_specs = ensure_iterable_license_specs(self.cfg['license_file'])223 lic_specs, self.license_env_var = find_flexlm_license(custom_env_vars=[default_lic_env_var],224 lic_specs=license_specs)225 if lic_specs:226 if self.license_env_var is None:227 self.log.info("Using Intel license specifications from 'license_file': %s", lic_specs)228 self.license_env_var = default_lic_env_var229 else:230 self.log.info("Using Intel license specifications from $%s: %s", self.license_env_var, lic_specs)231 self.license_file = os.pathsep.join(lic_specs)232 env.setvar(self.license_env_var, self.license_file)233 # if we have multiple retained lic specs, specify to 'use a license which exists on the system'234 if len(lic_specs) > 1:235 self.log.debug("More than one license specs found, using '%s' license activation instead of "236 "'%s'", ACTIVATION_EXIST_LIC, self.cfg['license_activation'])237 self.cfg['license_activation'] = ACTIVATION_EXIST_LIC238 # $INTEL_LICENSE_FILE should always be set during installation with existing license239 env.setvar(default_lic_env_var, self.license_file)240 else:241 msg = "No viable license specifications found; "242 msg += "specify 'license_file', or define $INTEL_LICENSE_FILE or $LM_LICENSE_FILE"243 raise EasyBuildError(msg)244 def configure_step(self):245 """Configure: handle license file and clean home dir."""246 # prepare (local) 'intel' home subdir247 self.setup_local_home_subdir()248 self.clean_home_subdir()249 # determine list of components, based on 'components' easyconfig parameter (if specified)250 if self.cfg['components']:251 self.parse_components_list()252 else:253 self.log.debug("No components specified")254 def build_step(self):255 """Binary installation files, so no building."""256 pass257 def install_step(self, silent_cfg_names_map=None, silent_cfg_extras=None):258 """Actual installation259 - create silent cfg file260 - set environment parameters261 - execute command262 """263 if silent_cfg_names_map is None:264 silent_cfg_names_map = {}265 if self.serial_number or self.requires_runtime_license:266 lic_entry = ""267 if self.serial_number:268 lic_entry = "%(license_serial_number)s=%(serial_number)s"269 self.cfg['license_activation'] = ACTIVATION_SERIAL270 else:271 # license file entry is only applicable with license file or server type of activation272 # also check whether specified activation type makes sense273 lic_file_server_activations = [ACTIVATION_LIC_FILE, ACTIVATION_LIC_SERVER]274 other_activations = [act for act in ACTIVATION_TYPES if act not in lic_file_server_activations]275 if self.cfg['license_activation'] in lic_file_server_activations:276 lic_entry = "%(license_file_name)s=%(license_file)s"277 elif not self.cfg['license_activation'] in other_activations:278 raise EasyBuildError("Unknown type of activation specified: %s (known :%s)",279 self.cfg['license_activation'], ACTIVATION_TYPES)280 silent = '\n'.join([281 "%(activation_name)s=%(activation)s",282 lic_entry,283 "" # Add a newline at the end, so we can easily append if needed284 ]) % {285 'activation_name': silent_cfg_names_map.get('activation_name', ACTIVATION_NAME),286 'activation': self.cfg['license_activation'],287 'license_file_name': silent_cfg_names_map.get('license_file_name', LICENSE_FILE_NAME),288 'license_file': self.license_file,289 'license_serial_number': silent_cfg_names_map.get('license_serial_number', LICENSE_SERIAL_NUMBER),290 'serial_number': self.serial_number,291 }292 else:293 self.log.debug("No license required, so not including license specifications in silent.cfg")294 silent = ''295 silent += '\n'.join([296 "%(install_dir_name)s=%(install_dir)s",297 "ACCEPT_EULA=accept",298 "%(install_mode_name)s=%(install_mode)s",299 "CONTINUE_WITH_OPTIONAL_ERROR=yes",300 "" # Add a newline at the end, so we can easily append if needed301 ]) % {302 'install_dir_name': silent_cfg_names_map.get('install_dir_name', INSTALL_DIR_NAME),303 'install_dir': silent_cfg_names_map.get('install_dir', self.installdir),304 'install_mode': silent_cfg_names_map.get('install_mode', INSTALL_MODE_2015),305 'install_mode_name': silent_cfg_names_map.get('install_mode_name', INSTALL_MODE_NAME_2015),306 }307 if self.install_components is not None:308 if len(self.install_components) == 1 and self.install_components[0] in [COMP_ALL, COMP_DEFAULTS]:309 # no quotes should be used for ALL or DEFAULTS310 silent += 'COMPONENTS=%s\n' % self.install_components[0]311 elif self.install_components:312 # a list of components is specified (needs quotes)313 components = ';'.join(self.install_components)314 if LooseVersion(self.version) >= LooseVersion('2017'):315 # for versions 2017.x and newer, double quotes should not be there...316 silent += 'COMPONENTS=%s\n' % components317 else:318 silent += 'COMPONENTS="%s"\n' % components319 else:320 raise EasyBuildError("Empty list of matching components obtained via %s", self.cfg['components'])321 if silent_cfg_extras is not None:322 if isinstance(silent_cfg_extras, dict):323 silent += '\n'.join("%s=%s" % (key, value) for (key, value) in silent_cfg_extras.iteritems())324 else:325 raise EasyBuildError("silent_cfg_extras needs to be a dict")326 # we should be already in the correct directory327 silentcfg = os.path.join(os.getcwd(), "silent.cfg")328 try:329 f = open(silentcfg, 'w')330 f.write(silent)331 f.close()332 except:333 raise EasyBuildError("Writing silent cfg, failed", silent)334 self.log.debug("Contents of %s:\n%s" % (silentcfg, silent))335 # workaround for mktmp: create tmp dir and use it336 tmpdir = os.path.join(self.cfg['start_dir'], 'mytmpdir')337 try:338 os.makedirs(tmpdir)339 except:340 raise EasyBuildError("Directory %s can't be created", tmpdir)341 tmppathopt = ''342 if self.cfg['usetmppath']:343 env.setvar('TMP_PATH', tmpdir)344 tmppathopt = "-t %s" % tmpdir345 # set some extra env variables346 env.setvar('LOCAL_INSTALL_VERBOSE', '1')347 env.setvar('VERBOSE_MODE', '1')348 env.setvar('INSTALL_PATH', self.installdir)349 # perform installation350 cmd = ' '.join([351 self.cfg['preinstallopts'],352 './​install.sh',353 tmppathopt,354 '-s ' + silentcfg,355 self.cfg['installopts'],356 ])357 return run_cmd(cmd, log_all=True, simple=True, log_output=True)358 def move_after_install(self):359 """Move installed files to correct location after installation."""360 subdir = os.path.join(self.installdir, self.name, self.version)361 self.log.debug("Moving contents of %s to %s" % (subdir, self.installdir))362 try:363 # remove senseless symlinks, e.g. impi_5.0.1 and impi_latest364 majver = '.'.join(self.version.split('.')[:-1])365 for symlink in ['%s_%s' % (self.name, majver), '%s_latest' % self.name]:366 symlink_fp = os.path.join(self.installdir, symlink)367 if os.path.exists(symlink_fp):368 remove_file(symlink_fp)369 # move contents of 'impi/​<version>' dir to installdir370 for fil in os.listdir(subdir):371 source = os.path.join(subdir, fil)372 target = os.path.join(self.installdir, fil)373 self.log.debug("Moving %s to %s" % (source, target))374 shutil.move(source, target)375 shutil.rmtree(os.path.join(self.installdir, self.name))376 except OSError as err:377 raise EasyBuildError("Failed to move contents of %s to %s: %s", subdir, self.installdir, err)378 def sanity_check_rpath(self):379 """Skip the rpath sanity check, this is binary software"""380 self.log.info("RPATH sanity check is skipped when using %s easyblock (derived from IntelBase)",381 self.__class__.__name__)382 def make_module_extra(self, *args, **kwargs):383 """Custom variable definitions in module file."""384 txt = super(IntelBase, self).make_module_extra(*args, **kwargs)385 if self.requires_runtime_license:386 txt += self.module_generator.prepend_paths(self.license_env_var, [self.license_file],387 allow_abs=True, expand_relpaths=False)388 return txt389 def cleanup_step(self):390 """Cleanup leftover mess391 - clean home dir392 - generic cleanup (get rid of build dir)393 """394 self.clean_home_subdir()395 super(IntelBase, self).cleanup_step()...

Full Screen

Full Screen

fabfile.py

Source:fabfile.py Github

copy

Full Screen

...43 44 # 安装所有本地的依赖包45 if local('/​bin/​bash install.sh').failed:46 abort('安装本地python packages失败。')47def install_components(pattern, install_path):48 """49 从Download_Path 查找符合pattern的第一个压缩包, 解压之后移动到install_path50 """51 tarfile_path = find_tarfile(opath.join(Download_Path, pattern)) # ./​pip-cache/​$pattern-x.x.x.tar.gz ,find ./​pip-cache -name pattern-*.tar.gz52 extract_path = find_tarfile_extract_path(tarfile_path) # pattern-x.x.x, tar xzf x.tar.gz | head -n153 54 if extract_path:55 extract_result = local("tar xzf %s" % (tarfile_path, ))56 57 if extract_result.failed:58 abort('cant not extract %s' % tarfile_path)59 60 if local("mv %s %s" % (extract_path, install_path)).failed:61 abort('can not move %s from %s to %s' % (pattern, extract_path, install_path))62def extract_swagger():63 install_components('swagger-*.tar.gz', 'nebula/​middleware/​tornado_rest_swagger/​assets')64def extract_grafana():65 install_components('grafana-*.tar.gz', 'nebula/​grafana_app')66def dep_web_front():67 pattern = 'nebula_web_frontend-*.tar.gz'68 tarfile_path = find_tarfile(opath.join(Download_Path, pattern)) # ./​pip-cache/​nebula_web_frontend-x.x.x.tar.gz69 extract_path = find_tarfile_extract_path(tarfile_path) # nebula_web_frontend-x.x.x70 71 FrontEnd_Install_Path = 'target/​nebula/​' # @todo nebula72 move_dirs = ['statics', 'templates']73 if extract_path:74 extract_result = local("tar xzf %s" % (tarfile_path, ))75 76 if extract_result.failed:77 abort('cant not extract %s' % tarfile_path)78 for _ in move_dirs:79 if local("mv %s %s" % ( opath.join(extract_path, _),...

Full Screen

Full Screen

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.

LambdaTest Learning Hubs:

YouTube

You could also refer to video tutorials over LambdaTest YouTube channel to get step by step demonstration from industry experts.

Run localstack automation tests on LambdaTest cloud grid

Perform automation testing on 3000+ real desktop and mobile devices online.

... View sample repo... View DocumentationSign up Free
›
_

Try LambdaTest Now !!

Get 100 minutes of automation test minutes FREE!!

...
Sign up with GoogleSign up with Email
Next-Gen App & Browser Testing Cloud

Was this article helpful?

Helpful

NotHelpful