How to use untar method in localstack

Best Python code snippet using localstack_python

tools.gyp

Source:tools.gyp

1# Copyright (c) 2011 The Chromium Authors. All rights reserved.2# Use of this source code is governed by a BSD-style license that can be3# found in the LICENSE file.4{5  ######################################################################6  'includes': [7    'build/common.gypi',8  ],9  ######################################################################10  'variables': {11    'disable_glibc%': 0,12    'disable_newlib%': 0,13    'disable_pnacl%': 0,14    'disable_arm%': 0,15    'disable_glibc_untar%': 0,16    'disable_newlib_untar%': 0,17    'disable_arm_untar%': 0,18    'disable_pnacl_untar%': 0,19  },20  'targets' : [21    {22      'target_name': 'prep_toolchain',23      'type': 'none',24      'dependencies': [25        'untar_toolchains',26        'prep_nacl_sdk',27      ],28      'conditions': [29        ['target_arch=="ia32" or target_arch=="x64"', {30          'dependencies': [31            'crt_init_32',32            'crt_fini_32',33            'crt_init_64',34            'crt_fini_64',35          ],36        }],37        ['target_arch=="arm"', {38          'dependencies': [39            'crt_init_arm',40            'crt_fini_arm',41          ]42        }],43      ],44    },45    {46      'target_name': 'untar_toolchains',47      'type': 'none',48      'variables': {49        'newlib_dir': '<(SHARED_INTERMEDIATE_DIR)/sdk/toolchain/<(OS)_x86_newlib',50        'glibc_dir': '<(SHARED_INTERMEDIATE_DIR)/sdk/toolchain/<(OS)_x86_glibc',51        'pnacl_dir': '<(SHARED_INTERMEDIATE_DIR)/sdk/toolchain/<(OS)_x86_pnacl',52        'arm_dir': '<(SHARED_INTERMEDIATE_DIR)/sdk/toolchain/<(OS)_arm_newlib',53      },54      'conditions': [55        ['disable_newlib==0 and disable_newlib_untar==0', {56          'actions': [57            {58              'action_name': 'Untar x86 newlib toolchain',59              'msvs_cygwin_shell': 0,60              'description': 'Untar x86 newlib toolchain',61              'inputs': [62                 '<(DEPTH)/native_client/build/cygtar.py',63                 '<(DEPTH)/native_client/toolchain/.tars/naclsdk_<(OS)_x86.tgz',64              ],65              'outputs': ['>(newlib_dir)/stamp.untar'],66              'action': [67                '>(python_exe)',68                '<(DEPTH)/native_client/build/untar_toolchain.py',69                '--tool', 'x86_newlib',70                '--tmp', '<(SHARED_INTERMEDIATE_DIR)/untar',71                '--sdk', '<(SHARED_INTERMEDIATE_DIR)/sdk',72                '--os', '<(OS)',73                '<(DEPTH)/native_client/toolchain/.tars/naclsdk_<(OS)_x86.tgz',74              ],75            },76          ]77        }],78        ['disable_glibc==0 and disable_glibc_untar==0', {79          'actions': [80            {81              'action_name': 'Untar x86 glibc toolchain',82              'msvs_cygwin_shell': 0,83              'description': 'Untar x86 glibc toolchain',84              'inputs': [85                 '<(DEPTH)/native_client/build/cygtar.py',86                 '<(DEPTH)/native_client/toolchain/.tars/toolchain_<(OS)_x86.tar.bz2',87              ],88              'outputs': ['>(glibc_dir)/stamp.untar'],89              'action': [90                '>(python_exe)',91                '<(DEPTH)/native_client/build/untar_toolchain.py',92                '--tool', 'x86_glibc',93                '--tmp', '<(SHARED_INTERMEDIATE_DIR)/untar',94                '--sdk', '<(SHARED_INTERMEDIATE_DIR)/sdk',95                '--os', '<(OS)',96                '<(DEPTH)/native_client/toolchain/.tars/toolchain_<(OS)_x86.tar.bz2',97              ],98            },99          ]100        }],101        ['disable_pnacl==0 and disable_pnacl_untar==0', {102          'actions': [103            {104              'action_name': 'Untar pnacl toolchain',105              'msvs_cygwin_shell': 0,106              'description': 'Untar pnacl toolchain',107              'inputs': [108                 '<(DEPTH)/native_client/build/cygtar.py',109                 '<(DEPTH)/native_client/toolchain/.tars/naclsdk_pnacl_<(OS)_x86.tgz',110              ],111              'outputs': ['>(pnacl_dir)/stamp.untar'],112              'action': [113                '>(python_exe)',114                '<(DEPTH)/native_client/build/untar_toolchain.py',115                '--tool', 'x86_pnacl',116                '--tmp', '<(SHARED_INTERMEDIATE_DIR)/untar',117                '--sdk', '<(SHARED_INTERMEDIATE_DIR)/sdk',118                '--os', '<(OS)',119                '<(DEPTH)/native_client/toolchain/.tars/naclsdk_pnacl_<(OS)_x86.tgz',120              ],121            },122          ]123        }],124        ['target_arch=="arm" and disable_arm==0 and disable_arm_untar==0', {125          'actions': [126            {127              'action_name': 'Untar arm toolchain',128              'msvs_cygwin_shell': 0,129              'description': 'Untar arm toolchain',130              'inputs': [131                 '<(DEPTH)/native_client/build/cygtar.py',132                 '<(DEPTH)/native_client/toolchain/.tars/gcc_arm.tgz',133                 '<(DEPTH)/native_client/toolchain/.tars/binutils_arm.tgz',134                 '<(DEPTH)/native_client/toolchain/.tars/newlib_arm.tgz',135                 '<(DEPTH)/native_client/toolchain/.tars/gcc_libs_arm.tgz',136              ],137              'outputs': ['>(arm_dir)/stamp.untar'],138              'action': [139                '>(python_exe)',140                '<(DEPTH)/native_client/build/untar_toolchain.py',141                '--tool', 'arm_newlib',142                '--tmp', '<(SHARED_INTERMEDIATE_DIR)/untar',143                '--sdk', '<(SHARED_INTERMEDIATE_DIR)/sdk',144                '--os', '<(OS)',145                '<(DEPTH)/native_client/toolchain/.tars/gcc_arm.tgz',146                '<(DEPTH)/native_client/toolchain/.tars/binutils_arm.tgz',147                '<(DEPTH)/native_client/toolchain/.tars/newlib_arm.tgz',148                '<(DEPTH)/native_client/toolchain/.tars/gcc_libs_arm.tgz',149              ],150            },151          ]152        }],153      ]154    },155    {156      'target_name': 'prep_nacl_sdk',157      'type': 'none',158      'dependencies': [159        'untar_toolchains',160      ],161      'variables': {162        'newlib_dir': '<(SHARED_INTERMEDIATE_DIR)/sdk/toolchain/<(OS)_x86_newlib',163        'glibc_dir': '<(SHARED_INTERMEDIATE_DIR)/sdk/toolchain/<(OS)_x86_glibc',164        'arm_dir': '<(SHARED_INTERMEDIATE_DIR)/sdk/toolchain/<(OS)_arm_newlib',165        'pnacl_dir': '<(SHARED_INTERMEDIATE_DIR)/sdk/toolchain/<(OS)_x86_pnacl',166      },167      'conditions': [168        ['disable_newlib==0', {169          'actions': [170            {171              'action_name': 'Prep x86 newlib toolchain',172              'msvs_cygwin_shell': 0,173              'description': 'Prep x86 newlib toolchain',174              'inputs': [175                 '<(newlib_dir)/stamp.untar',176                 '>!@pymod_do_main(prep_nacl_sdk --inputs --tool x86_newlib)',177              ],178              'outputs': ['<(newlib_dir)/stamp.prep'],179              'action': [180                '>(python_exe)',181                '<(DEPTH)/native_client/build/prep_nacl_sdk.py',182                '--tool', 'x86_newlib',183                '--path', '<(newlib_dir)',184              ],185            },186          ]187        }],188        ['disable_glibc==0', {189          'actions': [190            {191              'action_name': 'Prep x86 glibc toolchain',192              'msvs_cygwin_shell': 0,193              'description': 'Prep x86 glibc toolchain',194              'inputs': [195                 '<(glibc_dir)/stamp.untar',196                 '>!@pymod_do_main(prep_nacl_sdk --inputs --tool x86_glibc)',197              ],198              'outputs': ['<(glibc_dir)/stamp.prep'],199              'action': [200                '>(python_exe)',201                '<(DEPTH)/native_client/build/prep_nacl_sdk.py',202                '--tool', 'x86_glibc',203                '--path', '<(glibc_dir)',204              ],205            },206          ]207        }],208        ['target_arch=="arm" and disable_arm==0', {209          'actions': [210            {211              'action_name': 'Prep arm toolchain',212              'msvs_cygwin_shell': 0,213              'description': 'Prep arm toolchain',214              'inputs': [215                 '<(arm_dir)/stamp.untar',216                 '>!@pymod_do_main(prep_nacl_sdk --inputs --tool arm_newlib)',217              ],218              'outputs': ['<(arm_dir)/stamp.prep'],219              'action': [220                '>(python_exe)',221                '<(DEPTH)/native_client/build/prep_nacl_sdk.py',222                '--tool', 'arm_newlib',223                '--path', '<(arm_dir)',224              ],225            },226          ]227        }],228        ['disable_pnacl==0', {229          'actions': [230            {231              'action_name': 'Prep pnacl toolchain',232              'msvs_cygwin_shell': 0,233              'description': 'Prep pnacl toolchain',234              'inputs': [235                 '<(pnacl_dir)/stamp.untar',236                 '>!@pymod_do_main(prep_nacl_sdk --inputs --tool x86_pnacl)',237              ],238              'outputs': ['<(pnacl_dir)/stamp.prep'],239              'action': [240                '>(python_exe)',241                '<(DEPTH)/native_client/build/prep_nacl_sdk.py',242                '--tool', 'x86_pnacl',243                '--path', '<(pnacl_dir)',244              ],245            },246          ]247        }],248      ]249    },250  ],251  'conditions': [252    ['target_arch=="ia32" or target_arch=="x64"', {253      'targets' : [254        {255          'target_name': 'crt_init_64',256          'type': 'none',257          'dependencies': [258            'untar_toolchains',259            'prep_nacl_sdk'260          ],261          'variables': {262            'nlib_target': 'crt_init_dummy',263            'windows_asm_rule': 0,264            'build_glibc': 0,265            'build_newlib': 1,266            'enable_x86_32': 0,267            'extra_args': [268              '--compile',269              '--no-suffix',270              '--strip=_x86_64'271            ],272            'out_newlib64':273                '<(SHARED_INTERMEDIATE_DIR)/tc_newlib/lib64/crti.o',274            'objdir_newlib64':275                '<(SHARED_INTERMEDIATE_DIR)/tc_newlib/lib64',276          },277          'sources': [278            'src/untrusted/stubs/crti_x86_64.S',279          ]280        },281        {282          'target_name': 'crt_fini_64',283          'type': 'none',284          'dependencies': [285            'untar_toolchains',286            'prep_nacl_sdk'287          ],288          'variables': {289            'nlib_target': 'crt_fini_dummy',290            'windows_asm_rule': 0,291            'build_glibc': 0,292            'build_newlib': 1,293            'enable_x86_32': 0,294            'extra_args': [295              '--compile',296              '--no-suffix',297              '--strip=_x86_64'298            ],299            'out_newlib64':300                '<(SHARED_INTERMEDIATE_DIR)/tc_newlib/lib64/crtn.o',301            'objdir_newlib64':302                '<(SHARED_INTERMEDIATE_DIR)/tc_newlib/lib64',303          },304          'sources': [305            'src/untrusted/stubs/crtn_x86_64.S'306          ],307        }308      ],309    }],310    ['target_arch=="ia32" or target_arch=="x64"', {311      'targets' : [312        {313          'target_name': 'crt_init_32',314          'type': 'none',315          'dependencies': [316            'untar_toolchains',317            'prep_nacl_sdk'318          ],319          'variables': {320            'nlib_target': 'crt_init_dummy',321            'windows_asm_rule': 0,322            'build_glibc': 0,323            'build_newlib': 1,324            'enable_x86_64': 0,325            'extra_args': [326              '--compile',327              '--no-suffix',328              '--strip=_x86_32'329            ],330            'out_newlib32':331                '<(SHARED_INTERMEDIATE_DIR)/tc_newlib/lib32/crti.o',332            'objdir_newlib32':333                '<(SHARED_INTERMEDIATE_DIR)/tc_newlib/lib32',334          },335          'sources': [336            'src/untrusted/stubs/crti_x86_32.S',337          ],338        },339        {340          'target_name': 'crt_fini_32',341          'type': 'none',342          'dependencies': [343            'untar_toolchains',344            'prep_nacl_sdk'345          ],346          'variables': {347            'nlib_target': 'crt_fini_dummy',348            'windows_asm_rule': 0,349            'build_glibc': 0,350            'build_newlib': 1,351            'enable_x86_64': 0,352            'extra_args': [353              '--compile',354              '--no-suffix',355              '--strip=_x86_32'356            ],357            'out_newlib32':358                '<(SHARED_INTERMEDIATE_DIR)/tc_newlib/lib32/crtn.o',359            'objdir_newlib32':360                '<(SHARED_INTERMEDIATE_DIR)/tc_newlib/lib32',361          },362          'sources': [363            'src/untrusted/stubs/crtn_x86_32.S'364          ],365        }366      ],367    }],368    ['target_arch=="arm"', {369      'targets' : [370        {371          'target_name': 'crt_init_arm',372          'type': 'none',373          'dependencies': [374            'untar_toolchains',375            'prep_nacl_sdk'376          ],377          'variables': {378            'nlib_target': 'crt_init_dummy',379            'windows_asm_rule': 0,380            'build_glibc': 0,381            'build_newlib': 1,382            'extra_args': [383              '--compile',384              '--no-suffix',385              '--strip=_arm'386            ],387            'out_newlib_arm':388                '<(SHARED_INTERMEDIATE_DIR)/tc_newlib/libarm/crti.o',389            'objdir_newlib_arm':390                '<(SHARED_INTERMEDIATE_DIR)/tc_newlib/libarm',391          },392          'sources': [393            'src/untrusted/stubs/crti_arm.S',394          ],395        },396        {397          'target_name': 'crt_fini_arm',398          'type': 'none',399          'dependencies': [400            'untar_toolchains',401            'prep_nacl_sdk'402          ],403          'variables': {404            'nlib_target': 'crt_fini_dummy',405            'windows_asm_rule': 0,406            'build_glibc': 0,407            'build_newlib': 1,408            'extra_args': [409              '--compile',410              '--no-suffix',411              '--strip=_arm'412            ],413            'out_newlib_arm':414                '<(SHARED_INTERMEDIATE_DIR)/tc_newlib/libarm/crtn.o',415            'objdir_newlib_arm':416                '<(SHARED_INTERMEDIATE_DIR)/tc_newlib/libarm',417          },418          'sources': [419            'src/untrusted/stubs/crtn_arm.S'420          ],421        }422      ],423    }],424  ],...

file_utils.py

Source:file_utils.py

1from __future__ import absolute_import2from __future__ import print_function3import tarfile4import os5import sys6import shutil7import hashlib8from six.moves.urllib.request import urlopen9from six.moves.urllib.error import URLError, HTTPError10import requests11from generic_utils import Progbar12from  modac_utils import get_file_from_modac13# Under Python 2, 'urlretrieve' relies on FancyURLopener from legacy14# urllib module, known to have issues with proxy management15if sys.version_info[0] == 2:16    def urlretrieve(url, filename, reporthook=None, data=None):17        def chunk_read(response, chunk_size=8192, reporthook=None):18            total_size = response.info().get('Content-Length').strip()19            total_size = int(total_size)20            count = 021            while 1:22                chunk = response.read(chunk_size)23                count += 124                if not chunk:25                    reporthook(count, total_size, total_size)26                    break27                if reporthook:28                    reporthook(count, chunk_size, total_size)29                yield chunk30        response = urlopen(url, data)31        with open(filename, 'wb') as fd:32            for chunk in chunk_read(response, reporthook=reporthook):33                fd.write(chunk)34else:35    from six.moves.urllib.request import urlretrieve36def get_file(fname, origin, untar=False,37             # md5_hash=None, datadir='../Data/common'):38             # md5_hash=None, cache_subdir='common', datadir='../Data/common'):39             md5_hash=None, cache_subdir='common', datadir=None):  # datadir argument was never actually used so changing it to None40    """ Downloads a file from a URL if it not already in the cache.41        Passing the MD5 hash will verify the file after download as well42        as if it is already present in the cache.43        Parameters44        ----------45        fname : string46            name of the file47        origin : string48            original URL of the file49        untar : boolean50            whether the file should be decompressed51        md5_hash : string52            MD5 hash of the file for verification53        cache_subdir : string54            directory being used as the cache55        datadir : string56            if set, datadir becomes its setting (which could be e.g. an absolute path) and cache_subdir no longer matters57        Returns58        ----------59        Path to the downloaded file60    """61    if datadir is None:62        file_path = os.path.dirname(os.path.realpath(__file__))63        datadir_base = os.path.expanduser(os.path.join(file_path, '..', 'Data'))64        datadir = os.path.join(datadir_base, cache_subdir)65    if not os.path.exists(datadir):66        os.makedirs(datadir)67    # if untar:68    #    fnamesplit = fname.split('.tar.gz')69    #    untar_fpath = os.path.join(datadir, fnamesplit[0])70    if fname.endswith('.tar.gz'):71        fnamesplit = fname.split('.tar.gz')72        untar_fpath = os.path.join(datadir, fnamesplit[0])73        untar = True74    elif fname.endswith('.tgz'):75        fnamesplit = fname.split('.tgz')76        untar_fpath = os.path.join(datadir, fnamesplit[0])77        untar = True78    else:79        untar_fpath = None80    fpath = os.path.join(datadir, fname)81    download = False82    if os.path.exists(fpath) or (untar_fpath is not None and os.path.exists(untar_fpath)):83        # file found; verify integrity if a hash was provided84        if md5_hash is not None:85            if not validate_file(fpath, md5_hash):86                print('A local file was found, but it seems to be '87                      'incomplete or outdated.')88                download = True89    else:90        download = True91    # fix ftp protocol if needed92    '''93    if origin.startswith('ftp://'):94        new_url = origin.replace('ftp://','http://')95        origin = new_url96    print('Origin = ', origin)97    '''98    if download:99        if 'modac.cancer.gov' in origin:100            get_file_from_modac(fpath, origin)101        else:102            print('Downloading data from', origin)103            global progbar104            progbar = None105    106            def dl_progress(count, block_size, total_size):107                global progbar108                if progbar is None:109                    progbar = Progbar(total_size)110                else:111                    progbar.update(count * block_size)112    113            error_msg = 'URL fetch failure on {}: {} -- {}'114            try:115                try:116                    urlretrieve(origin, fpath, dl_progress)117                    # fpath = wget.download(origin)118                except URLError as e:119                    raise Exception(error_msg.format(origin, e.errno, e.reason))120                except HTTPError as e:121                    raise Exception(error_msg.format(origin, e.code, e.msg))122            except (Exception, KeyboardInterrupt) as e:123                if os.path.exists(fpath):124                    os.remove(fpath)125                raise126            progbar = None127            print()128    if untar:129        if not os.path.exists(untar_fpath):130            print('Untarring file...')131            tfile = tarfile.open(fpath, 'r:gz')132            try:133                tfile.extractall(path=datadir)134            except (Exception, KeyboardInterrupt) as e:135                if os.path.exists(untar_fpath):136                    if os.path.isfile(untar_fpath):137                        os.remove(untar_fpath)138                    else:139                        shutil.rmtree(untar_fpath)140                raise141            tfile.close()142        return untar_fpath143        print()144    return fpath145def validate_file(fpath, md5_hash):146    """ Validates a file against a MD5 hash147        Parameters148        ----------149        fpath : string150            path to the file being validated151        md5_hash : string152            the MD5 hash being validated against153        Returns154        ----------155        boolean156            Whether the file is valid157    """158    hasher = hashlib.md5()159    with open(fpath, 'rb') as f:160        buf = f.read()161        hasher.update(buf)162    if str(hasher.hexdigest()) == str(md5_hash):163        return True164    else:...

untar.py

Source:untar.py

1from gladier import GladierBaseTool, generate_flow_definition2def untar(**data):3    import tarfile4    import pathlib5    untar_input = pathlib.Path(data['untar_input']).expanduser()6    if data.get('untar_output', ''):7        untar_output = pathlib.Path(data['untar_output']).expanduser()8    else:9        untar_output = untar_input.parent / untar_input.stem10    with tarfile.open(untar_input) as file:11        untar_output.mkdir(parents=True, exist_ok=True)12        file.extractall(untar_output)13    return str(untar_output)14@generate_flow_definition(modifiers={15    'untar': {'ExceptionOnActionFailure': True,16              'WaitTime': 300}...

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.