How to use compare_checksum method in autotest

Best Python code snippet using autotest_python

jobsubmitter.py

Source:jobsubmitter.py

1#   Copyright notice:2#   Copyright  Members of the EMI Collaboration, 2013.3#4#   See www.eu-emi.eu for details on the copyright holders5#6#   Licensed under the Apache License, Version 2.0 (the "License");7#   you may not use this file except in compliance with the License.8#   You may obtain a copy of the License at9#10#       http://www.apache.org/licenses/LICENSE-2.011#12#   Unless required by applicable law or agreed to in writing, software13#   distributed under the License is distributed on an "AS IS" BASIS,14#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.15#   See the License for the specific language governing permissions and16#   limitations under the License.17from datetime import timedelta18try:19    import simplejson as json20except:21    import json22import logging23import sys24import time25from base import Base26from fts3.rest.client import Submitter, Delegator, Inquirer27DEFAULT_CHECKSUM = 'ADLER32' 28def _metadata(data):29    try:30        return json.loads(data)31    except:32        return str(data)33class JobSubmitter(Base):34    def __init__(self):35        super(JobSubmitter, self).__init__(36            extra_args='SOURCE DESTINATION [CHECKSUM]',37            description="""38            This command can be used to submit new jobs to FTS3. It supports simple and bulk submissions. The bulk39            format is as follows:40            ```json41            {42              "files": [43                {44                  "sources": [45                    "gsiftp://source.host/file"46                  ],47                  "destinations": [48                    "gsiftp://destination.host/file"49                  ],50                  "metadata": "file-metadata",51                  "checksum": "ADLER32:1234",52                  "filesize": 102453                },54                {55                  "sources": [56                    "gsiftp://source.host/file2"57                  ],58                  "destinations": [59                    "gsiftp://destination.host/file2"60                  ],61                  "metadata": "file2-metadata",62                  "checksum": "ADLER32:4321",63                  "filesize": 2048,64                  "activity": "default"65                }66              ]67            }68            ```69            """,70            example="""71            $ %(prog)s -s https://fts3-devel.cern.ch:8446 gsiftp://source.host/file gsiftp://destination.host/file72            Job successfully submitted.73            Job id: 9fee8c1e-c46d-11e3-8299-02163e00a17a74            $ %(prog)s -s https://fts3-devel.cern.ch:8446 -f bulk.json75            Job successfully submitted.76            Job id: 9fee8c1e-c46d-11e3-8299-02163e00a17a77            """78        )79        # Specific options80        self.opt_parser.add_option('-b', '--blocking', dest='blocking', default=False, action='store_true',81                                   help='blocking mode. Wait until the operation completes.')82        self.opt_parser.add_option('-i', '--interval', dest='poll_interval', type='int', default=30,83                                   help='interval between two poll operations in blocking mode.')84        self.opt_parser.add_option('-e', '--expire', dest='proxy_lifetime', type='int', default=420,85                                   help='expiration time of the delegation in minutes.')86        self.opt_parser.add_option('--delegate-when-lifetime-lt', type=int, default=120,87                                   help='delegate the proxy when the remote lifetime is less than this value (in minutes)')88        self.opt_parser.add_option('-o', '--overwrite', dest='overwrite', default=False, action='store_true',89                                   help='overwrite files.')90        self.opt_parser.add_option('-r', '--reuse', dest='reuse', default=False, action='store_true',91                                   help='enable session reuse for the transfer job.')92        self.opt_parser.add_option('--job-metadata', dest='job_metadata',93                                   help='transfer job metadata.')94        self.opt_parser.add_option('--file-metadata', dest='file_metadata',95                                   help='file metadata.')96        self.opt_parser.add_option('--file-size', dest='file_size', type='long',97                                   help='file size (in Bytes)')98        self.opt_parser.add_option('-g', '--gparam', dest='gridftp_params',99                                   help='GridFTP parameters.')100        self.opt_parser.add_option('-t', '--dest-token', dest='destination_token',101                                   help='the destination space token or its description.')102        self.opt_parser.add_option('-S', '--source-token', dest='source_token',103                                   help='the source space token or its description.')104        self.opt_parser.add_option('-K', '--compare-checksum', dest='compare_checksum', default=False, action='store_true',105                                   help='deprecated: compare checksums between source and destination.')106        self.opt_parser.add_option('-C', '--checksum-mode', dest='checksum_mode', type='string', default='none',107                                   help='compare checksums in source, target, both or none.')108        self.opt_parser.add_option('--copy-pin-lifetime', dest='pin_lifetime', type='long', default=-1,109                                   help='pin lifetime of the copy in seconds.')110        self.opt_parser.add_option('--bring-online', dest='bring_online', type='long', default=None,111                                   help='bring online timeout in seconds.')112        self.opt_parser.add_option('--timeout', dest='timeout', type='long', default=None,113                                   help='transfer timeout in seconds.')114        self.opt_parser.add_option('--fail-nearline', dest='fail_nearline', default=False, action='store_true',115                                   help='fail the transfer is the file is nearline.')116        self.opt_parser.add_option('--dry-run', dest='dry_run', default=False, action='store_true',117                                   help='do not send anything, just print the JSON message.')118        self.opt_parser.add_option('-f', '--file', dest='bulk_file', type='string',119                                   help='Name of configuration file')120        self.opt_parser.add_option('--retry', dest='retry', type='int', default=0,121                                   help='Number of retries. If 0, the server default will be used.'122                                        'If negative, there will be no retries.')123        self.opt_parser.add_option('-m', '--multi-hop', dest='multihop', default=False, action='store_true',124                                   help='submit a multihop transfer.')125        self.opt_parser.add_option('--cloud-credentials', dest='cloud_cred', default=None,126                                   help='use cloud credentials for the job (i.e. dropbox).')127        self.opt_parser.add_option('--nostreams', dest='nostreams', default=None,128                                   help='number of streams')129        self.opt_parser.add_option('--ipv4', dest='ipv4', default=False, action='store_true',130                                   help='force ipv4')131        self.opt_parser.add_option('--ipv6', dest='ipv6', default=False, action='store_true',132                                   help='force ipv6')133    def validate(self):134        self.checksum = None135        if not self.options.bulk_file:136            if len(self.args) < 2:137                self.logger.critical("Need a source and a destination")138                sys.exit(1)139            elif len(self.args) == 2:140                (self.source, self.destination) = self.args141                self.checksum = None142            elif len(self.args) == 3:143                (self.source, self.destination, self.checksum) = self.args144            else:145                self.logger.critical("Too many parameters")146                sys.exit(1)147        else:148            self.checksum = None149        if self.options.ipv4 and self.options.ipv6:150            self.opt_parser.error('ipv4 and ipv6 can not be used at the same time')151        if self.options.verbose:152            self.logger.setLevel(logging.DEBUG)153    def _build_transfers(self):154        if self.options.bulk_file:155            filecontent = open(self.options.bulk_file).read()156            bulk = json.loads(filecontent)157            if "files" in bulk:158                return bulk["files"]159            elif "Files" in bulk:160                return bulk["Files"]161            else:162                self.logger.critical("Could not find any transfers")163                sys.exit(1)164        else:165            return [{"sources": [self.source], "destinations": [self.destination]}]166    def _do_submit(self, context):167        #Backwards compatibility: compare_checksum parameter 168        if self.options.compare_checksum:169            checksum_mode = 'both' 170        else:171            if self.checksum:172                checksum_mode = 'target'173            else: 174                checksum = 'none'  175        #Compare checksum has major priority than checksum_mode     176        if not self.options.compare_checksum:   177            if len(self.options.checksum_mode) > 0:178                checksum_mode = self.options.checksum_mode   179            else:180                 checksum_mode = 'none'        181        182        if not self.checksum:183            self.checksum = DEFAULT_CHECKSUM184            185        delegator = Delegator(context)186        delegator.delegate(187            timedelta(minutes=self.options.proxy_lifetime),188            delegate_when_lifetime_lt=timedelta(minutes=self.options.delegate_when_lifetime_lt)189        )190        submitter = Submitter(context)191        job_id = submitter.submit(192            self._build_transfers(),193            checksum=self.checksum,194            bring_online=self.options.bring_online,195            timeout = self.options.timeout,196            verify_checksum=checksum_mode[0],197            spacetoken=self.options.destination_token,198            source_spacetoken=self.options.source_token,199            fail_nearline=self.options.fail_nearline,200            file_metadata=_metadata(self.options.file_metadata),201            filesize=self.options.file_size,202            gridftp=self.options.gridftp_params,203            job_metadata=_metadata(self.options.job_metadata),204            overwrite=self.options.overwrite,205            copy_pin_lifetime=self.options.pin_lifetime,206            reuse=self.options.reuse,207            retry=self.options.retry,208            multihop=self.options.multihop,209            credential=self.options.cloud_cred,210            nostreams=self.options.nostreams,211            ipv4=self.options.ipv4,212            ipv6=self.options.ipv6213        )214        if self.options.json:215            self.logger.info(json.dumps(job_id))216        else:217            self.logger.info("Job successfully submitted.")218            self.logger.info("Job id: %s" % job_id)219        if job_id and self.options.blocking:220            inquirer = Inquirer(context)221            job = inquirer.get_job_status(job_id)222            while job['job_state'] in ['SUBMITTED', 'READY', 'STAGING', 'ACTIVE']:223                self.logger.info("Job in state %s" % job['job_state'])224                time.sleep(self.options.poll_interval)225                job = inquirer.get_job_status(job_id)226            self.logger.info("Job finished with state %s" % job['job_state'])227            if job['reason']:228                self.logger.info("Reason: %s" % job['reason'])229        return job_id230    def _do_dry_run(self, context):231        #Backwards compatibility: compare_checksum parameter 232        if self.options.compare_checksum:233            checksum_mode = 'both' 234        else:235            if self.checksum:236                checksum_mode = 'target'237            else: 238                checksum = 'none'  239        #Compare checksum has major priority than checksum_mode     240        if not self.options.compare_checksum:   241            if len(self.options.checksum_mode) > 0:242                checksum_mode = self.options.checksum_mode   243            else:244                 checksum_mode = 'none'        245        246        if not self.checksum:247            self.checksum = DEFAULT_CHECKSUM248                249        submitter = Submitter(context)250        print submitter.build_submission(251            self._build_transfers(),252            checksum=self.checksum,253            bring_online=self.options.bring_online,254            timeout = self.options.timeout,255            verify_checksum=checksum_mode,256            spacetoken=self.options.destination_token,257            source_spacetoken=self.options.source_token,258            fail_nearline=self.options.fail_nearline,259            file_metadata=_metadata(self.options.file_metadata),260            filesize=self.options.file_size,261            gridftp=self.options.gridftp_params,262            job_metadata=_metadata(self.options.job_metadata),263            overwrite=self.options.overwrite,264            copy_pin_lifetime=self.options.pin_lifetime,265            reuse=self.options.reuse,266            retry=self.options.retry,267            multihop=self.options.multihop,268            credential=self.options.cloud_cred,269            nostreams=self.options.nostreams,270            ipv4=self.options.ipv4,271            ipv6=self.options.ipv6272        )273        return None274    def run(self):275        context = self._create_context()276        if not self.options.dry_run:277            return self._do_submit(context)278        else:...

compare_checksum.py

Source:compare_checksum.py

...16## @class compare_checksum17#  @brief yuntse should give a brief comment here18#  @details19#  yuntse should give a detailed comment here20class compare_checksum( ds_project_base ):21    # Define project name as class attribute22    _project = 'compare_checksum'23    # Define # of runs to process per request24    _nruns   = 525    ## @brief default ctor can take # runs to process for this instance26    def __init__( self, arg = '', nruns = None ):27        # Call base class ctor28        super( compare_checksum, self ).__init__( arg )29        if not arg:30            self.error('No project name specified!')31            raise Exception32        self._project = arg33        if nruns:34            self._nruns   = int(nruns)35        self._ref_project = ''36        self._parent_project = ''37        self._experts = ''38        self._data = ''39    ## @brief method to retrieve the project resource information if not yet done40    def get_resource( self ):41        resource = self._api.get_resource( self._project )42        self._nruns = int(resource['NRUNS'])43        self._ref_project = resource['REF_PROJECT']44        self._parent_project = resource['PARENT_PROJECT']45        self._experts = resource['EXPERTS']46    ## @brief access DB and retrieves new runs47    def compare( self ):48        # Attempt to connect DB. If failure, abort49        if not self.connect():50	    self.error('Cannot connect to DB! Aborting...')51	    return52        self.get_resource()53        # Fetch runs from DB and process for # runs specified for this instance.54        ctr = self._nruns55        for x in self.get_xtable_runs( [self._project, self._ref_project, self._parent_project],                                       56                                       [1, 0, 0] ):57            # Counter decreases by 158            ctr -=159            # Currently hard-coded seq = 060            (run, subrun, seq) = (int(x[0]), int(x[1]), 0)61            # Report starting62            now_str  = time.strftime('%Y-%m-%d %H:%M:%S')63            self.info('Comparing checksums: run=%d, subrun=%d @ %s' % ( run, subrun, now_str ))64            statusCode = 165            # Get status objects66            RefStatus = self._api.get_status( ds_status( self._ref_project, run, subrun, seq ))67            ParentStatus = self._api.get_status( ds_status( self._parent_project, run, subrun, seq ))68            if RefStatus._data == ParentStatus._data:69                statusCode = 070            else:71                subject = 'Checksum different in run %d, subrun %d between %s and %s' % ( run, subrun, self._ref_project, self._parent_project )72                text = '%s\n' % subject73                text += 'Run %d, subrun %d\n' % ( run, subrun )74                text += '%s checksum: %s\n' % ( self._ref_project, RefStatus._data )75                text += '%s checksum: %s\n' % ( self._parent_project, ParentStatus._data )76                pub_smtp( os.environ['PUB_SMTP_ACCT'], os.environ['PUB_SMTP_SRVR'], os.environ['PUB_SMTP_PASS'], self._experts, subject, text )77                statusCode = 100078                self._data = '%s:%s;%s:%s' % ( self._ref_project, RefStatus._data, self._parent_project, ParentStatus._data )79            # Pretend I'm doing something80            #time.sleep(0.5)81            # Report finishing82            now_str  = time.strftime('%Y-%m-%d %H:%M:%S')83            self.info('Finished comparing checksums: run=%d, subrun=%d @ %s' % ( run, subrun, now_str ))84            # Create a status object to be logged to DB (if necessary)85            # Let's say we set the status to be 1086            status = ds_status( project = self._project,87                                run     = int(x[0]),88                                subrun  = int(x[1]),89                                seq     = seq,90                                status  = statusCode,91                                data    = self._data )92            # Log status93            self.log_status( status )94            #self.info('Logged status: run=%d, subrun=%d @ %s' % ( run, subrun, now_str ))95            # Break from loop if counter became 096            if not ctr: break97# A unit test section98if __name__ == '__main__':99    proj_name = sys.argv[1]100    obj = compare_checksum( proj_name, 5 )...

fletcher.py

Source:fletcher.py

...6        sum1 = (sum1 + i) % 2557        sum2 = (sum2 + sum1) % 2558    return (sum2 << 8) | sum19	10def compare_checksum(data_to_checksum, checksum):11    chk = fletcher16(data_to_checksum)12    print "Checking: " + " ".join([hex(x) for x in data_to_checksum])13    return chk == (checksum[0] << 8) + checksum[1]14	15	16if __name__ == "__main__":17    test_arr = [1, 0x22, 0]18    c16 = fletcher16(test_arr)19    # Results generated using C functions20    assert (c16 >> 8) == 0x4721    assert (c16 & 0xFF) == 0x2322    test_arr += [(c16 >> 8) & 0xFF]23    test_arr += [c16 & 0xFF]24    assert compare_checksum(test_arr[:3], test_arr[3:])...

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.