How to use runCmd method in Cypress

Best JavaScript code snippet using cypress

devtool.py

Source:devtool.py Github

copy

Full Screen

...25        self.assertEqual(checkvars, {}, 'Some variables not found: %s' % checkvars)26        for inherit in checkinherits:27            self.assertIn(inherit, inherits, 'Missing inherit of %s' % inherit)28    def _check_bbappend(self, testrecipe, recipefile, appenddir):29        result = runCmd('bitbake-layers show-appends', cwd=self.builddir)30        resultlines = result.output.splitlines()31        inrecipe = False32        bbappends = []33        bbappendfile = None34        for line in resultlines:35            if inrecipe:36                if line.startswith(' '):37                    bbappends.append(line.strip())38                else:39                    break40            elif line == '%s:' % os.path.basename(recipefile):41                inrecipe = True42        self.assertLessEqual(len(bbappends), 2, '%s recipe is being bbappended by another layer - bbappends found:\n  %s' % (testrecipe, '\n  '.join(bbappends)))43        for bbappend in bbappends:44            if bbappend.startswith(appenddir):45                bbappendfile = bbappend46                break47        else:48            self.assertTrue(False, 'bbappend for recipe %s does not seem to be created in test layer' % testrecipe)49        return bbappendfile50    def _create_temp_layer(self, templayerdir, addlayer, templayername, priority=999, recipepathspec='recipes-*/*'):51        create_temp_layer(templayerdir, templayername, priority, recipepathspec)52        if addlayer:53            self.add_command_to_tearDown('bitbake-layers remove-layer %s || true' % templayerdir)54            result = runCmd('bitbake-layers add-layer %s' % templayerdir, cwd=self.builddir)55    def _process_ls_output(self, output):56        """57        Convert ls -l output to a format we can reasonably compare from one context58        to another (e.g. from host to target)59        """60        filelist = []61        for line in output.splitlines():62            splitline = line.split()63            # Remove trailing . on perms64            splitline[0] = splitline[0].rstrip('.')65            # Remove leading . on paths66            splitline[-1] = splitline[-1].lstrip('.')67            # Drop fields we don't want to compare68            del splitline[7]69            del splitline[6]70            del splitline[5]71            del splitline[4]72            del splitline[1]73            filelist.append(' '.join(splitline))74        return filelist75class DevtoolTests(DevtoolBase):76    @testcase(1158)77    def test_create_workspace(self):78        # Check preconditions79        workspacedir = os.path.join(self.builddir, 'workspace')80        self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')81        result = runCmd('bitbake-layers show-layers')82        self.assertTrue('/workspace' not in result.output, 'This test cannot be run with a workspace layer in bblayers.conf')83        # Try creating a workspace layer with a specific path84        tempdir = tempfile.mkdtemp(prefix='devtoolqa')85        self.track_for_cleanup(tempdir)86        result = runCmd('devtool create-workspace %s' % tempdir)87        self.assertTrue(os.path.isfile(os.path.join(tempdir, 'conf', 'layer.conf')), msg = "No workspace created. devtool output: %s " % result.output)88        result = runCmd('bitbake-layers show-layers')89        self.assertIn(tempdir, result.output)90        # Try creating a workspace layer with the default path91        self.track_for_cleanup(workspacedir)92        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')93        result = runCmd('devtool create-workspace')94        self.assertTrue(os.path.isfile(os.path.join(workspacedir, 'conf', 'layer.conf')), msg = "No workspace created. devtool output: %s " % result.output)95        result = runCmd('bitbake-layers show-layers')96        self.assertNotIn(tempdir, result.output)97        self.assertIn(workspacedir, result.output)98    @testcase(1159)99    def test_devtool_add(self):100        # Check preconditions101        workspacedir = os.path.join(self.builddir, 'workspace')102        self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')103        # Fetch source104        tempdir = tempfile.mkdtemp(prefix='devtoolqa')105        self.track_for_cleanup(tempdir)106        url = 'http://www.ivarch.com/programs/sources/pv-1.5.3.tar.bz2'107        result = runCmd('wget %s' % url, cwd=tempdir)108        result = runCmd('tar xfv pv-1.5.3.tar.bz2', cwd=tempdir)109        srcdir = os.path.join(tempdir, 'pv-1.5.3')110        self.assertTrue(os.path.isfile(os.path.join(srcdir, 'configure')), 'Unable to find configure script in source directory')111        # Test devtool add112        self.track_for_cleanup(workspacedir)113        self.add_command_to_tearDown('bitbake -c cleansstate pv')114        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')115        result = runCmd('devtool add pv %s' % srcdir)116        self.assertTrue(os.path.exists(os.path.join(workspacedir, 'conf', 'layer.conf')), 'Workspace directory not created')117        # Test devtool status118        result = runCmd('devtool status')119        self.assertIn('pv', result.output)120        self.assertIn(srcdir, result.output)121        # Clean up anything in the workdir/sysroot/sstate cache (have to do this *after* devtool add since the recipe only exists then)122        bitbake('pv -c cleansstate')123        # Test devtool build124        result = runCmd('devtool build pv')125        installdir = get_bb_var('D', 'pv')126        self.assertTrue(installdir, 'Could not query installdir variable')127        bindir = get_bb_var('bindir', 'pv')128        self.assertTrue(bindir, 'Could not query bindir variable')129        if bindir[0] == '/':130            bindir = bindir[1:]131        self.assertTrue(os.path.isfile(os.path.join(installdir, bindir, 'pv')), 'pv binary not found in D')132    @testcase(1162)133    def test_devtool_add_library(self):134        # Check preconditions135        workspacedir = os.path.join(self.builddir, 'workspace')136        self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')137        # We don't have the ability to pick up this dependency automatically yet...138        bitbake('libusb1')139        # Fetch source140        tempdir = tempfile.mkdtemp(prefix='devtoolqa')141        self.track_for_cleanup(tempdir)142        url = 'http://www.intra2net.com/en/developer/libftdi/download/libftdi1-1.1.tar.bz2'143        result = runCmd('wget %s' % url, cwd=tempdir)144        result = runCmd('tar xfv libftdi1-1.1.tar.bz2', cwd=tempdir)145        srcdir = os.path.join(tempdir, 'libftdi1-1.1')146        self.assertTrue(os.path.isfile(os.path.join(srcdir, 'CMakeLists.txt')), 'Unable to find CMakeLists.txt in source directory')147        # Test devtool add (and use -V so we test that too)148        self.track_for_cleanup(workspacedir)149        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')150        result = runCmd('devtool add libftdi %s -V 1.1' % srcdir)151        self.assertTrue(os.path.exists(os.path.join(workspacedir, 'conf', 'layer.conf')), 'Workspace directory not created')152        # Test devtool status153        result = runCmd('devtool status')154        self.assertIn('libftdi', result.output)155        self.assertIn(srcdir, result.output)156        # Clean up anything in the workdir/sysroot/sstate cache (have to do this *after* devtool add since the recipe only exists then)157        bitbake('libftdi -c cleansstate')158        # Test devtool build159        result = runCmd('devtool build libftdi')160        self.add_command_to_tearDown('bitbake -c cleansstate libftdi')161        staging_libdir = get_bb_var('STAGING_LIBDIR', 'libftdi')162        self.assertTrue(staging_libdir, 'Could not query STAGING_LIBDIR variable')163        self.assertTrue(os.path.isfile(os.path.join(staging_libdir, 'libftdi1.so.2.1.0')), "libftdi binary not found in STAGING_LIBDIR. Output of devtool build libftdi %s" % result.output)164        # Test devtool reset165        stampprefix = get_bb_var('STAMP', 'libftdi')166        result = runCmd('devtool reset libftdi')167        result = runCmd('devtool status')168        self.assertNotIn('libftdi', result.output)169        self.assertTrue(stampprefix, 'Unable to get STAMP value for recipe libftdi')170        matches = glob.glob(stampprefix + '*')171        self.assertFalse(matches, 'Stamp files exist for recipe libftdi that should have been cleaned')172        self.assertFalse(os.path.isfile(os.path.join(staging_libdir, 'libftdi1.so.2.1.0')), 'libftdi binary still found in STAGING_LIBDIR after cleaning')173    @testcase(1160)174    def test_devtool_add_fetch(self):175        # Check preconditions176        workspacedir = os.path.join(self.builddir, 'workspace')177        self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')178        # Fetch source179        tempdir = tempfile.mkdtemp(prefix='devtoolqa')180        self.track_for_cleanup(tempdir)181        testver = '0.23'182        url = 'https://pypi.python.org/packages/source/M/MarkupSafe/MarkupSafe-%s.tar.gz' % testver183        testrecipe = 'python-markupsafe'184        srcdir = os.path.join(tempdir, testrecipe)185        # Test devtool add186        self.track_for_cleanup(workspacedir)187        self.add_command_to_tearDown('bitbake -c cleansstate %s' % testrecipe)188        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')189        result = runCmd('devtool add %s %s -f %s' % (testrecipe, srcdir, url))190        self.assertTrue(os.path.exists(os.path.join(workspacedir, 'conf', 'layer.conf')), 'Workspace directory not created. %s' % result.output)191        self.assertTrue(os.path.isfile(os.path.join(srcdir, 'setup.py')), 'Unable to find setup.py in source directory')192        # Test devtool status193        result = runCmd('devtool status')194        self.assertIn(testrecipe, result.output)195        self.assertIn(srcdir, result.output)196        # Check recipe197        recipefile = get_bb_var('FILE', testrecipe)198        self.assertIn('%s.bb' % testrecipe, recipefile, 'Recipe file incorrectly named')199        checkvars = {}200        checkvars['S'] = '${WORKDIR}/MarkupSafe-%s' % testver201        checkvars['SRC_URI'] = url202        self._test_recipe_contents(recipefile, checkvars, [])203        # Try with version specified204        result = runCmd('devtool reset -n %s' % testrecipe)205        shutil.rmtree(srcdir)206        result = runCmd('devtool add %s %s -f %s -V %s' % (testrecipe, srcdir, url, testver))207        self.assertTrue(os.path.isfile(os.path.join(srcdir, 'setup.py')), 'Unable to find setup.py in source directory')208        # Test devtool status209        result = runCmd('devtool status')210        self.assertIn(testrecipe, result.output)211        self.assertIn(srcdir, result.output)212        # Check recipe213        recipefile = get_bb_var('FILE', testrecipe)214        self.assertIn('%s_%s.bb' % (testrecipe, testver), recipefile, 'Recipe file incorrectly named')215        checkvars = {}216        checkvars['S'] = '${WORKDIR}/MarkupSafe-${PV}'217        checkvars['SRC_URI'] = url.replace(testver, '${PV}')218        self._test_recipe_contents(recipefile, checkvars, [])219    @testcase(1161)220    def test_devtool_add_fetch_git(self):221        # Check preconditions222        workspacedir = os.path.join(self.builddir, 'workspace')223        self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')224        # Fetch source225        tempdir = tempfile.mkdtemp(prefix='devtoolqa')226        self.track_for_cleanup(tempdir)227        url = 'git://git.yoctoproject.org/libmatchbox'228        checkrev = '462f0652055d89c648ddd54fd7b03f175c2c6973'229        testrecipe = 'libmatchbox2'230        srcdir = os.path.join(tempdir, testrecipe)231        # Test devtool add232        self.track_for_cleanup(workspacedir)233        self.add_command_to_tearDown('bitbake -c cleansstate %s' % testrecipe)234        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')235        result = runCmd('devtool add %s %s -f %s' % (testrecipe, srcdir, url))236        self.assertTrue(os.path.exists(os.path.join(workspacedir, 'conf', 'layer.conf')), 'Workspace directory not created: %s' % result.output)237        self.assertTrue(os.path.isfile(os.path.join(srcdir, 'configure.ac')), 'Unable to find configure.ac in source directory')238        # Test devtool status239        result = runCmd('devtool status')240        self.assertIn(testrecipe, result.output)241        self.assertIn(srcdir, result.output)242        # Check recipe243        recipefile = get_bb_var('FILE', testrecipe)244        self.assertIn('_git.bb', recipefile, 'Recipe file incorrectly named')245        checkvars = {}246        checkvars['S'] = '${WORKDIR}/git'247        checkvars['PV'] = '1.0+git${SRCPV}'248        checkvars['SRC_URI'] = url249        checkvars['SRCREV'] = '${AUTOREV}'250        self._test_recipe_contents(recipefile, checkvars, [])251        # Try with revision and version specified252        result = runCmd('devtool reset -n %s' % testrecipe)253        shutil.rmtree(srcdir)254        url_rev = '%s;rev=%s' % (url, checkrev)255        result = runCmd('devtool add %s %s -f "%s" -V 1.5' % (testrecipe, srcdir, url_rev))256        self.assertTrue(os.path.isfile(os.path.join(srcdir, 'configure.ac')), 'Unable to find configure.ac in source directory')257        # Test devtool status258        result = runCmd('devtool status')259        self.assertIn(testrecipe, result.output)260        self.assertIn(srcdir, result.output)261        # Check recipe262        recipefile = get_bb_var('FILE', testrecipe)263        self.assertIn('_git.bb', recipefile, 'Recipe file incorrectly named')264        checkvars = {}265        checkvars['S'] = '${WORKDIR}/git'266        checkvars['PV'] = '1.5+git${SRCPV}'267        checkvars['SRC_URI'] = url268        checkvars['SRCREV'] = checkrev269        self._test_recipe_contents(recipefile, checkvars, [])270    @testcase(1164)271    def test_devtool_modify(self):272        # Check preconditions273        workspacedir = os.path.join(self.builddir, 'workspace')274        self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')275        # Clean up anything in the workdir/sysroot/sstate cache276        bitbake('mdadm -c cleansstate')277        # Try modifying a recipe278        tempdir = tempfile.mkdtemp(prefix='devtoolqa')279        self.track_for_cleanup(tempdir)280        self.track_for_cleanup(workspacedir)281        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')282        self.add_command_to_tearDown('bitbake -c clean mdadm')283        result = runCmd('devtool modify mdadm -x %s' % tempdir)284        self.assertTrue(os.path.exists(os.path.join(tempdir, 'Makefile')), 'Extracted source could not be found')285        self.assertTrue(os.path.isdir(os.path.join(tempdir, '.git')), 'git repository for external source tree not found')286        self.assertTrue(os.path.exists(os.path.join(workspacedir, 'conf', 'layer.conf')), 'Workspace directory not created')287        matches = glob.glob(os.path.join(workspacedir, 'appends', 'mdadm_*.bbappend'))288        self.assertTrue(matches, 'bbappend not created %s' % result.output)289        # Test devtool status290        result = runCmd('devtool status')291        self.assertIn('mdadm', result.output)292        self.assertIn(tempdir, result.output)293        # Check git repo294        result = runCmd('git status --porcelain', cwd=tempdir)295        self.assertEqual(result.output.strip(), "", 'Created git repo is not clean')296        result = runCmd('git symbolic-ref HEAD', cwd=tempdir)297        self.assertEqual(result.output.strip(), "refs/heads/devtool", 'Wrong branch in git repo')298        # Try building299        bitbake('mdadm')300        # Try making (minor) modifications to the source301        result = runCmd("sed -i 's!^\.TH.*!.TH MDADM 8 \"\" v9.999-custom!' %s" % os.path.join(tempdir, 'mdadm.8.in'))302        bitbake('mdadm -c package')303        pkgd = get_bb_var('PKGD', 'mdadm')304        self.assertTrue(pkgd, 'Could not query PKGD variable')305        mandir = get_bb_var('mandir', 'mdadm')306        self.assertTrue(mandir, 'Could not query mandir variable')307        if mandir[0] == '/':308            mandir = mandir[1:]309        with open(os.path.join(pkgd, mandir, 'man8', 'mdadm.8'), 'r') as f:310            for line in f:311                if line.startswith('.TH'):312                    self.assertEqual(line.rstrip(), '.TH MDADM 8 "" v9.999-custom', 'man file not modified. man searched file path: %s' % os.path.join(pkgd, mandir, 'man8', 'mdadm.8'))313        # Test devtool reset314        stampprefix = get_bb_var('STAMP', 'mdadm')315        result = runCmd('devtool reset mdadm')316        result = runCmd('devtool status')317        self.assertNotIn('mdadm', result.output)318        self.assertTrue(stampprefix, 'Unable to get STAMP value for recipe mdadm')319        matches = glob.glob(stampprefix + '*')320        self.assertFalse(matches, 'Stamp files exist for recipe mdadm that should have been cleaned')321    @testcase(1166)322    def test_devtool_modify_invalid(self):323        # Check preconditions324        workspacedir = os.path.join(self.builddir, 'workspace')325        self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')326        # Try modifying some recipes327        tempdir = tempfile.mkdtemp(prefix='devtoolqa')328        self.track_for_cleanup(tempdir)329        self.track_for_cleanup(workspacedir)330        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')331        testrecipes = 'perf kernel-devsrc package-index core-image-minimal meta-toolchain packagegroup-core-sdk meta-ide-support'.split()332        # Find actual name of gcc-source since it now includes the version - crude, but good enough for this purpose333        result = runCmd('bitbake-layers show-recipes gcc-source*')334        reading = False335        for line in result.output.splitlines():336            if line.startswith('=='):337                reading = True338            elif reading and not line.startswith(' '):339                testrecipes.append(line.split(':')[0])340        for testrecipe in testrecipes:341            # Check it's a valid recipe342            bitbake('%s -e' % testrecipe)343            # devtool extract should fail344            result = runCmd('devtool extract %s %s' % (testrecipe, os.path.join(tempdir, testrecipe)), ignore_status=True)345            self.assertNotEqual(result.status, 0, 'devtool extract on %s should have failed. devtool output: %s' % (testrecipe, result.output))346            self.assertNotIn('Fetching ', result.output, 'devtool extract on %s should have errored out before trying to fetch' % testrecipe)347            self.assertIn('ERROR: ', result.output, 'devtool extract on %s should have given an ERROR' % testrecipe)348            # devtool modify should fail349            result = runCmd('devtool modify %s -x %s' % (testrecipe, os.path.join(tempdir, testrecipe)), ignore_status=True)350            self.assertNotEqual(result.status, 0, 'devtool modify on %s should have failed. devtool output: %s' %  (testrecipe, result.output))351            self.assertIn('ERROR: ', result.output, 'devtool modify on %s should have given an ERROR' % testrecipe)352    @testcase(1165)353    def test_devtool_modify_git(self):354        # Check preconditions355        workspacedir = os.path.join(self.builddir, 'workspace')356        self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')357        testrecipe = 'mkelfimage'358        src_uri = get_bb_var('SRC_URI', testrecipe)359        self.assertIn('git://', src_uri, 'This test expects the %s recipe to be a git recipe' % testrecipe)360        # Clean up anything in the workdir/sysroot/sstate cache361        bitbake('%s -c cleansstate' % testrecipe)362        # Try modifying a recipe363        tempdir = tempfile.mkdtemp(prefix='devtoolqa')364        self.track_for_cleanup(tempdir)365        self.track_for_cleanup(workspacedir)366        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')367        self.add_command_to_tearDown('bitbake -c clean %s' % testrecipe)368        result = runCmd('devtool modify %s -x %s' % (testrecipe, tempdir))369        self.assertTrue(os.path.exists(os.path.join(tempdir, 'Makefile')), 'Extracted source could not be found')370        self.assertTrue(os.path.isdir(os.path.join(tempdir, '.git')), 'git repository for external source tree not found')371        self.assertTrue(os.path.exists(os.path.join(workspacedir, 'conf', 'layer.conf')), 'Workspace directory not created. devtool output: %s' % result.output)372        matches = glob.glob(os.path.join(workspacedir, 'appends', 'mkelfimage_*.bbappend'))373        self.assertTrue(matches, 'bbappend not created')374        # Test devtool status375        result = runCmd('devtool status')376        self.assertIn(testrecipe, result.output)377        self.assertIn(tempdir, result.output)378        # Check git repo379        result = runCmd('git status --porcelain', cwd=tempdir)380        self.assertEqual(result.output.strip(), "", 'Created git repo is not clean')381        result = runCmd('git symbolic-ref HEAD', cwd=tempdir)382        self.assertEqual(result.output.strip(), "refs/heads/devtool", 'Wrong branch in git repo')383        # Try building384        bitbake(testrecipe)385    @testcase(1167)386    def test_devtool_modify_localfiles(self):387        # Check preconditions388        workspacedir = os.path.join(self.builddir, 'workspace')389        self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')390        testrecipe = 'lighttpd'391        src_uri = (get_bb_var('SRC_URI', testrecipe) or '').split()392        foundlocal = False393        for item in src_uri:394            if item.startswith('file://') and '.patch' not in item:395                foundlocal = True396                break397        self.assertTrue(foundlocal, 'This test expects the %s recipe to fetch local files and it seems that it no longer does' % testrecipe)398        # Clean up anything in the workdir/sysroot/sstate cache399        bitbake('%s -c cleansstate' % testrecipe)400        # Try modifying a recipe401        tempdir = tempfile.mkdtemp(prefix='devtoolqa')402        self.track_for_cleanup(tempdir)403        self.track_for_cleanup(workspacedir)404        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')405        self.add_command_to_tearDown('bitbake -c clean %s' % testrecipe)406        result = runCmd('devtool modify %s -x %s' % (testrecipe, tempdir))407        self.assertTrue(os.path.exists(os.path.join(tempdir, 'configure.ac')), 'Extracted source could not be found')408        self.assertTrue(os.path.exists(os.path.join(workspacedir, 'conf', 'layer.conf')), 'Workspace directory not created')409        matches = glob.glob(os.path.join(workspacedir, 'appends', '%s_*.bbappend' % testrecipe))410        self.assertTrue(matches, 'bbappend not created')411        # Test devtool status412        result = runCmd('devtool status')413        self.assertIn(testrecipe, result.output)414        self.assertIn(tempdir, result.output)415        # Try building416        bitbake(testrecipe)417    @testcase(1169)418    def test_devtool_update_recipe(self):419        # Check preconditions420        workspacedir = os.path.join(self.builddir, 'workspace')421        self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')422        testrecipe = 'minicom'423        recipefile = get_bb_var('FILE', testrecipe)424        src_uri = get_bb_var('SRC_URI', testrecipe)425        self.assertNotIn('git://', src_uri, 'This test expects the %s recipe to NOT be a git recipe' % testrecipe)426        result = runCmd('git status . --porcelain', cwd=os.path.dirname(recipefile))427        self.assertEqual(result.output.strip(), "", '%s recipe is not clean' % testrecipe)428        # First, modify a recipe429        tempdir = tempfile.mkdtemp(prefix='devtoolqa')430        self.track_for_cleanup(tempdir)431        self.track_for_cleanup(workspacedir)432        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')433        # (don't bother with cleaning the recipe on teardown, we won't be building it)434        result = runCmd('devtool modify %s -x %s' % (testrecipe, tempdir))435        # Check git repo436        self.assertTrue(os.path.isdir(os.path.join(tempdir, '.git')), 'git repository for external source tree not found')437        result = runCmd('git status --porcelain', cwd=tempdir)438        self.assertEqual(result.output.strip(), "", 'Created git repo is not clean')439        result = runCmd('git symbolic-ref HEAD', cwd=tempdir)440        self.assertEqual(result.output.strip(), "refs/heads/devtool", 'Wrong branch in git repo')441        # Add a couple of commits442        # FIXME: this only tests adding, need to also test update and remove443        result = runCmd('echo "Additional line" >> README', cwd=tempdir)444        result = runCmd('git commit -a -m "Change the README"', cwd=tempdir)445        result = runCmd('echo "A new file" > devtool-new-file', cwd=tempdir)446        result = runCmd('git add devtool-new-file', cwd=tempdir)447        result = runCmd('git commit -m "Add a new file"', cwd=tempdir)448        self.add_command_to_tearDown('cd %s; rm %s/*.patch; git checkout %s %s' % (os.path.dirname(recipefile), testrecipe, testrecipe, os.path.basename(recipefile)))449        result = runCmd('devtool update-recipe %s' % testrecipe)450        result = runCmd('git status . --porcelain', cwd=os.path.dirname(recipefile))451        self.assertNotEqual(result.output.strip(), "", '%s recipe should be modified' % testrecipe)452        status = result.output.splitlines()453        self.assertEqual(len(status), 3, 'Less/more files modified than expected. Entire status:\n%s' % result.output)454        for line in status:455            if line.endswith('0001-Change-the-README.patch'):456                self.assertEqual(line[:3], '?? ', 'Unexpected status in line: %s' % line)457            elif line.endswith('0002-Add-a-new-file.patch'):458                self.assertEqual(line[:3], '?? ', 'Unexpected status in line: %s' % line)459            elif re.search('%s_[^_]*.bb$' % testrecipe, line):460                self.assertEqual(line[:3], ' M ', 'Unexpected status in line: %s' % line)461            else:462                raise AssertionError('Unexpected modified file in status: %s' % line)463    @testcase(1172)464    def test_devtool_update_recipe_git(self):465        # Check preconditions466        workspacedir = os.path.join(self.builddir, 'workspace')467        self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')468        testrecipe = 'mtd-utils'469        recipefile = get_bb_var('FILE', testrecipe)470        src_uri = get_bb_var('SRC_URI', testrecipe)471        self.assertIn('git://', src_uri, 'This test expects the %s recipe to be a git recipe' % testrecipe)472        patches = []473        for entry in src_uri.split():474            if entry.startswith('file://') and entry.endswith('.patch'):475                patches.append(entry[7:].split(';')[0])476        self.assertGreater(len(patches), 0, 'The %s recipe does not appear to contain any patches, so this test will not be effective' % testrecipe)477        result = runCmd('git status . --porcelain', cwd=os.path.dirname(recipefile))478        self.assertEqual(result.output.strip(), "", '%s recipe is not clean' % testrecipe)479        # First, modify a recipe480        tempdir = tempfile.mkdtemp(prefix='devtoolqa')481        self.track_for_cleanup(tempdir)482        self.track_for_cleanup(workspacedir)483        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')484        # (don't bother with cleaning the recipe on teardown, we won't be building it)485        result = runCmd('devtool modify %s -x %s' % (testrecipe, tempdir))486        # Check git repo487        self.assertTrue(os.path.isdir(os.path.join(tempdir, '.git')), 'git repository for external source tree not found')488        result = runCmd('git status --porcelain', cwd=tempdir)489        self.assertEqual(result.output.strip(), "", 'Created git repo is not clean')490        result = runCmd('git symbolic-ref HEAD', cwd=tempdir)491        self.assertEqual(result.output.strip(), "refs/heads/devtool", 'Wrong branch in git repo')492        # Add a couple of commits493        # FIXME: this only tests adding, need to also test update and remove494        result = runCmd('echo "# Additional line" >> Makefile', cwd=tempdir)495        result = runCmd('git commit -a -m "Change the Makefile"', cwd=tempdir)496        result = runCmd('echo "A new file" > devtool-new-file', cwd=tempdir)497        result = runCmd('git add devtool-new-file', cwd=tempdir)498        result = runCmd('git commit -m "Add a new file"', cwd=tempdir)499        self.add_command_to_tearDown('cd %s; git checkout %s %s' % (os.path.dirname(recipefile), testrecipe, os.path.basename(recipefile)))500        result = runCmd('devtool update-recipe %s' % testrecipe)501        result = runCmd('git status . --porcelain', cwd=os.path.dirname(recipefile))502        self.assertNotEqual(result.output.strip(), "", '%s recipe should be modified' % testrecipe)503        status = result.output.splitlines()504        for line in status:505            for patch in patches:506                if line.endswith(patch):507                    self.assertEqual(line[:3], ' D ', 'Unexpected status in line: %s' % line)508                    break509            else:510                if re.search('%s_[^_]*.bb$' % testrecipe, line):511                    self.assertEqual(line[:3], ' M ', 'Unexpected status in line: %s' % line)512                else:513                    raise AssertionError('Unexpected modified file in status: %s' % line)514        result = runCmd('git diff %s' % os.path.basename(recipefile), cwd=os.path.dirname(recipefile))515        addlines = ['SRCREV = ".*"', 'SRC_URI = "git://git.infradead.org/mtd-utils.git"']516        srcurilines = src_uri.split()517        srcurilines[0] = 'SRC_URI = "' + srcurilines[0]518        srcurilines.append('"')519        removelines = ['SRCREV = ".*"'] + srcurilines520        for line in result.output.splitlines():521            if line.startswith('+++') or line.startswith('---'):522                continue523            elif line.startswith('+'):524                matched = False525                for item in addlines:526                    if re.match(item, line[1:].strip()):527                        matched = True528                        break529                self.assertTrue(matched, 'Unexpected diff add line: %s' % line)530            elif line.startswith('-'):531                matched = False532                for item in removelines:533                    if re.match(item, line[1:].strip()):534                        matched = True535                        break536                self.assertTrue(matched, 'Unexpected diff remove line: %s' % line)537    @testcase(1170)538    def test_devtool_update_recipe_append(self):539        # Check preconditions540        workspacedir = os.path.join(self.builddir, 'workspace')541        self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')542        testrecipe = 'mdadm'543        recipefile = get_bb_var('FILE', testrecipe)544        src_uri = get_bb_var('SRC_URI', testrecipe)545        self.assertNotIn('git://', src_uri, 'This test expects the %s recipe to NOT be a git recipe' % testrecipe)546        result = runCmd('git status . --porcelain', cwd=os.path.dirname(recipefile))547        self.assertEqual(result.output.strip(), "", '%s recipe is not clean' % testrecipe)548        # First, modify a recipe549        tempdir = tempfile.mkdtemp(prefix='devtoolqa')550        tempsrcdir = os.path.join(tempdir, 'source')551        templayerdir = os.path.join(tempdir, 'layer')552        self.track_for_cleanup(tempdir)553        self.track_for_cleanup(workspacedir)554        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')555        # (don't bother with cleaning the recipe on teardown, we won't be building it)556        result = runCmd('devtool modify %s -x %s' % (testrecipe, tempsrcdir))557        # Check git repo558        self.assertTrue(os.path.isdir(os.path.join(tempsrcdir, '.git')), 'git repository for external source tree not found')559        result = runCmd('git status --porcelain', cwd=tempsrcdir)560        self.assertEqual(result.output.strip(), "", 'Created git repo is not clean')561        result = runCmd('git symbolic-ref HEAD', cwd=tempsrcdir)562        self.assertEqual(result.output.strip(), "refs/heads/devtool", 'Wrong branch in git repo')563        # Add a commit564        result = runCmd("sed 's!\\(#define VERSION\\W*\"[^\"]*\\)\"!\\1-custom\"!' -i ReadMe.c", cwd=tempsrcdir)565        result = runCmd('git commit -a -m "Add our custom version"', cwd=tempsrcdir)566        self.add_command_to_tearDown('cd %s; rm -f %s/*.patch; git checkout .' % (os.path.dirname(recipefile), testrecipe))567        # Create a temporary layer and add it to bblayers.conf568        self._create_temp_layer(templayerdir, True, 'selftestupdaterecipe')569        # Create the bbappend570        result = runCmd('devtool update-recipe %s -a %s' % (testrecipe, templayerdir))571        self.assertNotIn('WARNING:', result.output)572        # Check recipe is still clean573        result = runCmd('git status . --porcelain', cwd=os.path.dirname(recipefile))574        self.assertEqual(result.output.strip(), "", '%s recipe is not clean' % testrecipe)575        # Check bbappend was created576        splitpath = os.path.dirname(recipefile).split(os.sep)577        appenddir = os.path.join(templayerdir, splitpath[-2], splitpath[-1])578        bbappendfile = self._check_bbappend(testrecipe, recipefile, appenddir)579        patchfile = os.path.join(appenddir, testrecipe, '0001-Add-our-custom-version.patch')580        self.assertTrue(os.path.exists(patchfile), 'Patch file not created')581        # Check bbappend contents582        expectedlines = ['FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n',583                         '\n',584                         'SRC_URI += "file://0001-Add-our-custom-version.patch"\n',585                         '\n']586        with open(bbappendfile, 'r') as f:587            self.assertEqual(expectedlines, f.readlines())588        # Check we can run it again and bbappend isn't modified589        result = runCmd('devtool update-recipe %s -a %s' % (testrecipe, templayerdir))590        with open(bbappendfile, 'r') as f:591            self.assertEqual(expectedlines, f.readlines())592        # Drop new commit and check patch gets deleted593        result = runCmd('git reset HEAD^', cwd=tempsrcdir)594        result = runCmd('devtool update-recipe %s -a %s' % (testrecipe, templayerdir))595        self.assertFalse(os.path.exists(patchfile), 'Patch file not deleted')596        expectedlines2 = ['FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n',597                         '\n']598        with open(bbappendfile, 'r') as f:599            self.assertEqual(expectedlines2, f.readlines())600        # Put commit back and check we can run it if layer isn't in bblayers.conf601        os.remove(bbappendfile)602        result = runCmd('git commit -a -m "Add our custom version"', cwd=tempsrcdir)603        result = runCmd('bitbake-layers remove-layer %s' % templayerdir, cwd=self.builddir)604        result = runCmd('devtool update-recipe %s -a %s' % (testrecipe, templayerdir))605        self.assertIn('WARNING: Specified layer is not currently enabled in bblayers.conf', result.output)606        self.assertTrue(os.path.exists(patchfile), 'Patch file not created (with disabled layer)')607        with open(bbappendfile, 'r') as f:608            self.assertEqual(expectedlines, f.readlines())609        # Deleting isn't expected to work under these circumstances610    @testcase(1171)611    def test_devtool_update_recipe_append_git(self):612        # Check preconditions613        workspacedir = os.path.join(self.builddir, 'workspace')614        self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')615        testrecipe = 'mtd-utils'616        recipefile = get_bb_var('FILE', testrecipe)617        src_uri = get_bb_var('SRC_URI', testrecipe)618        self.assertIn('git://', src_uri, 'This test expects the %s recipe to be a git recipe' % testrecipe)619        for entry in src_uri.split():620            if entry.startswith('git://'):621                git_uri = entry622                break623        result = runCmd('git status . --porcelain', cwd=os.path.dirname(recipefile))624        self.assertEqual(result.output.strip(), "", '%s recipe is not clean' % testrecipe)625        # First, modify a recipe626        tempdir = tempfile.mkdtemp(prefix='devtoolqa')627        tempsrcdir = os.path.join(tempdir, 'source')628        templayerdir = os.path.join(tempdir, 'layer')629        self.track_for_cleanup(tempdir)630        self.track_for_cleanup(workspacedir)631        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')632        # (don't bother with cleaning the recipe on teardown, we won't be building it)633        result = runCmd('devtool modify %s -x %s' % (testrecipe, tempsrcdir))634        # Check git repo635        self.assertTrue(os.path.isdir(os.path.join(tempsrcdir, '.git')), 'git repository for external source tree not found')636        result = runCmd('git status --porcelain', cwd=tempsrcdir)637        self.assertEqual(result.output.strip(), "", 'Created git repo is not clean')638        result = runCmd('git symbolic-ref HEAD', cwd=tempsrcdir)639        self.assertEqual(result.output.strip(), "refs/heads/devtool", 'Wrong branch in git repo')640        # Add a commit641        result = runCmd('echo "# Additional line" >> Makefile', cwd=tempsrcdir)642        result = runCmd('git commit -a -m "Change the Makefile"', cwd=tempsrcdir)643        self.add_command_to_tearDown('cd %s; rm -f %s/*.patch; git checkout .' % (os.path.dirname(recipefile), testrecipe))644        # Create a temporary layer645        os.makedirs(os.path.join(templayerdir, 'conf'))646        with open(os.path.join(templayerdir, 'conf', 'layer.conf'), 'w') as f:647            f.write('BBPATH .= ":${LAYERDIR}"\n')648            f.write('BBFILES += "${LAYERDIR}/recipes-*/*/*.bbappend"\n')649            f.write('BBFILE_COLLECTIONS += "oeselftesttemplayer"\n')650            f.write('BBFILE_PATTERN_oeselftesttemplayer = "^${LAYERDIR}/"\n')651            f.write('BBFILE_PRIORITY_oeselftesttemplayer = "999"\n')652            f.write('BBFILE_PATTERN_IGNORE_EMPTY_oeselftesttemplayer = "1"\n')653        self.add_command_to_tearDown('bitbake-layers remove-layer %s || true' % templayerdir)654        result = runCmd('bitbake-layers add-layer %s' % templayerdir, cwd=self.builddir)655        # Create the bbappend656        result = runCmd('devtool update-recipe %s -a %s' % (testrecipe, templayerdir))657        self.assertNotIn('WARNING:', result.output)658        # Check recipe is still clean659        result = runCmd('git status . --porcelain', cwd=os.path.dirname(recipefile))660        self.assertEqual(result.output.strip(), "", '%s recipe is not clean' % testrecipe)661        # Check bbappend was created662        splitpath = os.path.dirname(recipefile).split(os.sep)663        appenddir = os.path.join(templayerdir, splitpath[-2], splitpath[-1])664        bbappendfile = self._check_bbappend(testrecipe, recipefile, appenddir)665        self.assertFalse(os.path.exists(os.path.join(appenddir, testrecipe)), 'Patch directory should not be created')666        # Check bbappend contents667        result = runCmd('git rev-parse HEAD', cwd=tempsrcdir)668        expectedlines = ['SRCREV = "%s"\n' % result.output,669                         '\n',670                         'SRC_URI = "%s"\n' % git_uri,671                         '\n']672        with open(bbappendfile, 'r') as f:673            self.assertEqual(expectedlines, f.readlines())674        # Check we can run it again and bbappend isn't modified675        result = runCmd('devtool update-recipe %s -a %s' % (testrecipe, templayerdir))676        with open(bbappendfile, 'r') as f:677            self.assertEqual(expectedlines, f.readlines())678        # Drop new commit and check SRCREV changes679        result = runCmd('git reset HEAD^', cwd=tempsrcdir)680        result = runCmd('devtool update-recipe %s -a %s' % (testrecipe, templayerdir))681        self.assertFalse(os.path.exists(os.path.join(appenddir, testrecipe)), 'Patch directory should not be created')682        result = runCmd('git rev-parse HEAD', cwd=tempsrcdir)683        expectedlines = ['SRCREV = "%s"\n' % result.output,684                         '\n',685                         'SRC_URI = "%s"\n' % git_uri,686                         '\n']687        with open(bbappendfile, 'r') as f:688            self.assertEqual(expectedlines, f.readlines())689        # Put commit back and check we can run it if layer isn't in bblayers.conf690        os.remove(bbappendfile)691        result = runCmd('git commit -a -m "Change the Makefile"', cwd=tempsrcdir)692        result = runCmd('bitbake-layers remove-layer %s' % templayerdir, cwd=self.builddir)693        result = runCmd('devtool update-recipe %s -a %s' % (testrecipe, templayerdir))694        self.assertIn('WARNING: Specified layer is not currently enabled in bblayers.conf', result.output)695        self.assertFalse(os.path.exists(os.path.join(appenddir, testrecipe)), 'Patch directory should not be created')696        result = runCmd('git rev-parse HEAD', cwd=tempsrcdir)697        expectedlines = ['SRCREV = "%s"\n' % result.output,698                         '\n',699                         'SRC_URI = "%s"\n' % git_uri,700                         '\n']701        with open(bbappendfile, 'r') as f:702            self.assertEqual(expectedlines, f.readlines())703        # Deleting isn't expected to work under these circumstances704    @testcase(1163)705    def test_devtool_extract(self):706        # Check preconditions707        workspacedir = os.path.join(self.builddir, 'workspace')708        self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')709        tempdir = tempfile.mkdtemp(prefix='devtoolqa')710        # Try devtool extract711        self.track_for_cleanup(tempdir)712        self.track_for_cleanup(workspacedir)713        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')714        result = runCmd('devtool extract remake %s' % tempdir)715        self.assertTrue(os.path.exists(os.path.join(tempdir, 'Makefile.am')), 'Extracted source could not be found')716        self.assertTrue(os.path.isdir(os.path.join(tempdir, '.git')), 'git repository for external source tree not found')717    @testcase(1168)718    def test_devtool_reset_all(self):719        # Check preconditions720        workspacedir = os.path.join(self.builddir, 'workspace')721        self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')722        tempdir = tempfile.mkdtemp(prefix='devtoolqa')723        self.track_for_cleanup(tempdir)724        self.track_for_cleanup(workspacedir)725        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')726        testrecipe1 = 'mdadm'727        testrecipe2 = 'cronie'728        result = runCmd('devtool modify -x %s %s' % (testrecipe1, os.path.join(tempdir, testrecipe1)))729        result = runCmd('devtool modify -x %s %s' % (testrecipe2, os.path.join(tempdir, testrecipe2)))730        result = runCmd('devtool build %s' % testrecipe1)731        result = runCmd('devtool build %s' % testrecipe2)732        stampprefix1 = get_bb_var('STAMP', testrecipe1)733        self.assertTrue(stampprefix1, 'Unable to get STAMP value for recipe %s' % testrecipe1)734        stampprefix2 = get_bb_var('STAMP', testrecipe2)735        self.assertTrue(stampprefix2, 'Unable to get STAMP value for recipe %s' % testrecipe2)736        result = runCmd('devtool reset -a')737        self.assertIn(testrecipe1, result.output)738        self.assertIn(testrecipe2, result.output)739        result = runCmd('devtool status')740        self.assertNotIn(testrecipe1, result.output)741        self.assertNotIn(testrecipe2, result.output)742        matches1 = glob.glob(stampprefix1 + '*')743        self.assertFalse(matches1, 'Stamp files exist for recipe %s that should have been cleaned' % testrecipe1)744        matches2 = glob.glob(stampprefix2 + '*')745        self.assertFalse(matches2, 'Stamp files exist for recipe %s that should have been cleaned' % testrecipe2)746    def test_devtool_deploy_target(self):747        # NOTE: Whilst this test would seemingly be better placed as a runtime test,748        # unfortunately the runtime tests run under bitbake and you can't run749        # devtool within bitbake (since devtool needs to run bitbake itself).750        # Additionally we are testing build-time functionality as well, so751        # really this has to be done as an oe-selftest test.752        #753        # Check preconditions754        machine = get_bb_var('MACHINE')755        if not machine.startswith('qemu'):756            self.skipTest('This test only works with qemu machines')757        if not os.path.exists('/etc/runqemu-nosudo'):758            self.skipTest('You must set up tap devices with scripts/runqemu-gen-tapdevs before running this test')759        result = runCmd('PATH="$PATH:/sbin:/usr/sbin" ip tuntap show', ignore_status=True)760        if result.status != 0:761            result = runCmd('PATH="$PATH:/sbin:/usr/sbin" ifconfig -a', ignore_status=True)762            if result.status != 0:763                self.skipTest('Failed to determine if tap devices exist with ifconfig or ip: %s' % result.output)764        for line in result.output.splitlines():765            if line.startswith('tap'):766                break767        else:768            self.skipTest('No tap devices found - you must set up tap devices with scripts/runqemu-gen-tapdevs before running this test')769        workspacedir = os.path.join(self.builddir, 'workspace')770        self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')771        import pexpect772        # Definitions773        testrecipe = 'mdadm'774        testfile = '/sbin/mdadm'775        testimage = 'oe-selftest-image'776        testhost = '192.168.7.2'777        testcommand = '/sbin/mdadm --help'778        # Build an image to run779        bitbake("%s qemu-native qemu-helper-native" % testimage)780        deploy_dir_image = get_bb_var('DEPLOY_DIR_IMAGE')781        self.add_command_to_tearDown('bitbake -c clean %s' % testimage)782        self.add_command_to_tearDown('rm -f %s/%s*' % (deploy_dir_image, testimage))783        # Clean recipe so the first deploy will fail784        bitbake("%s -c clean" % testrecipe)785        # Try devtool modify786        tempdir = tempfile.mkdtemp(prefix='devtoolqa')787        self.track_for_cleanup(tempdir)788        self.track_for_cleanup(workspacedir)789        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')790        self.add_command_to_tearDown('bitbake -c clean %s' % testrecipe)791        result = runCmd('devtool modify %s -x %s' % (testrecipe, tempdir))792        # Test that deploy-target at this point fails (properly)793        result = runCmd('devtool deploy-target -n %s root@%s' % (testrecipe, testhost), ignore_status=True)794        self.assertNotEqual(result.output, 0, 'devtool deploy-target should have failed, output: %s' % result.output)795        self.assertNotIn(result.output, 'Traceback', 'devtool deploy-target should have failed with a proper error not a traceback, output: %s' % result.output)796        result = runCmd('devtool build %s' % testrecipe)797        # First try a dry-run of deploy-target798        result = runCmd('devtool deploy-target -n %s root@%s' % (testrecipe, testhost))799        self.assertIn('  %s' % testfile, result.output)800        # Boot the image801        console = pexpect.spawn('runqemu %s %s qemuparams="-snapshot" nographic' % (machine, testimage))802        console.expect("login:", timeout=120)803        # Now really test deploy-target804        result = runCmd('devtool deploy-target -c %s root@%s' % (testrecipe, testhost))805        # Run a test command to see if it was installed properly806        sshargs = '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no'807        result = runCmd('ssh %s root@%s %s' % (sshargs, testhost, testcommand))808        # Check if it deployed all of the files with the right ownership/perms809        # First look on the host - need to do this under pseudo to get the correct ownership/perms810        installdir = get_bb_var('D', testrecipe)811        fakerootenv = get_bb_var('FAKEROOTENV', testrecipe)812        fakerootcmd = get_bb_var('FAKEROOTCMD', testrecipe)813        result = runCmd('%s %s find . -type f -exec ls -l {} \;' % (fakerootenv, fakerootcmd), cwd=installdir)814        filelist1 = self._process_ls_output(result.output)815        # Now look on the target816        tempdir2 = tempfile.mkdtemp(prefix='devtoolqa')817        self.track_for_cleanup(tempdir2)818        tmpfilelist = os.path.join(tempdir2, 'files.txt')819        with open(tmpfilelist, 'w') as f:820            for line in filelist1:821                splitline = line.split()822                f.write(splitline[-1] + '\n')823        result = runCmd('cat %s | ssh -q %s root@%s \'xargs ls -l\'' % (tmpfilelist, sshargs, testhost))824        filelist2 = self._process_ls_output(result.output)825        filelist1.sort(key=lambda item: item.split()[-1])826        filelist2.sort(key=lambda item: item.split()[-1])827        self.assertEqual(filelist1, filelist2)828        # Test undeploy-target829        result = runCmd('devtool undeploy-target -c %s root@%s' % (testrecipe, testhost))830        result = runCmd('ssh %s root@%s %s' % (sshargs, testhost, testcommand), ignore_status=True)831        self.assertNotEqual(result, 0, 'undeploy-target did not remove command as it should have')...

Full Screen

Full Screen

setup_federated_node.py

Source:setup_federated_node.py Github

copy

Full Screen

1#! /usr/bin/env python32"""3Sets up an Ubuntu 14.04 x64 server to be a Counterblock Federated Node.4NOTE: The system should be properly secured before running this script.5TODO: This is admittedly a (bit of a) hack. In the future, take this kind of functionality out to a .deb with6      a postinst script to do all of this, possibly.7"""8import os9import sys10import re11import time12import getopt13import logging14import shutil15import socket16import urllib17import zipfile18import platform19import collections20import tempfile21import subprocess22import stat23import string24import random25try: #ignore import errors on windows26    import pwd27    import grp28except ImportError:29    pass30USERNAME = "xch"31DAEMON_USERNAME = "xchd"32REPO_COUNTERPARTYD_BUILD = "https://github.com/ClearingHouse/clearinghoused_build.git"33REPO_COUNTERWALLET = "https://github.com/ClearingHouse/clearwallet.git"34def pass_generator(size=14, chars=string.ascii_uppercase + string.ascii_lowercase + string.digits):35    return ''.join(random.choice(chars) for x in range(size))36def runcmd(command, abort_on_failure=True):37    logging.debug("RUNNING COMMAND: %s" % command)38    ret = os.system(command)39    if abort_on_failure and ret != 0:40        logging.error("Command failed: '%s'" % command)41        sys.exit(1)42def modify_config(param_re, content_to_add, filenames, replace_if_exists=True, dotall=False, add_newline=True):43    if not isinstance(filenames, (list, tuple)):44        filenames = [filenames,]45    re_flags = re.MULTILINE | re.DOTALL if dotall else re.MULTILINE46    if add_newline and not content_to_add.endswith('\n'):47        content_to_add += '\n'48    for filename in filenames:49        f = open(filename, 'r')50        content = f.read()51        f.close()52        if content[-1] != '\n':53            content += '\n'54        if not re.search(param_re, content, re_flags): #missing; add to config55            content += content_to_add56        elif replace_if_exists: #replace in config57            content = re.sub(param_re, content_to_add, content, flags=re_flags)58        f = open(filename, 'w')59        f.write(content)60        f.close()61def modify_cp_config(param_re, content_to_add, replace_if_exists=True, config='clearinghoused', net='both'):62    assert config in ('clearinghoused', 'clearblockd', 'both')63    assert net in ('mainnet', 'testnet', 'both')64    cfg_filenames = []65    if config in ('clearinghoused', 'both'):66        if net in ('mainnet', 'both'):67            cfg_filenames.append(os.path.join(os.path.expanduser('~'+USERNAME), ".config", "clearinghoused", "clearinghoused.conf"))68        if net in ('testnet', 'both'):69            cfg_filenames.append(os.path.join(os.path.expanduser('~'+USERNAME), ".config", "clearinghoused-testnet", "clearinghoused.conf"))70    if config in ('clearblockd', 'both'):71        if net in ('mainnet', 'both'):72            cfg_filenames.append(os.path.join(os.path.expanduser('~'+USERNAME), ".config", "clearblockd", "clearblockd.conf"))73        if net in ('testnet', 'both'):74            cfg_filenames.append(os.path.join(os.path.expanduser('~'+USERNAME), ".config", "clearblockd-testnet", "clearblockd.conf"))75    modify_config(param_re, content_to_add, cfg_filenames, replace_if_exists=replace_if_exists)76def ask_question(question, options, default_option):77    assert isinstance(options, (list, tuple))78    assert default_option in options79    answer = None80    while True:81        answer = input(question + ": ")82        answer = answer.lower()83        if answer and answer not in options:84            logging.error("Please enter one of: " + ', '.join(options))85        else:86            if answer == '': answer = default_option87            break88    return answer89def git_repo_clone(branch, repo_dir, repo_url, run_as_user, hash=None):90    if branch == 'AUTO':91        try:92            branch = subprocess.check_output("cd %s && git rev-parse --abbrev-ref HEAD" % (93                os.path.expanduser("~%s/%s" % (USERNAME, repo_dir))), shell=True).strip().decode('utf-8')94        except:95            raise Exception("Cannot get current get branch for %s." % repo_dir)96    logging.info("Checking out/updating %s:%s from git..." % (repo_dir, branch))97    if os.path.exists(os.path.expanduser("~%s/%s" % (USERNAME, repo_dir))):98        runcmd("cd ~%s/%s && git pull origin %s" % (USERNAME, repo_dir, branch))99    else:100        runcmd("git clone -b %s %s ~%s/%s" % (branch, repo_url, USERNAME, repo_dir))101    if hash:102        runcmd("cd ~%s/%s && git reset --hard %s" % (USERNAME, repo_dir, hash))103    runcmd("cd ~%s/%s && git config core.sharedRepository group && find ~%s/%s -type d -print0 | xargs -0 chmod g+s" % (104        USERNAME, repo_dir, USERNAME, repo_dir)) #to allow for group git actions105    runcmd("chown -R %s:%s ~%s/%s" % (USERNAME, USERNAME, USERNAME, repo_dir))106    runcmd("chmod -R u+rw,g+rw,o+r,o-w ~%s/%s" % (USERNAME, repo_dir)) #just in case107def do_prerun_checks():108    #make sure this is running on a supported OS109    if os.name != "posix" or platform.dist()[0] != "Ubuntu" or platform.architecture()[0] != '64bit':110        logging.error("Only 64bit Ubuntu Linux is supported at this time")111        sys.exit(1)112    ubuntu_release = platform.linux_distribution()[1]113    if ubuntu_release != "14.04":114        logging.error("Only Ubuntu 14.04 supported for Clearblock Federated Node install.")115        sys.exit(1)116    #script must be run as root117    if os.geteuid() != 0:118        logging.error("This script must be run as root (use 'sudo' to run)")119        sys.exit(1)120    if os.name == "posix" and "SUDO_USER" not in os.environ:121        logging.error("Please use `sudo` to run this script.")122        sys.exit(1)123def do_base_setup(run_as_user, branch, base_path, dist_path):124    """This creates the xch and xchd users and checks out the counterpartyd_build system from git"""125    #change time to UTC126    runcmd("ln -sf /usr/share/zoneinfo/UTC /etc/localtime")127    #install some necessary base deps128    runcmd("apt-get update")129    runcmd("apt-get -y install git-core software-properties-common python-software-properties build-essential ssl-cert ntp")130    runcmd("apt-get update")131    #node-gyp building for insight has ...issues out of the box on Ubuntu... use Chris Lea's nodejs build instead, which is newer132    runcmd("apt-get -y remove nodejs npm gyp")133    runcmd("add-apt-repository -y ppa:chris-lea/node.js")134    runcmd("apt-get update")135    runcmd("apt-get -y install nodejs") #includes npm136    #Create xcp user, under which the files will be stored, and who will own the files, etc137    try:138        pwd.getpwnam(USERNAME)139    except:140        logging.info("Creating user '%s' ..." % USERNAME)141        runcmd("adduser --system --disabled-password --shell /bin/false --group %s" % USERNAME)142    #Create xcpd user (to run counterpartyd, counterblockd, insight, bitcoind, nginx) if not already made143    try:144        pwd.getpwnam(DAEMON_USERNAME)145    except:146        logging.info("Creating user '%s' ..." % DAEMON_USERNAME)147        user_homedir = os.path.expanduser("~" + USERNAME)148        runcmd("adduser --system --disabled-password --shell /bin/false --ingroup nogroup --home %s %s" % (user_homedir, DAEMON_USERNAME))149    #add the run_as_user to the xcp group150    runcmd("adduser %s %s" % (run_as_user, USERNAME))151    #Check out counterpartyd-build repo under this user's home dir and use that for the build152    git_repo_clone(branch, "clearinghoused_build", REPO_COUNTERPARTYD_BUILD, run_as_user)153    #enhance fd limits for the xcpd user154    runcmd("cp -af %s/linux/other/xchd_security_limits.conf /etc/security/limits.d/" % dist_path)155def do_security_setup(run_as_user, branch, base_path, dist_path):156    """Some helpful security-related tasks, to tighten up the box"""157    #modify host.conf158    modify_config(r'^nospoof on$', 'nospoof on', '/etc/host.conf')159    #enable automatic security updates160    runcmd("apt-get -y install unattended-upgrades")161    runcmd('''bash -c "echo -e 'APT::Periodic::Update-Package-Lists "1";\nAPT::Periodic::Unattended-Upgrade "1";' > /etc/apt/apt.conf.d/20auto-upgrades" ''')162    runcmd("dpkg-reconfigure -fnoninteractive -plow unattended-upgrades")163    #sysctl164    runcmd("install -m 0644 -o root -g root -D %s/linux/other/sysctl_rules.conf /etc/sysctl.d/60-tweaks.conf" % dist_path)165    #set up fail2ban166    runcmd("apt-get -y install fail2ban")167    runcmd("install -m 0644 -o root -g root -D %s/linux/other/fail2ban.jail.conf /etc/fail2ban/jail.d/clearblock.conf" % dist_path)168    runcmd("service fail2ban restart")169    #set up psad170    runcmd("apt-get -y install psad")171    modify_config(r'^ENABLE_AUTO_IDS\s+?N;$', 'ENABLE_AUTO_IDS\tY;', '/etc/psad/psad.conf')172    modify_config(r'^ENABLE_AUTO_IDS_EMAILS\s+?Y;$', 'ENABLE_AUTO_IDS_EMAILS\tN;', '/etc/psad/psad.conf')173    for f in ['/etc/ufw/before.rules', '/etc/ufw/before6.rules']:174        modify_config(r'^# End required lines.*?# allow all on loopback$',175            '# End required lines\n\n#CUSTOM: for psad\n-A INPUT -j LOG\n-A FORWARD -j LOG\n\n# allow all on loopback',176            f, dotall=True, add_newline=False)177    runcmd("psad -R && psad --sig-update")178    runcmd("service ufw restart")179    runcmd("service psad restart")180    #set up chkrootkit, rkhunter181    runcmd("apt-get -y install rkhunter chkrootkit")182    runcmd('bash -c "rkhunter --update; exit 0"')183    runcmd("rkhunter --propupd")184    runcmd('bash -c "rkhunter --check --sk; exit 0"')185    runcmd("rkhunter --propupd")186    #logwatch187    runcmd("apt-get -y install logwatch libdate-manip-perl")188    #apparmor189    runcmd("apt-get -y install apparmor apparmor-profiles")190    #auditd191    #note that auditd will need a reboot to fully apply the rules, due to it operating in "immutable mode" by default192    runcmd("apt-get -y install auditd audispd-plugins")193    runcmd("install -m 0640 -o root -g root -D %s/linux/other/audit.rules /etc/audit/rules.d/clearblock.rules" % dist_path)194    runcmd("service auditd restart")195    #iwatch196    runcmd("apt-get -y install iwatch")197    modify_config(r'^START_DAEMON=.*?$', 'START_DAEMON=true', '/etc/default/iwatch')198    runcmd("install -m 0644 -o root -g root -D %s/linux/other/iwatch.xml /etc/iwatch/iwatch.xml" % dist_path)199    modify_config(r'guard email="root@localhost"', 'guard email="noreply@%s"' % socket.gethostname(), '/etc/iwatch/iwatch.xml')200    runcmd("service iwatch restart")201def do_bitcoind_setup(run_as_user, branch, base_path, dist_path, run_mode):202    """Installs and configures bitcoind"""203    user_homedir = os.path.expanduser("~" + USERNAME)204    bitcoind_rpc_password = pass_generator()205    bitcoind_rpc_password_testnet = pass_generator()206    #Install bitcoind207    BITCOIND_VER = "0.10.8"208    runcmd("rm -rf /tmp/viacoind.tar.gz /tmp/viacoin-%s-linux" % BITCOIND_VER)209    runcmd("wget -O /tmp/viacoind.tar.gz https://github.com/viacoin/viacoin/releases/download/v%s/viacoin-%s-linux64.tar.gz" % (BITCOIND_VER, BITCOIND_VER))210    runcmd("tar -C /tmp -zxvf /tmp/viacoind.tar.gz")211    runcmd("cp -af /tmp/viacoin-%s/bin/viacoind /usr/local/bin" % BITCOIND_VER)212    runcmd("cp -af /tmp/viacoin-%s/bin/viacoin-cli /usr/local/bin" % BITCOIND_VER)213    runcmd("rm -rf /tmp/viacoind.tar.gz /tmp/viacoin-%s-linux" % BITCOIND_VER)214    #Do basic inital bitcoin config (for both testnet and mainnet)215    runcmd("mkdir -p ~%s/.viacoin ~%s/.viacoin-testnet" % (USERNAME, USERNAME))216    if not os.path.exists(os.path.join(user_homedir, '.viacoin', 'viacoin.conf')):217        runcmd(r"""bash -c 'echo -e "rpcuser=rpc\nrpcpassword=%s\nserver=1\ndaemon=1\ntxindex=1\naddrindex=1\nrpcthreads=1000\nrpctimeout=300" > ~%s/.viacoin/viacoin.conf'""" % (218            bitcoind_rpc_password, USERNAME))219    else: #grab the existing RPC password220        bitcoind_rpc_password = subprocess.check_output(221            r"""bash -c "cat ~%s/.viacoin/viacoin.conf | sed -n 's/.*rpcpassword=\([^ \n]*\).*/\1/p'" """ % USERNAME, shell=True).strip().decode('utf-8')222    if not os.path.exists(os.path.join(user_homedir, '.viacoin-testnet', 'viacoin.conf')):223        runcmd(r"""bash -c 'echo -e "rpcuser=rpc\nrpcpassword=%s\nserver=1\ndaemon=1\ntxindex=1\naddrindex=1\ntestnet=1\nrpcthreads=1000\nrpctimeout=300" > ~%s/.viacoin-testnet/viacoin.conf'""" % (224            bitcoind_rpc_password_testnet, USERNAME))225    else:226        bitcoind_rpc_password_testnet = subprocess.check_output(227            r"""bash -c "cat ~%s/.viacoin-testnet/viacoin.conf | sed -n 's/.*rpcpassword=\([^ \n]*\).*/\1/p'" """228            % USERNAME, shell=True).strip().decode('utf-8')229    #Set up bitcoind startup scripts (will be disabled later from autostarting on system startup if necessary)230    runcmd("rm -f /etc/init/viacoin.conf /etc/init/viacoin-testnet.conf")231    runcmd("cp -af %s/linux/init/viacoind.conf.template /etc/init/viacoind.conf" % dist_path)232    runcmd("sed -ri \"s/\!RUN_AS_USER\!/%s/g\" /etc/init/viacoind.conf" % DAEMON_USERNAME)233    runcmd("sed -ri \"s/\!USER_HOMEDIR\!/%s/g\" /etc/init/viacoind.conf" % user_homedir.replace('/', '\/'))234    runcmd("cp -af %s/linux/init/viacoind-testnet.conf.template /etc/init/viacoind-testnet.conf" % dist_path)235    runcmd("sed -ri \"s/\!RUN_AS_USER\!/%s/g\" /etc/init/viacoind-testnet.conf" % DAEMON_USERNAME)236    runcmd("sed -ri \"s/\!USER_HOMEDIR\!/%s/g\" /etc/init/viacoind-testnet.conf" % user_homedir.replace('/', '\/'))237    #install logrotate file238    runcmd("cp -af %s/linux/logrotate/viacoind /etc/logrotate.d/viacoind" % dist_path)239    runcmd("sed -ri \"s/\!USER_HOMEDIR\!/%s/g\" /etc/logrotate.d/viacoind" % user_homedir.replace('/', '\/'))240    #disable upstart scripts from autostarting on system boot if necessary241    if run_mode == 't': #disable mainnet daemons from autostarting242        runcmd(r"""bash -c "echo 'manual' >> /etc/init/viacoind.override" """)243    else:244        runcmd("rm -f /etc/init/viacoind.override")245    if run_mode == 'm': #disable testnet daemons from autostarting246        runcmd(r"""bash -c "echo 'manual' >> /etc/init/viacoind-testnet.override" """)247    else:248        runcmd("rm -f /etc/init/viacoind-testnet.override")249    runcmd("chown -R %s:%s ~%s/.viacoin ~%s/.viacoin-testnet" % (DAEMON_USERNAME, USERNAME, USERNAME, USERNAME))250    return bitcoind_rpc_password, bitcoind_rpc_password_testnet251def do_counterparty_setup(role, run_as_user, branch, base_path, dist_path, run_mode, bitcoind_rpc_password,252bitcoind_rpc_password_testnet, clearinghoused_public, clearwallet_support_email):253    """Installs and configures counterpartyd and counterblockd"""254    user_homedir = os.path.expanduser("~" + USERNAME)255    counterpartyd_rpc_password = '1234' if role == 'clearinghoused_only' and clearinghoused_public == 'y' else pass_generator()256    counterpartyd_rpc_password_testnet = '1234' if role == 'clearinghoused_only' and clearinghoused_public == 'y' else pass_generator()257    #Run setup.py (as the XCP user, who runs it sudoed) to install and set up counterpartyd, counterblockd258    # as -y is specified, this will auto install counterblockd full node (mongo and redis) as well as setting259    # counterpartyd/counterblockd to start up at startup for both mainnet and testnet (we will override this as necessary260    # based on run_mode later in this function)261    runcmd("~%s/clearinghoused_build/setup.py -y %s --with-testnet --for-user=%s" % (262        USERNAME, '--with-clearblockd' if role != 'clearinghoused_only' else '', USERNAME))263    runcmd("cd ~%s/clearinghoused_build && git config core.sharedRepository group && find ~%s/clearinghoused_build -type d -print0 | xargs -0 chmod g+s" % (264        USERNAME, USERNAME)) #to allow for group git actions265    runcmd("chown -R %s:%s ~%s/clearinghoused_build" % (USERNAME, USERNAME, USERNAME)) #just in case266    runcmd("chmod -R u+rw,g+rw,o+r,o-w ~%s/clearinghoused_build" % USERNAME) #just in case267    #now change the counterpartyd directories to be owned by the xcpd user (and the xcp group),268    # so that the xcpd account can write to the database, saved image files (counterblockd), log files, etc269    runcmd("mkdir -p ~%s/.config/clearinghoused ~%s/.config/clearinghoused-testnet" % (USERNAME, USERNAME))270    runcmd("chown -R %s:%s ~%s/.config/clearinghoused ~%s/.config/clearinghoused-testnet" % (271        DAEMON_USERNAME, USERNAME, USERNAME, USERNAME))272    runcmd("sed -ri \"s/USER=%s/USER=%s/g\" /etc/init/clearinghoused.conf /etc/init/clearinghoused-testnet.conf" % (273        USERNAME, DAEMON_USERNAME))274    #modify the default stored bitcoind passwords in counterpartyd.conf275    modify_cp_config(r'^(backend|bitcoind)\-rpc\-password=.*?$',276        'backend-rpc-password=%s' % bitcoind_rpc_password, config='clearinghoused', net='mainnet')277    modify_cp_config(r'^(backend|bitcoind)\-rpc\-password=.*?$',278        'backend-rpc-password=%s' % bitcoind_rpc_password_testnet, config='clearinghoused', net='testnet')279    #modify the counterpartyd API rpc password in counterpartyd.conf280    modify_cp_config(r'^rpc\-password=.*?$', 'rpc-password=%s' % counterpartyd_rpc_password,281        config='clearinghoused', net='mainnet')282    modify_cp_config(r'^rpc\-password=.*?$', 'rpc-password=%s' % counterpartyd_rpc_password_testnet,283        config='clearinghoused', net='testnet')284    if role == 'clearinghoused_only' and clearinghoused_public == 'y':285        modify_cp_config(r'^rpc\-host=.*?$', 'rpc-host=0.0.0.0', config='clearinghoused')286    #disable upstart scripts from autostarting on system boot if necessary287    if run_mode == 't': #disable mainnet daemons from autostarting288        runcmd(r"""bash -c "echo 'manual' >> /etc/init/clearinghoused.override" """)289    else:290        runcmd("rm -f /etc/init/clearinghoused.override")291    if run_mode == 'm': #disable testnet daemons from autostarting292        runcmd(r"""bash -c "echo 'manual' >> /etc/init/clearinghoused-testnet.override" """)293    else:294        runcmd("rm -f /etc/init/clearinghoused-testnet.override")295    if role != 'clearinghoused_only':296        #now change the counterblockd directories to be owned by the xcpd user (and the xcp group),297        runcmd("mkdir -p ~%s/.config/clearblockd ~%s/.config/clearblockd-testnet" % (USERNAME, USERNAME))298        runcmd("chown -R %s:%s ~%s/.config/clearblockd ~%s/.config/clearblockd-testnet" % (299            DAEMON_USERNAME, USERNAME, USERNAME, USERNAME))300        runcmd("sed -ri \"s/USER=%s/USER=%s/g\" /etc/init/clearblockd.conf /etc/init/clearblockd-testnet.conf" % (301            USERNAME, DAEMON_USERNAME))302        #modify the default stored bitcoind passwords in counterblockd.conf303        modify_cp_config(r'^(backend|bitcoind)\-rpc\-password=.*?$',304            'backend-rpc-password=%s' % bitcoind_rpc_password, config='clearblockd', net='mainnet')305        modify_cp_config(r'^(backend|bitcoind)\-rpc\-password=.*?$',306            'backend-rpc-password=%s' % bitcoind_rpc_password_testnet, config='clearblockd', net='testnet')307        #modify the counterpartyd API rpc password in counterblockd.conf308        modify_cp_config(r'^clearinghoused\-rpc\-password=.*?$',309            'clearinghoused-rpc-password=%s' % counterpartyd_rpc_password, config='clearblockd', net='mainnet')310        modify_cp_config(r'^clearinghoused\-rpc\-password=.*?$',311            'clearinghoused-rpc-password=%s' % counterpartyd_rpc_password_testnet, config='clearblockd', net='testnet')312        #role-specific counterblockd.conf values313        modify_cp_config(r'^auto\-btc\-escrow\-enable=.*?$', 'auto-btc-escrow-enable=%s' %314            '1' if role == 'btcpayescrow' else '0', config='clearblockd')315        modify_cp_config(r'^armory\-utxsvr\-enable=.*?$', 'armory-utxsvr-enable=%s' %316            '1' if role == 'counterwallet' else '0', config='clearblockd')317        if role == 'counterwallet':318            modify_cp_config(r'^support\-email=.*?$', 'support-email=%s' % clearwallet_support_email,319                config='clearblockd') #may be blank string320        #disable upstart scripts from autostarting on system boot if necessary321        if run_mode == 't': #disable mainnet daemons from autostarting322            runcmd(r"""bash -c "echo 'manual' >> /etc/init/clearblockd.override" """)323        else:324            runcmd("rm -f /etc/init/clearblockd.override")325        if run_mode == 'm': #disable testnet daemons from autostarting326            runcmd(r"""bash -c "echo 'manual' >> /etc/init/clearblockd-testnet.override" """)327        else:328            runcmd("rm -f /etc/init/clearblockd-testnet.override")329def do_blockchain_service_setup(run_as_user, base_path, dist_path, run_mode, blockchain_service, bitcoin_rpc_password, bitcoin_rpc_password_testnet):330    def do_insight_setup():331        """This installs and configures insight"""332        assert blockchain_service333        user_homedir = os.path.expanduser("~" + USERNAME)334        gypdir = None335        try:336            import gyp337            gypdir = os.path.dirname(gyp.__file__)338        except:339            pass340        else:341            runcmd("mv %s %s_bkup" % (gypdir, gypdir))342            #^ fix for https://github.com/TooTallNate/node-gyp/issues/363343        git_repo_clone("master", "insight-api", "https://github.com/viacoin/insight-api.git",344            run_as_user, hash="master") #insight 0.2.7345        runcmd("rm -rf ~%s/insight-api/node-modules && cd ~%s/insight-api && npm install" % (USERNAME, USERNAME))346        #Set up insight startup scripts (will be disabled later from autostarting on system startup if necessary)347        runcmd("rm -f /etc/init/insight.conf /etc/init/insight-testnet.conf")348        runcmd("cp -af %s/linux/init/insight.conf.template /etc/init/insight.conf" % dist_path)349        runcmd("sed -ri \"s/\!RUN_AS_USER\!/%s/g\" /etc/init/insight.conf" % DAEMON_USERNAME)350        runcmd("sed -ri \"s/\!USER_HOMEDIR\!/%s/g\" /etc/init/insight.conf" % user_homedir.replace('/', '\/'))351        runcmd("cp -af %s/linux/init/insight-testnet.conf.template /etc/init/insight-testnet.conf" % dist_path)352        runcmd("sed -ri \"s/\!RUN_AS_USER\!/%s/g\" /etc/init/insight-testnet.conf" % DAEMON_USERNAME)353        runcmd("sed -ri \"s/\!USER_HOMEDIR\!/%s/g\" /etc/init/insight-testnet.conf" % user_homedir.replace('/', '\/'))354        #install logrotate file355        runcmd("cp -af %s/linux/logrotate/insight /etc/logrotate.d/insight" % dist_path)356        runcmd("sed -ri \"s/\!RUN_AS_USER\!/%s/g\" /etc/logrotate.d/insight" % DAEMON_USERNAME)357        runcmd("sed -ri \"s/\!USER_HOMEDIR\!/%s/g\" /etc/logrotate.d/insight" % user_homedir.replace('/', '\/'))358        runcmd("mkdir -p ~%s/insight-api/db" % USERNAME)359        runcmd("chown -R %s:%s ~%s/insight-api" % (USERNAME, USERNAME, USERNAME))360        runcmd("chown -R %s:%s ~%s/insight-api/db" % (DAEMON_USERNAME, USERNAME, USERNAME))361        modify_cp_config(r'^blockchain\-service\-name=.*?$', 'blockchain-service-name=insight', config='both')362    def do_blockr_setup():363        modify_cp_config(r'^blockchain\-service\-name=.*?$', 'blockchain-service-name=blockr', config='both')364    def do_addrindex_setup():365        modify_cp_config(r'^insight\-enable=.*?$', 'insight-enable=0', config='both')366        modify_cp_config(r'^blockchain\-service\-name=.*?$', 'blockchain-service-name=addrindex', config='both')367        modify_cp_config(r'^blockchain\-service\-connect=.*?$', 'blockchain-service-connect=http://rpc:%s@localhost:5222' % bitcoin_rpc_password, config='clearblockd', net='mainnet')368        modify_cp_config(r'^blockchain\-service\-connect=.*?$', 'blockchain-service-connect=http://rpc:%s@localhost:25222' % bitcoin_rpc_password_testnet, config='clearblockd', net='testnet')369    #disable upstart scripts from autostarting on system boot if necessary370    if blockchain_service == 'i':371        do_insight_setup()372        if run_mode == 't': #disable mainnet daemons from autostarting373            runcmd(r"""bash -c "echo 'manual' >> /etc/init/insight.override" """)374        else:375            runcmd("rm -f /etc/init/insight.override")376        if run_mode == 'm': #disable testnet daemons from autostarting377            runcmd(r"""bash -c "echo 'manual' >> /etc/init/insight-testnet.override" """)378        else:379            runcmd("rm -f /etc/init/insight-testnet.override")380    if blockchain_service == 'a':381        runcmd("rm -f /etc/init/insight.override /etc/init/insight-testnet.override")382        do_addrindex_setup()383    else: #insight not being used as blockchain service384        runcmd("rm -f /etc/init/insight.override /etc/init/insight-testnet.override")385        #^ so insight doesn't start if it was in use before386        do_blockr_setup()387def do_nginx_setup(run_as_user, base_path, dist_path):388    #Build and install nginx (openresty) on Ubuntu389    #Most of these build commands from http://brian.akins.org/blog/2013/03/19/building-openresty-on-ubuntu/390    OPENRESTY_VER = "1.7.0.1"391    #install deps392    runcmd("apt-get -y install make ruby1.9.1 ruby1.9.1-dev git-core libpcre3-dev libxslt1-dev libgd2-xpm-dev libgeoip-dev unzip zip build-essential libssl-dev")393    runcmd("gem install fpm")394    #grab openresty and compile395    runcmd("rm -rf /tmp/openresty /tmp/ngx_openresty-* /tmp/nginx-openresty.tar.gz /tmp/nginx-openresty*.deb")396    runcmd('''wget -O /tmp/nginx-openresty.tar.gz http://openresty.org/download/ngx_openresty-%s.tar.gz''' % OPENRESTY_VER)397    runcmd("tar -C /tmp -zxvf /tmp/nginx-openresty.tar.gz")398    runcmd('''cd /tmp/ngx_openresty-%s && ./configure \399--with-luajit \400--sbin-path=/usr/sbin/nginx \401--conf-path=/etc/nginx/nginx.conf \402--error-log-path=/var/log/nginx/error.log \403--http-client-body-temp-path=/var/lib/nginx/body \404--http-fastcgi-temp-path=/var/lib/nginx/fastcgi \405--http-log-path=/var/log/nginx/access.log \406--http-proxy-temp-path=/var/lib/nginx/proxy \407--http-scgi-temp-path=/var/lib/nginx/scgi \408--http-uwsgi-temp-path=/var/lib/nginx/uwsgi \409--lock-path=/var/lock/nginx.lock \410--pid-path=/var/run/nginx.pid \411--with-http_geoip_module \412--with-http_gzip_static_module \413--with-http_realip_module \414--with-http_ssl_module \415--with-http_sub_module \416--with-http_xslt_module \417--with-ipv6 \418--with-sha1=/usr/include/openssl \419--with-md5=/usr/include/openssl \420--with-http_stub_status_module \421--with-http_secure_link_module \422--with-http_sub_module && make''' % OPENRESTY_VER)423    #set up the build environment424    runcmd('''cd /tmp/ngx_openresty-%s && make install DESTDIR=/tmp/openresty \425&& mkdir -p /tmp/openresty/var/lib/nginx \426&& install -m 0755 -D %s/linux/nginx/nginx.init /tmp/openresty/etc/init.d/nginx \427&& install -m 0755 -D %s/linux/nginx/nginx.conf /tmp/openresty/etc/nginx/nginx.conf \428&& install -m 0755 -D %s/linux/nginx/clearblock.conf /tmp/openresty/etc/nginx/sites-enabled/clearblock.conf \429&& install -m 0755 -D %s/linux/nginx/clearblock_api.inc /tmp/openresty/etc/nginx/sites-enabled/clearblock_api.inc \430&& install -m 0755 -D %s/linux/nginx/clearblock_api_cache.inc /tmp/openresty/etc/nginx/sites-enabled/clearblock_api_cache.inc \431&& install -m 0755 -D %s/linux/nginx/clearblock_socketio.inc /tmp/openresty/etc/nginx/sites-enabled/clearblock_socketio.inc \432&& install -m 0755 -D %s/linux/logrotate/nginx /tmp/openresty/etc/logrotate.d/nginx''' % (433    OPENRESTY_VER, dist_path, dist_path, dist_path, dist_path, dist_path, dist_path, dist_path))434    #package it up using fpm435    runcmd('''cd /tmp && fpm -s dir -t deb -n nginx-openresty -v %s --iteration 1 -C /tmp/openresty \436--description "openresty %s" \437--conflicts nginx \438--conflicts nginx-common \439-d libxslt1.1 \440-d libgeoip1 \441-d geoip-database \442-d libpcre3 \443--config-files /etc/nginx/nginx.conf \444--config-files /etc/nginx/sites-enabled/clearblock.conf \445--config-files /etc/nginx/fastcgi.conf.default \446--config-files /etc/nginx/win-utf \447--config-files /etc/nginx/fastcgi_params \448--config-files /etc/nginx/nginx.conf \449--config-files /etc/nginx/koi-win \450--config-files /etc/nginx/nginx.conf.default \451--config-files /etc/nginx/mime.types.default \452--config-files /etc/nginx/koi-utf \453--config-files /etc/nginx/uwsgi_params \454--config-files /etc/nginx/uwsgi_params.default \455--config-files /etc/nginx/fastcgi_params.default \456--config-files /etc/nginx/mime.types \457--config-files /etc/nginx/scgi_params.default \458--config-files /etc/nginx/scgi_params \459--config-files /etc/nginx/fastcgi.conf \460etc usr var''' % (OPENRESTY_VER, OPENRESTY_VER))461    #now install the .deb package that was created (along with its deps)462    runcmd("apt-get -y install libxslt1.1 libgeoip1 geoip-database libpcre3")463    runcmd("dpkg -i /tmp/nginx-openresty_%s-1_amd64.deb" % OPENRESTY_VER)464    #remove any .dpkg-old or .dpkg-dist files that might have been installed out of the nginx config dir465    runcmd("rm -f /etc/nginx/sites-enabled/*.dpkg-old /etc/nginx/sites-enabled/*.dpkg-dist")466    #clean up after ourselves467    runcmd("rm -rf /tmp/openresty /tmp/ngx_openresty-* /tmp/nginx-openresty.tar.gz /tmp/nginx-openresty*.deb")468    runcmd("update-rc.d nginx defaults")469def do_armory_utxsvr_setup(run_as_user, base_path, dist_path, run_mode, enable=True):470    user_homedir = os.path.expanduser("~" + USERNAME)471    runcmd("apt-get -y install xvfb python-qt4 python-twisted python-psutil xdg-utils")472    runcmd("rm -f /tmp/armory.deb")473    runcmd("wget -O /tmp/armory.deb https://s3.amazonaws.com/bitcoinarmory-releases/armory_0.92.1_ubuntu-64bit.deb")474    runcmd("mkdir -p /usr/share/desktop-directories/") #bug fix (see http://askubuntu.com/a/406015)475    runcmd("dpkg -i /tmp/armory.deb")476    runcmd("rm -f /tmp/armory.deb")477    runcmd("mkdir -p ~%s/.armory" % USERNAME)478    runcmd("chown -R %s:%s ~%s/.armory" % (DAEMON_USERNAME, USERNAME, USERNAME))479    runcmd("sudo ln -sf ~%s/.viacoin-testnet/testnet3 ~%s/.viacoin/" % (USERNAME, USERNAME))480    #^ ghetto hack, as armory has hardcoded dir settings in certain place481    #make a short script to launch armory_utxsvr482    f = open("/usr/local/bin/armory_utxsvr", 'w')483    f.write("#!/bin/sh\n%s/run.py armory_utxsvr \"$@\"" % base_path)484    f.close()485    runcmd("chmod +x /usr/local/bin/armory_utxsvr")486    #Set up upstart scripts (will be disabled later from autostarting on system startup if necessary)487    if enable:488        runcmd("rm -f /etc/init/armory_utxsvr.conf /etc/init/armory_utxsvr-testnet.conf")489        runcmd("cp -af %s/linux/init/armory_utxsvr.conf.template /etc/init/armory_utxsvr.conf" % dist_path)490        runcmd("sed -ri \"s/\!RUN_AS_USER\!/%s/g\" /etc/init/armory_utxsvr.conf" % DAEMON_USERNAME)491        runcmd("sed -ri \"s/\!USER_HOMEDIR\!/%s/g\" /etc/init/armory_utxsvr.conf" % user_homedir.replace('/', '\/'))492        runcmd("cp -af %s/linux/init/armory_utxsvr-testnet.conf.template /etc/init/armory_utxsvr-testnet.conf" % dist_path)493        runcmd("sed -ri \"s/\!RUN_AS_USER\!/%s/g\" /etc/init/armory_utxsvr-testnet.conf" % DAEMON_USERNAME)494        runcmd("sed -ri \"s/\!USER_HOMEDIR\!/%s/g\" /etc/init/armory_utxsvr-testnet.conf" % user_homedir.replace('/', '\/'))495    else: #disable496        runcmd("rm -f /etc/init/armory_utxsvr.conf /etc/init/armory_utxsvr-testnet.conf")497    #disable upstart scripts from autostarting on system boot if necessary498    if run_mode == 't': #disable mainnet daemons from autostarting499        runcmd(r"""bash -c "echo 'manual' >> /etc/init/armory_utxsvr.override" """)500    else:501        runcmd("rm -f /etc/init/armory_utxsvr.override")502    if run_mode == 'm': #disable testnet daemons from autostarting503        runcmd(r"""bash -c "echo 'manual' >> /etc/init/armory_utxsvr-testnet.override" """)504    else:505        runcmd("rm -f /etc/init/armory_utxsvr-testnet.override")506def do_counterwallet_setup(run_as_user, branch, updateOnly=False):507    #check out counterwallet from git508    git_repo_clone(branch, "clearwallet", REPO_COUNTERWALLET, run_as_user)509    if not updateOnly:510        runcmd("npm install -g grunt-cli bower")511    runcmd("cd ~%s/clearwallet/src && bower --allow-root --config.interactive=false install" % USERNAME)512    runcmd("cd ~%s/clearwallet && npm install" % USERNAME)513    runcmd("cd ~%s/clearwallet && grunt build" % USERNAME) #will generate the minified site514    runcmd("chown -R %s:%s ~%s/clearwallet" % (USERNAME, USERNAME, USERNAME)) #just in case515    runcmd("chmod -R u+rw,g+rw,o+r,o-w ~%s/clearwallet" % USERNAME) #just in case516def do_newrelic_setup(run_as_user, base_path, dist_path, run_mode):517    ##518    ## NOTE: NEW RELIC DAEMON CAUSES ISSUES WITH COUNTERBLOCKD (SOCKETS STICKING IN CLOSE_WAIT STATE)519    ##  -- DO NOT USE IT FOR COUNTERBLOCKD MONITORING520    ##521    NR_PREFS_LICENSE_KEY_PATH = "/etc/newrelic/LICENSE_KEY"522    NR_PREFS_HOSTNAME_PATH = "/etc/newrelic/HOSTNAME"523    runcmd("mkdir -p /etc/newrelic /var/log/newrelic /var/run/newrelic")524    runcmd("chown %s:%s /etc/newrelic /var/log/newrelic /var/run/newrelic" % (DAEMON_USERNAME, USERNAME))525    #try to find existing license key526    nr_license_key = None527    if os.path.exists(NR_PREFS_LICENSE_KEY_PATH):528        nr_license_key = open(NR_PREFS_LICENSE_KEY_PATH).read().strip()529    else:530        while True:531            nr_license_key = input("Enter New Relic license key (or blank to not setup New Relic): ") #gather license key532            nr_license_key = nr_license_key.strip()533            if not nr_license_key:534                return #skipping new relic535            nr_license_key_confirm = ask_question("You entererd '%s', is that right? (Y/n): " % nr_license_key, ('y', 'n'), 'y')536            if nr_license_key_confirm == 'y': break537        open(NR_PREFS_LICENSE_KEY_PATH, 'w').write(nr_license_key)538    assert nr_license_key539    logging.info("NewRelic license key: %s" % nr_license_key)540    #try to find existing app prefix541    nr_hostname = None542    if os.path.exists(NR_PREFS_HOSTNAME_PATH):543        nr_hostname = open(NR_PREFS_HOSTNAME_PATH).read().strip()544    else:545        while True:546            nr_hostname = input("Enter newrelic hostname/app prefix (e.g. 'cw01'): ") #gather app prefix547            nr_hostname = nr_hostname.strip()548            nr_hostname_confirm = ask_question("You entererd '%s', is that right? (Y/n): " % nr_hostname, ('y', 'n'), 'y')549            if nr_hostname_confirm == 'y': break550        open(NR_PREFS_HOSTNAME_PATH, 'w').write(nr_hostname)551    assert nr_hostname552    logging.info("NewRelic hostname/app prefix: %s" % nr_hostname)553    #install some deps...554    runcmd("sudo apt-get -y install libyaml-dev")555    #install/setup python agent for both counterpartyd and counterblockd556    #counterpartyd557    runcmd("%s/env/bin/pip install newrelic" % base_path)558    runcmd("cp -af %s/linux/newrelic/nr_clearinghoused.ini.template /etc/newrelic/nr_clearinghoused.ini" % dist_path)559    runcmd("sed -ri \"s/\!LICENSE_KEY\!/%s/g\" /etc/newrelic/nr_clearinghoused.ini" % nr_license_key)560    runcmd("sed -ri \"s/\!HOSTNAME\!/%s/g\" /etc/newrelic/nr_clearinghoused.ini" % nr_hostname)561    #counterblockd562    runcmd("%s/env.clearblockd/bin/pip install newrelic" % base_path)563    runcmd("cp -af %s/linux/newrelic/nr_clearblockd.ini.template /etc/newrelic/nr_clearblockd.ini" % dist_path)564    runcmd("sed -ri \"s/\!LICENSE_KEY\!/%s/g\" /etc/newrelic/nr_clearblockd.ini" % nr_license_key)565    runcmd("sed -ri \"s/\!HOSTNAME\!/%s/g\" /etc/newrelic/nr_clearblockd.ini" % nr_hostname)566    #install init scripts (overwrite the existing ones for now at least)567    runcmd("cp -af %s/linux/newrelic/init/nr-clearinghoused.conf /etc/init/clearinghoused.conf" % dist_path) #overwrite568    #runcmd("cp -af %s/linux/newrelic/init/nr-clearblockd.conf /etc/init/clearblockd.conf" % dist_path) #overwrite569    runcmd("cp -af %s/linux/newrelic/init/nr-clearinghoused-testnet.conf /etc/init/clearinghoused-testnet.conf" % dist_path) #overwrite570    #runcmd("cp -af %s/linux/newrelic/init/nr-clearblockd-testnet.conf /etc/init/clearblockd-testnet.conf" % dist_path) #overwrite571    #upstart enablement (overrides) should be fine as established in do_counterparty_setup...572    #install/setup server agent573    runcmd("add-apt-repository \"deb http://apt.newrelic.com/debian/ newrelic non-free\"")574    runcmd("wget -O- https://download.newrelic.com/548C16BF.gpg | apt-key add -")575    runcmd("apt-get update")576    runcmd("apt-get -y install newrelic-sysmond")577    runcmd("cp -af %s/linux/newrelic/nrsysmond.cfg.template /etc/newrelic/nrsysmond.cfg" % dist_path)578    runcmd("sed -ri \"s/\!LICENSE_KEY\!/%s/g\" /etc/newrelic/nrsysmond.cfg" % nr_license_key)579    runcmd("sed -ri \"s/\!HOSTNAME\!/%s/g\" /etc/newrelic/nrsysmond.cfg" % nr_hostname)580    runcmd("/etc/init.d/newrelic-sysmond restart")581    #install/setup meetme agent (mongo, redis)582    runcmd("pip install newrelic_plugin_agent pymongo")583    runcmd("cp -af %s/linux/newrelic/newrelic-plugin-agent.cfg.template /etc/newrelic/newrelic-plugin-agent.cfg" % dist_path)584    runcmd("sed -ri \"s/\!LICENSE_KEY\!/%s/g\" /etc/newrelic/newrelic-plugin-agent.cfg" % nr_license_key)585    runcmd("sed -ri \"s/\!HOSTNAME\!/%s/g\" /etc/newrelic/newrelic-plugin-agent.cfg" % nr_hostname)586    runcmd("ln -sf %s/linux/newrelic/init/newrelic-plugin-agent /etc/init.d/newrelic-plugin-agent" % dist_path)587    runcmd("update-rc.d newrelic-plugin-agent defaults")588    runcmd("/etc/init.d/newrelic-plugin-agent restart")589    #install/setup nginx agent590    runcmd("sudo apt-get -y install ruby ruby-bundler")591    runcmd("rm -rf /tmp/newrelic_nginx_agent.tar.gz /opt/newrelic_nginx_agent")592    #runcmd("wget -O /tmp/newrelic_nginx_agent.tar.gz http://nginx.com/download/newrelic/newrelic_nginx_agent.tar.gz")593    #runcmd("tar -C /opt -zxvf /tmp/newrelic_nginx_agent.tar.gz")594    runcmd("git clone https://github.com/crowdlab-uk/newrelic-nginx-agent.git /opt/newrelic_nginx_agent")595    runcmd("cd /opt/newrelic_nginx_agent && bundle install")596    runcmd("cp -af %s/linux/newrelic/newrelic_nginx_plugin.yml.template /opt/newrelic_nginx_agent/config/newrelic_plugin.yml" % dist_path)597    runcmd("sed -ri \"s/\!LICENSE_KEY\!/%s/g\" /opt/newrelic_nginx_agent/config/newrelic_plugin.yml" % nr_license_key)598    runcmd("sed -ri \"s/\!HOSTNAME\!/%s/g\" /opt/newrelic_nginx_agent/config/newrelic_plugin.yml" % nr_hostname)599    runcmd("ln -sf /opt/newrelic_nginx_agent/newrelic_nginx_agent.daemon /etc/init.d/newrelic_nginx_agent")600    runcmd("update-rc.d newrelic_nginx_agent defaults")601    runcmd("/etc/init.d/newrelic_nginx_agent restart")602def command_services(command, prompt=False):603    assert command in ("stop", "restart")604    if prompt:605        confirmation = ask_question("%s services? (y/N)" % command.capitalize(), ('y', 'n',), 'n')606        if confirmation == 'n':607            return False608    #restart/shutdown services if they may be running on the box609    if os.path.exists("/etc/init.d/iwatch"):610        runcmd("service iwatch %s" % command, abort_on_failure=False)611    if os.path.exists("/etc/init/clearinghoused.conf"):612        logging.warn("STOPPING SERVICES" if command == 'stop' else "RESTARTING SERVICES")613        runcmd("service viacoind %s" % command, abort_on_failure=False)614        runcmd("service viacoind-testnet %s" % command, abort_on_failure=False)615        if os.path.exists("/etc/init/insight.conf"):616            if command == "restart":617                logging.info("Waiting 45 seconds before starting insight, to allow viacoind to fully initialize...")618                time.sleep(45)619            runcmd("service insight %s" % command, abort_on_failure=False)620            runcmd("service insight-testnet %s" % command, abort_on_failure=False)621        runcmd("service clearinghoused %s" % command, abort_on_failure=False)622        runcmd("service clearinghoused-testnet %s" % command, abort_on_failure=False)623        runcmd("service clearblockd %s" % command, abort_on_failure=False)624        runcmd("service clearblockd-testnet %s" % command, abort_on_failure=False)625        if os.path.exists("/etc/init/armory_utxsvr.conf"):626            runcmd("service armory_utxsvr %s" % command, abort_on_failure=False)627            runcmd("service armory_utxsvr-testnet %s" % command, abort_on_failure=False)628        return True629QUESTION_FLAGS = collections.OrderedDict({630    "op": ('u', 'r'),631    "role": ('clearwallet', 'vendingmachine', 'blockexplorer', 'clearinghoused_only', 'btcpayescrow'),632    "branch": ('master', 'develop'),633    "run_mode": ('t', 'm', 'b'),634    "blockchain_service": ('b', 'i', 'a'),635    "security_hardening": ('y', 'n'),636    "clearinghoused_public": ('y', 'n'),637    "clearwallet_support_email": None638})639def gather_build_questions(answered_questions):640    if 'role' not in answered_questions:641        role = ask_question("Enter the number for the role you want to build:\n"642                + "\t1: Clearwallet server\n\t2: Vending machine\n\t3: Blockexplorer server\n"643                + "\t4: clearinghoused-only\n\t5: VIAPay Escrow Server\n"644                + "Your choice",645            ('1', '2', '3', '4', '5'), '1')646        if role == '1':647            role = 'clearwallet'648            role_desc = "Clearwallet server"649        elif role == '2':650            role = 'vendingmachine'651            role_desc = "Vending machine/gateway"652        elif role == '3':653            role = 'blockexplorer'654            role_desc = "Block explorer server"655        elif role == '4':656            role = 'clearinghoused_only'657            role_desc = "Counterpartyd server"658        elif role == '5':659            role = 'btcpayescrow'660            role_desc = "BTCPay escrow server"661        print("\tBuilding a %s" % role_desc)662        answered_questions['role'] = role663    assert answered_questions['role'] in QUESTION_FLAGS['role']664    if answered_questions['role'] in ('vendingmachine', 'blockexplorer'):665        raise NotImplementedError("This role not implemented yet...")666    if 'branch' not in answered_questions:667        branch = ask_question("Build from branch (M)aster or (d)evelop? (M/d)", ('m', 'd'), 'm')668        if branch == 'm': branch = 'master'669        elif branch == 'd': branch = 'develop'670        print("\tWorking with branch: %s" % branch)671        answered_questions['branch'] = branch672    assert answered_questions['branch'] in QUESTION_FLAGS['branch']673    if 'run_mode' not in answered_questions:674        answered_questions['run_mode'] = ask_question("Run as (t)estnet node, (m)ainnet node, or (B)oth? (t/m/B)", ('t', 'm', 'b'), 'b')675        print("\tSetting up to run on %s" % ('testnet' if answered_questions['run_mode'].lower() == 't'676            else ('mainnet' if answered_questions['run_mode'].lower() == 'm' else 'testnet and mainnet')))677    assert answered_questions['run_mode'] in QUESTION_FLAGS['run_mode']678    if 'blockchain_service' not in answered_questions:679        answered_questions['blockchain_service'] = ask_question("Blockchain services, use (B)lockr.io (remote), (i)nsight (local), (a)ddrindex? (b/i/A)", ('b', 'i', 'a'), 'a')680        if answered_questions['blockchain_service'] == 'b':681            answer = 'blockr.io'682        elif answered_questions['blockchain_service'] == 'a':683            answer = 'addrindex'684        else:685            answer = 'insight'686        print("\tUsing %s" % answer)687    assert answered_questions['blockchain_service'] in QUESTION_FLAGS['blockchain_service']688    if role == 'clearinghoused_only' and 'clearinghoused_public' not in answered_questions:689        answered_questions['clearinghoused_public'] = ask_question(690            "Enable public Clearinghoused setup (listen on all hosts w/ rpc/1234 user) (Y/n)", ('y', 'n'), 'y')691    else:692        answered_questions['clearinghoused_public'] = answered_questions.get('clearinghoused_public', 'n') #does not apply693    assert answered_questions['clearinghoused_public'] in QUESTION_FLAGS['clearinghoused_public']694    counterwallet_support_email = None695    if role == 'clearwallet' and 'clearwallet_support_email' not in answered_questions:696        while True:697            counterwallet_support_email = input("Email address where support cases should go (blank to disable): ")698            counterwallet_support_email = counterwallet_support_email.strip()699            if counterwallet_support_email:700                counterwallet_support_email_confirm = ask_question(701                    "You entered '%s', is that right? (Y/n): " % counterwallet_support_email, ('y', 'n'), 'y')702                if counterwallet_support_email_confirm == 'y': break703            else: break704        answered_questions['clearwallet_support_email'] = counterwallet_support_email705    else:706        answered_questions['clearwallet_support_email'] = answered_questions.get('clearwallet_support_email', '')707    if 'security_hardening' not in answered_questions:708        answered_questions['security_hardening'] = ask_question("Set up security hardening? (Y/n)", ('y', 'n'), 'y')709    assert answered_questions['security_hardening'] in QUESTION_FLAGS['security_hardening']710    return answered_questions711def usage():712    print("SYNTAX: %s [-h] %s" % (sys.argv[0], ' '.join([('[-%s=%s]' % (q, '|'.join(v) if v else '')) for q, v in QUESTION_FLAGS.items()])))713def main():714    logging.basicConfig(level=logging.DEBUG, format='%(asctime)s|%(levelname)s: %(message)s')715    do_prerun_checks()716    run_as_user = os.environ["SUDO_USER"]717    assert run_as_user718    answered_questions = {}719    #parse any command line objects720    branch = "master"721    try:722        opts, args = getopt.getopt(sys.argv[1:], "h", ["help",] + ['%s=' % q for q in QUESTION_FLAGS.keys()])723    except getopt.GetoptError as err:724        usage()725        sys.exit(2)726    for o, a in opts:727        if o in ("-h", "--help"):728            usage()729            sys.exit()730        elif o in ['--%s' % q for q in QUESTION_FLAGS.keys()]: #process flags for non-interactivity731            answered_questions[o.lstrip('-')] = a732        else:733            assert False, "Unhandled or unimplemented switch or option"734    base_path = os.path.expanduser("~%s/clearinghoused_build" % USERNAME)735    dist_path = os.path.join(base_path, "dist")736    #Detect if we should ask the user if they just want to update the source and not do a rebuild737    do_rebuild = 'r'738    try:739        pwd.getpwnam(USERNAME) #hacky check ...as this user is created by the script740    except:741        answered_questions['op'] = 'r' #do a build742    else: #setup has already been run at least once743        if not (answered_questions.get('op', None) in QUESTION_FLAGS['op']):744            answered_questions['op'] = ask_question(745                "It appears this setup has been run already. (r)ebuild node, or just (U)pdate from git? (r/U)",746                ('r', 'u'), 'u')747            assert answered_questions['op'] in QUESTION_FLAGS['op']748    if answered_questions['op'] == 'u': #just refresh counterpartyd, counterblockd, and counterwallet, etc. from github749        if os.path.exists("/etc/init.d/iwatch"):750            runcmd("service iwatch stop", abort_on_failure=False)751        #refresh counterpartyd_build, counterpartyd and counterblockd (if available)752        runcmd("%s/setup.py %s --for-user=xch update" % (base_path,753            '--with-clearblockd' if os.path.exists(os.path.join(dist_path, "clearblockd")) else ''))754        #refresh counterwallet (if available)755        if os.path.exists(os.path.exists(os.path.expanduser("~%s/clearwallet" % USERNAME))):756            do_counterwallet_setup(run_as_user, "AUTO", updateOnly=True)757        #offer to restart services758        restarted = command_services("restart", prompt=True)759        if not restarted and os.path.exists("/etc/init.d/iwatch"):760            runcmd("service iwatch start", abort_on_failure=False)761        sys.exit(0) #all done762    #If here, a) federated node has not been set up yet or b) the user wants a rebuild763    answered_questions = gather_build_questions(answered_questions)764    command_services("stop")765    do_base_setup(run_as_user, answered_questions['branch'], base_path, dist_path)766    bitcoind_rpc_password, bitcoind_rpc_password_testnet \767        = do_bitcoind_setup(run_as_user, answered_questions['branch'], base_path, dist_path, answered_questions['run_mode'])768    do_counterparty_setup(answered_questions['role'], run_as_user, answered_questions['branch'], base_path, dist_path, answered_questions['run_mode'],769        bitcoind_rpc_password, bitcoind_rpc_password_testnet,770        answered_questions['clearinghoused_public'], answered_questions['clearwallet_support_email'])771    do_blockchain_service_setup(run_as_user, base_path, dist_path,772        answered_questions['run_mode'], answered_questions['blockchain_service'], bitcoind_rpc_password, bitcoind_rpc_password_testnet)773    if answered_questions['role'] != "clearinghoused_only":774        do_nginx_setup(run_as_user, base_path, dist_path)775    do_armory_utxsvr_setup(run_as_user, base_path, dist_path,776        answered_questions['run_mode'], enable=answered_questions['role']=='clearwallet')777    if answered_questions['role'] == 'clearwallet':778        do_counterwallet_setup(run_as_user, answered_questions['branch'])779    do_newrelic_setup(run_as_user, base_path, dist_path, answered_questions['run_mode']) #optional780    if answered_questions['security_hardening']:781        do_security_setup(run_as_user, answered_questions['branch'], base_path, dist_path)782    logging.info("Clearblock Federated Node Build Complete (whew).")783    command_services("restart", prompt=True)784if __name__ == "__main__":...

Full Screen

Full Screen

pkgdata.py

Source:pkgdata.py Github

copy

Full Screen

...16        bitbake('glibc busybox zlib bash')17    @testcase(1203)18    def test_lookup_pkg(self):19        # Forward tests20        result = runCmd('oe-pkgdata-util lookup-pkg "glibc busybox"')21        self.assertEqual(result.output, 'libc6\nbusybox')22        result = runCmd('oe-pkgdata-util lookup-pkg zlib-dev')23        self.assertEqual(result.output, 'libz-dev')24        result = runCmd('oe-pkgdata-util lookup-pkg nonexistentpkg', ignore_status=True)25        self.assertEqual(result.status, 1)26        self.assertEqual(result.output, 'ERROR: The following packages could not be found: nonexistentpkg')27        # Reverse tests28        result = runCmd('oe-pkgdata-util lookup-pkg -r "libc6 busybox"')29        self.assertEqual(result.output, 'glibc\nbusybox')30        result = runCmd('oe-pkgdata-util lookup-pkg -r libz-dev')31        self.assertEqual(result.output, 'zlib-dev')32        result = runCmd('oe-pkgdata-util lookup-pkg -r nonexistentpkg', ignore_status=True)33        self.assertEqual(result.status, 1)34        self.assertEqual(result.output, 'ERROR: The following packages could not be found: nonexistentpkg')35    @testcase(1205)36    def test_read_value(self):37        result = runCmd('oe-pkgdata-util read-value PN libz1')38        self.assertEqual(result.output, 'zlib')39        result = runCmd('oe-pkgdata-util read-value PKGSIZE bash')40        pkgsize = int(result.output.strip())41        self.assertGreater(pkgsize, 1)42    @testcase(1198)43    def test_find_path(self):44        result = runCmd('oe-pkgdata-util find-path /lib/libc.so.6')45        self.assertEqual(result.output, 'glibc: /lib/libc.so.6')46        result = runCmd('oe-pkgdata-util find-path /bin/bash')47        self.assertEqual(result.output, 'bash: /bin/bash')48        result = runCmd('oe-pkgdata-util find-path /not/exist', ignore_status=True)49        self.assertEqual(result.status, 1)50        self.assertEqual(result.output, 'ERROR: Unable to find any package producing path /not/exist')51    @testcase(1204)52    def test_lookup_recipe(self):53        result = runCmd('oe-pkgdata-util lookup-recipe "libc6-staticdev busybox"')54        self.assertEqual(result.output, 'glibc\nbusybox')55        result = runCmd('oe-pkgdata-util lookup-recipe libz-dbg')56        self.assertEqual(result.output, 'zlib')57        result = runCmd('oe-pkgdata-util lookup-recipe nonexistentpkg', ignore_status=True)58        self.assertEqual(result.status, 1)59        self.assertEqual(result.output, 'ERROR: The following packages could not be found: nonexistentpkg')60    @testcase(1202)61    def test_list_pkgs(self):62        # No arguments63        result = runCmd('oe-pkgdata-util list-pkgs')64        pkglist = result.output.split()65        self.assertIn('glibc-utils', pkglist)66        self.assertIn('zlib-dev', pkglist)67        # No pkgspec, runtime68        result = runCmd('oe-pkgdata-util list-pkgs -r')69        pkglist = result.output.split()70        self.assertIn('libc6-utils', pkglist)71        self.assertIn('libz-dev', pkglist)72        # With recipe specified73        result = runCmd('oe-pkgdata-util list-pkgs -p zlib')74        pkglist = sorted(result.output.split())75        try:76            pkglist.remove('zlib-ptest') # in case ptest is disabled77        except ValueError:78            pass79        self.assertEqual(pkglist, ['zlib', 'zlib-dbg', 'zlib-dev', 'zlib-doc', 'zlib-staticdev'])80        # With recipe specified, runtime81        result = runCmd('oe-pkgdata-util list-pkgs -p zlib -r')82        pkglist = sorted(result.output.split())83        try:84            pkglist.remove('libz-ptest') # in case ptest is disabled85        except ValueError:86            pass87        self.assertEqual(pkglist, ['libz-dbg', 'libz-dev', 'libz-doc', 'libz-staticdev', 'libz1'])88        # With recipe specified and unpackaged89        result = runCmd('oe-pkgdata-util list-pkgs -p zlib -u')90        pkglist = sorted(result.output.split())91        self.assertIn('zlib-locale', pkglist)92        # With recipe specified and unpackaged, runtime93        result = runCmd('oe-pkgdata-util list-pkgs -p zlib -u -r')94        pkglist = sorted(result.output.split())95        self.assertIn('libz-locale', pkglist)96        # With recipe specified and pkgspec97        result = runCmd('oe-pkgdata-util list-pkgs -p zlib "*-d*"')98        pkglist = sorted(result.output.split())99        self.assertEqual(pkglist, ['zlib-dbg', 'zlib-dev', 'zlib-doc'])100        # With recipe specified and pkgspec, runtime101        result = runCmd('oe-pkgdata-util list-pkgs -p zlib -r "*-d*"')102        pkglist = sorted(result.output.split())103        self.assertEqual(pkglist, ['libz-dbg', 'libz-dev', 'libz-doc'])104    @testcase(1201)105    def test_list_pkg_files(self):106        def splitoutput(output):107            files = {}108            curpkg = None109            for line in output.splitlines():110                if line.startswith('\t'):111                    self.assertTrue(curpkg, 'Unexpected non-package line:\n%s' % line)112                    files[curpkg].append(line.strip())113                else:114                    self.assertTrue(line.rstrip().endswith(':'), 'Invalid package line in output:\n%s' % line)115                    curpkg = line.split(':')[0]116                    files[curpkg] = []117            return files118        base_libdir = get_bb_var('base_libdir')119        libdir = get_bb_var('libdir')120        includedir = get_bb_var('includedir')121        mandir = get_bb_var('mandir')122        # Test recipe-space package name123        result = runCmd('oe-pkgdata-util list-pkg-files zlib-dev zlib-doc')124        files = splitoutput(result.output)125        self.assertIn('zlib-dev', files.keys())126        self.assertIn('zlib-doc', files.keys())127        self.assertIn(os.path.join(includedir, 'zlib.h'), files['zlib-dev'])128        self.assertIn(os.path.join(mandir, 'man3/zlib.3'), files['zlib-doc'])129        # Test runtime package name130        result = runCmd('oe-pkgdata-util list-pkg-files -r libz1 libz-dev')131        files = splitoutput(result.output)132        self.assertIn('libz1', files.keys())133        self.assertIn('libz-dev', files.keys())134        self.assertGreater(len(files['libz1']), 1)135        libspec = os.path.join(base_libdir, 'libz.so.1.*')136        found = False137        for fileitem in files['libz1']:138            if fnmatch.fnmatchcase(fileitem, libspec):139                found = True140                break141        self.assertTrue(found, 'Could not find zlib library file %s in libz1 package file list: %s' % (libspec, files['libz1']))142        self.assertIn(os.path.join(includedir, 'zlib.h'), files['libz-dev'])143        # Test recipe144        result = runCmd('oe-pkgdata-util list-pkg-files -p zlib')145        files = splitoutput(result.output)146        self.assertIn('zlib-dbg', files.keys())147        self.assertIn('zlib-doc', files.keys())148        self.assertIn('zlib-dev', files.keys())149        self.assertIn('zlib-staticdev', files.keys())150        self.assertIn('zlib', files.keys())151        self.assertNotIn('zlib-locale', files.keys())152        # (ignore ptest, might not be there depending on config)153        self.assertIn(os.path.join(includedir, 'zlib.h'), files['zlib-dev'])154        self.assertIn(os.path.join(mandir, 'man3/zlib.3'), files['zlib-doc'])155        self.assertIn(os.path.join(libdir, 'libz.a'), files['zlib-staticdev'])156        # Test recipe, runtime157        result = runCmd('oe-pkgdata-util list-pkg-files -p zlib -r')158        files = splitoutput(result.output)159        self.assertIn('libz-dbg', files.keys())160        self.assertIn('libz-doc', files.keys())161        self.assertIn('libz-dev', files.keys())162        self.assertIn('libz-staticdev', files.keys())163        self.assertIn('libz1', files.keys())164        self.assertNotIn('libz-locale', files.keys())165        self.assertIn(os.path.join(includedir, 'zlib.h'), files['libz-dev'])166        self.assertIn(os.path.join(mandir, 'man3/zlib.3'), files['libz-doc'])167        self.assertIn(os.path.join(libdir, 'libz.a'), files['libz-staticdev'])168        # Test recipe, unpackaged169        result = runCmd('oe-pkgdata-util list-pkg-files -p zlib -u')170        files = splitoutput(result.output)171        self.assertIn('zlib-dbg', files.keys())172        self.assertIn('zlib-doc', files.keys())173        self.assertIn('zlib-dev', files.keys())174        self.assertIn('zlib-staticdev', files.keys())175        self.assertIn('zlib', files.keys())176        self.assertIn('zlib-locale', files.keys()) # this is the key one177        self.assertIn(os.path.join(includedir, 'zlib.h'), files['zlib-dev'])178        self.assertIn(os.path.join(mandir, 'man3/zlib.3'), files['zlib-doc'])179        self.assertIn(os.path.join(libdir, 'libz.a'), files['zlib-staticdev'])180        # Test recipe, runtime, unpackaged181        result = runCmd('oe-pkgdata-util list-pkg-files -p zlib -r -u')182        files = splitoutput(result.output)183        self.assertIn('libz-dbg', files.keys())184        self.assertIn('libz-doc', files.keys())185        self.assertIn('libz-dev', files.keys())186        self.assertIn('libz-staticdev', files.keys())187        self.assertIn('libz1', files.keys())188        self.assertIn('libz-locale', files.keys()) # this is the key one189        self.assertIn(os.path.join(includedir, 'zlib.h'), files['libz-dev'])190        self.assertIn(os.path.join(mandir, 'man3/zlib.3'), files['libz-doc'])191        self.assertIn(os.path.join(libdir, 'libz.a'), files['libz-staticdev'])192    @testcase(1200)193    def test_glob(self):194        tempdir = tempfile.mkdtemp(prefix='pkgdataqa')195        self.track_for_cleanup(tempdir)196        pkglistfile = os.path.join(tempdir, 'pkglist')197        with open(pkglistfile, 'w') as f:198            f.write('libc6\n')199            f.write('libz1\n')200            f.write('busybox\n')201        result = runCmd('oe-pkgdata-util glob %s "*-dev"' % pkglistfile)202        desiredresult = ['libc6-dev', 'libz-dev', 'busybox-dev']203        self.assertEqual(sorted(result.output.split()), sorted(desiredresult))204        # The following should not error (because when we use this during rootfs construction, sometimes the complementary package won't exist)205        result = runCmd('oe-pkgdata-util glob %s "*-nonexistent"' % pkglistfile)206        self.assertEqual(result.output, '')207        # Test exclude option208        result = runCmd('oe-pkgdata-util glob %s "*-dev *-dbg" -x "^libz"' % pkglistfile)209        resultlist = result.output.split()210        self.assertNotIn('libz-dev', resultlist)211        self.assertNotIn('libz-dbg', resultlist)212    @testcase(1206)213    def test_specify_pkgdatadir(self):214        result = runCmd('oe-pkgdata-util -p %s lookup-pkg glibc' % get_bb_var('PKGDATA_DIR'))...

Full Screen

Full Screen

smbcontrol.py

Source:smbcontrol.py Github

copy

Full Screen

1#!/usr/bin/python2#3# Test for smbcontrol command line argument handling.4#5import comfychair6class NoArgs(comfychair.TestCase):7    """Test no arguments produces usage message."""8    def runtest(self):9        out = self.runcmd("smbcontrol", expectedResult = 1)10        self.assert_re_match("Usage: smbcontrol", out[1])11        12class OneArg(comfychair.TestCase):13    """Test single argument produces usage message."""14    def runtest(self):15        out = self.runcmd("smbcontrol foo", expectedResult = 1)16        self.assert_re_match("Usage: smbcontrol", out[1])17        18class SmbdDest(comfychair.TestCase):19    """Test the broadcast destination 'smbd'."""20    def runtest(self):21        out = self.runcmd("smbcontrol smbd noop")22class NmbdDest(comfychair.TestCase):23    """Test the destination 'nmbd'."""24    def runtest(self):25        # We need a way to start/stop/whatever nmbd26        raise comfychair.NotRunError, "not implemented"27class PidDest(comfychair.TestCase):28    """Test a pid number destination'."""29    def runtest(self):30        out = self.runcmd("smbcontrol 1234 noop")31class SelfDest(comfychair.TestCase):32    """Test the destination 'self'."""33    def runtest(self):34        out = self.runcmd("smbcontrol self noop")35class WinbinddDest(comfychair.TestCase):36    """Test the destination 'winbindd'."""37    def runtest(self):38        # We need a way to start/stop/whatever winbindd39        raise comfychair.NotRunError, "not implemented"40class BadDest(comfychair.TestCase):41    """Test a bad destination."""42    def runtest(self):43        out = self.runcmd("smbcontrol foo noop", expectedResult = 1)44class BadCmd(comfychair.TestCase):45    """Test a bad command."""46    def runtest(self):47        out = self.runcmd("smbcontrol self spottyfoot", expectedResult = 1)48        self.assert_re_match("smbcontrol: unknown command", out[1]);49class NoArgCmdTest(comfychair.TestCase):50    """A test class that tests a command with no argument."""51    def runtest(self):52        self.require_root()53        out = self.runcmd("smbcontrol self %s" % self.cmd)54        out = self.runcmd("smbcontrol self %s spottyfoot" % self.cmd,55                          expectedResult = 1)56class ForceElection(NoArgCmdTest):57    """Test a force-election message."""58    def setup(self):59        self.cmd = "force-election"60class SamSync(NoArgCmdTest):61    """Test a samsync message."""62    def setup(self):63        self.cmd = "samsync"64class SamRepl(NoArgCmdTest):65    """Test a samrepl message."""66    def setup(self):67        self.cmd = "samrepl"68class DmallocChanged(NoArgCmdTest):69    """Test a dmalloc-changed message."""70    def setup(self):71        self.cmd = "dmalloc-log-changed"72class DmallocMark(NoArgCmdTest):73    """Test a dmalloc-mark message."""74    def setup(self):75        self.cmd = "dmalloc-mark"76class Shutdown(NoArgCmdTest):77    """Test a shutdown message."""78    def setup(self):79        self.cmd = "shutdown"80class Ping(NoArgCmdTest):81    """Test a ping message."""82    def setup(self):83        self.cmd = "ping"84class Debuglevel(NoArgCmdTest):85    """Test a debuglevel message."""86    def setup(self):87        self.cmd = "debuglevel"88class OneArgCmdTest(comfychair.TestCase):89    """A test class that tests a command with one argument."""90    def runtest(self):91        self.require_root()92        out = self.runcmd("smbcontrol self %s spottyfoot" % self.cmd)93        out = self.runcmd("smbcontrol self %s" % self.cmd, expectedResult = 1)94class DrvUpgrade(OneArgCmdTest):95    """Test driver upgrade message."""96    def setup(self):97        self.cmd = "drvupgrade"98        99class CloseShare(OneArgCmdTest):100    """Test close share message."""101    def setup(self):102        self.cmd = "close-share"103        104class Debug(OneArgCmdTest):105    """Test a debug message."""106    def setup(self):107        self.cmd = "debug"108class PrintNotify(comfychair.TestCase):109    """Test print notification commands."""110    def runtest(self):111        # No subcommand112        out = self.runcmd("smbcontrol self printnotify", expectedResult = 1)113        self.assert_re_match("Must specify subcommand", out[1]);114        # Invalid subcommand name115        out = self.runcmd("smbcontrol self printnotify spottyfoot",116                          expectedResult = 1)117        self.assert_re_match("Invalid subcommand", out[1]);118        # Queue commands119        for cmd in ["queuepause", "queueresume"]:120            out = self.runcmd("smbcontrol self printnotify %s" % cmd,121                              expectedResult = 1)122            self.assert_re_match("Usage:", out[1])123        124            out = self.runcmd("smbcontrol self printnotify %s spottyfoot"125                              % cmd)126        # Job commands127        for cmd in ["jobpause", "jobresume", "jobdelete"]:128            out = self.runcmd("smbcontrol self printnotify %s" % cmd,129                              expectedResult = 1)130            self.assert_re_match("Usage:", out[1])131            out = self.runcmd("smbcontrol self printnotify %s spottyfoot"132                              % cmd, expectedResult = 1)133            self.assert_re_match("Usage:", out[1])134            out = self.runcmd("smbcontrol self printnotify %s spottyfoot 123"135                              % cmd)136        # Printer properties137        out = self.runcmd("smbcontrol self printnotify printer",138                          expectedResult = 1)139        self.assert_re_match("Usage", out[1])140        out = self.runcmd("smbcontrol self printnotify printer spottyfoot",141                          expectedResult = 1)142        self.assert_re_match("Usage", out[1])143        for cmd in ["comment", "port", "driver"]:144            out = self.runcmd("smbcontrol self printnotify printer spottyfoot "145                              "%s" % cmd, expectedResult = 1)146            self.assert_re_match("Usage", out[1])147            out = self.runcmd("smbcontrol self printnotify printer spottyfoot "148                              "%s value" % cmd)149class Profile(comfychair.TestCase):150    """Test setting the profiling level."""151    def runtest(self):152        self.require_root()153        out = self.runcmd("smbcontrol self profile", expectedResult = 1)154        self.assert_re_match("Usage", out[1])155        156        out = self.runcmd("smbcontrol self profile spottyfoot",157                          expectedResult = 1)158        self.assert_re_match("Unknown", out[1])159        160        for cmd in ["off", "count", "on", "flush"]:161            out = self.runcmd("smbcontrol self profile %s" % cmd)162class ProfileLevel(comfychair.TestCase):163    """Test requesting the current profiling level."""164    def runtest(self):165        self.require_root()166        out = self.runcmd("smbcontrol self profilelevel spottyfoot",167                          expectedResult = 1)168        self.assert_re_match("Usage", out[1])169        170        out = self.runcmd("smbcontrol self profilelevel")171        172class TimeoutArg(comfychair.TestCase):173    """Test the --timeout argument."""174    def runtest(self):175        out = self.runcmd("smbcontrol --timeout 5 self noop")176        out = self.runcmd("smbcontrol --timeout spottyfoot self noop",177                          expectedResult = 1)178class ConfigFileArg(comfychair.TestCase):179    """Test the --configfile argument."""180    def runtest(self):181        out = self.runcmd("smbcontrol --configfile /dev/null self noop")182class BogusArg(comfychair.TestCase):183    """Test a bogus command line argument."""184    def runtest(self):185        out = self.runcmd("smbcontrol --bogus self noop", expectedResult = 1)186tests = [NoArgs, OneArg, SmbdDest, NmbdDest, WinbinddDest, PidDest,187         SelfDest, BadDest, BadCmd, Debug, ForceElection, SamSync,188         SamRepl, DmallocMark, DmallocChanged, Shutdown, DrvUpgrade,189         CloseShare, Ping, Debuglevel, PrintNotify, Profile, ProfileLevel,190         TimeoutArg, ConfigFileArg, BogusArg]191# Handle execution of this file as a main program192if __name__ == '__main__':...

Full Screen

Full Screen

bblayers.py

Source:bblayers.py Github

copy

Full Screen

...6from oeqa.core.decorator.oeid import OETestID7class BitbakeLayers(OESelftestTestCase):8    @OETestID(756)9    def test_bitbakelayers_showcrossdepends(self):10        result = runCmd('bitbake-layers show-cross-depends')11        self.assertTrue('aspell' in result.output, msg = "No dependencies were shown. bitbake-layers show-cross-depends output: %s" % result.output)12    @OETestID(83)13    def test_bitbakelayers_showlayers(self):14        result = runCmd('bitbake-layers show-layers')15        self.assertTrue('meta-selftest' in result.output, msg = "No layers were shown. bitbake-layers show-layers output: %s" % result.output)16    @OETestID(93)17    def test_bitbakelayers_showappends(self):18        recipe = "xcursor-transparent-theme"19        bb_file = self.get_recipe_basename(recipe)20        result = runCmd('bitbake-layers show-appends')21        self.assertTrue(bb_file in result.output, msg="%s file was not recognised. bitbake-layers show-appends output: %s" % (bb_file, result.output))22    @OETestID(90)23    def test_bitbakelayers_showoverlayed(self):24        result = runCmd('bitbake-layers show-overlayed')25        self.assertTrue('aspell' in result.output, msg="aspell overlayed recipe was not recognised bitbake-layers show-overlayed %s" % result.output)26    @OETestID(95)27    def test_bitbakelayers_flatten(self):28        recipe = "xcursor-transparent-theme"29        recipe_path = "recipes-graphics/xcursor-transparent-theme"30        recipe_file = self.get_recipe_basename(recipe)31        testoutdir = os.path.join(self.builddir, 'test_bitbakelayers_flatten')32        self.assertFalse(os.path.isdir(testoutdir), msg = "test_bitbakelayers_flatten should not exist at this point in time")33        self.track_for_cleanup(testoutdir)34        result = runCmd('bitbake-layers flatten %s' % testoutdir)35        bb_file = os.path.join(testoutdir, recipe_path, recipe_file)36        self.assertTrue(os.path.isfile(bb_file), msg = "Cannot find xcursor-transparent-theme_0.1.1.bb in the test_bitbakelayers_flatten local dir.")37        contents = ftools.read_file(bb_file)38        find_in_contents = re.search("##### bbappended from meta-selftest #####\n(.*\n)*include test_recipe.inc", contents)39        self.assertTrue(find_in_contents, msg = "Flattening layers did not work. bitbake-layers flatten output: %s" % result.output)40    @OETestID(1195)41    def test_bitbakelayers_add_remove(self):42        test_layer = os.path.join(get_bb_var('COREBASE'), 'meta-skeleton')43        result = runCmd('bitbake-layers show-layers')44        self.assertNotIn('meta-skeleton', result.output, "This test cannot run with meta-skeleton in bblayers.conf. bitbake-layers show-layers output: %s" % result.output)45        result = runCmd('bitbake-layers add-layer %s' % test_layer)46        result = runCmd('bitbake-layers show-layers')47        self.assertIn('meta-skeleton', result.output, msg = "Something wrong happened. meta-skeleton layer was not added to conf/bblayers.conf.  bitbake-layers show-layers output: %s" % result.output)48        result = runCmd('bitbake-layers remove-layer %s' % test_layer)49        result = runCmd('bitbake-layers show-layers')50        self.assertNotIn('meta-skeleton', result.output, msg = "meta-skeleton should have been removed at this step.  bitbake-layers show-layers output: %s" % result.output)51        result = runCmd('bitbake-layers add-layer %s' % test_layer)52        result = runCmd('bitbake-layers show-layers')53        self.assertIn('meta-skeleton', result.output, msg = "Something wrong happened. meta-skeleton layer was not added to conf/bblayers.conf.  bitbake-layers show-layers output: %s" % result.output)54        result = runCmd('bitbake-layers remove-layer */meta-skeleton')55        result = runCmd('bitbake-layers show-layers')56        self.assertNotIn('meta-skeleton', result.output, msg = "meta-skeleton should have been removed at this step.  bitbake-layers show-layers output: %s" % result.output)57    @OETestID(1384)58    def test_bitbakelayers_showrecipes(self):59        result = runCmd('bitbake-layers show-recipes')60        self.assertIn('aspell:', result.output)61        self.assertIn('mtd-utils:', result.output)62        self.assertIn('core-image-minimal:', result.output)63        result = runCmd('bitbake-layers show-recipes mtd-utils')64        self.assertIn('mtd-utils:', result.output)65        self.assertNotIn('aspell:', result.output)66        result = runCmd('bitbake-layers show-recipes -i image')67        self.assertIn('core-image-minimal', result.output)68        self.assertNotIn('mtd-utils:', result.output)69        result = runCmd('bitbake-layers show-recipes -i cmake,pkgconfig')70        self.assertIn('libproxy:', result.output)71        self.assertNotIn('mtd-utils:', result.output) # doesn't inherit either72        self.assertNotIn('wget:', result.output) # doesn't inherit cmake73        self.assertNotIn('waffle:', result.output) # doesn't inherit pkgconfig74        result = runCmd('bitbake-layers show-recipes -i nonexistentclass', ignore_status=True)75        self.assertNotEqual(result.status, 0, 'bitbake-layers show-recipes -i nonexistentclass should have failed')76        self.assertIn('ERROR:', result.output)77    def get_recipe_basename(self, recipe):78        recipe_file = ""79        result = runCmd("bitbake-layers show-recipes -f %s" % recipe)80        for line in result.output.splitlines():81            if recipe in line:82                recipe_file = line83                break84        self.assertTrue(os.path.isfile(recipe_file), msg = "Can't find recipe file for %s" % recipe)...

Full Screen

Full Screen

test_handler_runcmd.py

Source:test_handler_runcmd.py Github

copy

Full Screen

1# This file is part of cloud-init. See LICENSE file for license information.2from cloudinit.config import cc_runcmd, schema3from cloudinit.sources import DataSourceNone4from cloudinit import (distros, helpers, cloud, util)5from cloudinit.tests.helpers import (6    CiTestCase, FilesystemMockingTestCase, skipUnlessJsonSchema)7import logging8import os9import stat10LOG = logging.getLogger(__name__)11class TestRuncmd(FilesystemMockingTestCase):12    with_logs = True13    def setUp(self):14        super(TestRuncmd, self).setUp()15        self.subp = util.subp16        self.new_root = self.tmp_dir()17    def _get_cloud(self, distro):18        self.patchUtils(self.new_root)19        paths = helpers.Paths({'scripts': self.new_root})20        cls = distros.fetch(distro)21        mydist = cls(distro, {}, paths)22        myds = DataSourceNone.DataSourceNone({}, mydist, paths)23        paths.datasource = myds24        return cloud.Cloud(myds, paths, {}, mydist, None)25    def test_handler_skip_if_no_runcmd(self):26        """When the provided config doesn't contain runcmd, skip it."""27        cfg = {}28        mycloud = self._get_cloud('ubuntu')29        cc_runcmd.handle('notimportant', cfg, mycloud, LOG, None)30        self.assertIn(31            "Skipping module named notimportant, no 'runcmd' key",32            self.logs.getvalue())33    def test_handler_invalid_command_set(self):34        """Commands which can't be converted to shell will raise errors."""35        invalid_config = {'runcmd': 1}36        cc = self._get_cloud('ubuntu')37        cc_runcmd.handle('cc_runcmd', invalid_config, cc, LOG, [])38        self.assertIn(39            'Failed to shellify 1 into file'40            ' /var/lib/cloud/instances/iid-datasource-none/scripts/runcmd',41            self.logs.getvalue())42    @skipUnlessJsonSchema()43    def test_handler_schema_validation_warns_non_array_type(self):44        """Schema validation warns of non-array type for runcmd key.45        Schema validation is not strict, so runcmd attempts to shellify the46        invalid content.47        """48        invalid_config = {'runcmd': 1}49        cc = self._get_cloud('ubuntu')50        cc_runcmd.handle('cc_runcmd', invalid_config, cc, LOG, [])51        self.assertIn(52            'Invalid config:\nruncmd: 1 is not of type \'array\'',53            self.logs.getvalue())54        self.assertIn('Failed to shellify', self.logs.getvalue())55    @skipUnlessJsonSchema()56    def test_handler_schema_validation_warns_non_array_item_type(self):57        """Schema validation warns of non-array or string runcmd items.58        Schema validation is not strict, so runcmd attempts to shellify the59        invalid content.60        """61        invalid_config = {62            'runcmd': ['ls /', 20, ['wget', 'http://stuff/blah'], {'a': 'n'}]}63        cc = self._get_cloud('ubuntu')64        cc_runcmd.handle('cc_runcmd', invalid_config, cc, LOG, [])65        expected_warnings = [66            'runcmd.1: 20 is not valid under any of the given schemas',67            'runcmd.3: {\'a\': \'n\'} is not valid under any of the given'68            ' schema'69        ]70        logs = self.logs.getvalue()71        for warning in expected_warnings:72            self.assertIn(warning, logs)73        self.assertIn('Failed to shellify', logs)74    def test_handler_write_valid_runcmd_schema_to_file(self):75        """Valid runcmd schema is written to a runcmd shell script."""76        valid_config = {'runcmd': [['ls', '/']]}77        cc = self._get_cloud('ubuntu')78        cc_runcmd.handle('cc_runcmd', valid_config, cc, LOG, [])79        runcmd_file = os.path.join(80            self.new_root,81            'var/lib/cloud/instances/iid-datasource-none/scripts/runcmd')82        self.assertEqual("#!/bin/sh\n'ls' '/'\n", util.load_file(runcmd_file))83        file_stat = os.stat(runcmd_file)84        self.assertEqual(0o700, stat.S_IMODE(file_stat.st_mode))85class TestSchema(CiTestCase):86    """Directly test schema rather than through handle."""87    def test_duplicates_are_fine_array(self):88        """Duplicated array entries are allowed."""89        try:90            byebye = ["echo", "bye"]91            schema.validate_cloudconfig_schema(92                {'runcmd': [byebye, byebye]}, cc_runcmd.schema, strict=True)93        except schema.SchemaValidationError as e:94            self.fail("runcmd entries as list are allowed to be duplicates.")95    def test_duplicates_are_fine_string(self):96        """Duplicated string entries are allowed."""97        try:98            byebye = "echo bye"99            schema.validate_cloudconfig_schema(100                {'runcmd': [byebye, byebye]}, cc_runcmd.schema, strict=True)101        except schema.SchemaValidationError as e:102            self.fail("runcmd entries are allowed to be duplicates.")...

Full Screen

Full Screen

mpc.py

Source:mpc.py Github

copy

Full Screen

1from subprocess import call, check_output2import shlex3import socket4#TODO: Implement the rest5def onoff(val):6    if val == True:7        return "on"8    else:9        return "off"10class MPCClient:11    def __init__(self,hostname="localhost",port="6600"):12        self.host = hostname13        self.port = port14    def __runcmd(self,cmd,output=False,args=[]):15        c = ["mpc","-h",self.host,"-p",self.port,cmd]16        if(output):17            return check_output(c+args).decode()18        else:19            return call(c+args)20    def __runtcp(self,cmd,output=False,args=[]):21        BUFFER = 51222        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)23        try:24            s.connect((self.host,int(self.port)))25        except:26            raise Exception("Couldn't connect to MPD")27        data = s.recv(BUFFER).decode()28        if not data.startswith("OK MPD"):29            s.close()30            raise Exception("Couldn't connect to MPD")31        msg = cmd+" "+" ".join(args)+"\n"32        s.send(msg.encode())33        data = s.recv(BUFFER)34        s.close()35        return True36    def set_crossfade(self,seconds):37        return self.__runcmd("crossfade",False,[str(seconds)])38    def get_crossfade(self):39        cfade = self.__runcmd("crossfade",True)40        if(cfade):41            cfade = int(cfade[11:])42        return cfade43    def current(self):44        return self.__runcmd("current",True)45    def crop(self):46        return self.__runcmd("crop")47    def clear(self):48        return self.__runcmd("clear")49    def pause(self):50        return self.__runcmd("pause")51    def delete(self,songid=0):52        return self.__runcmd("del",False,[songid])53    def idle(self,filter=None):54        if filter != None:55            return self.__runcmd("idle",True,[filter])56        else:57            return self.__runcmd("idle",True)58    def play(self,songid=None):59        if(songid != None):60            return self.__runcmd("play",False,[songid])61        else:62            return self.__runcmd("play",False)63    def listall(self,playlist=None):64        if playlist != None:65            return self.__runcmd("listall",True,[playlist]).split("\n")66        else:67            return self.__runcmd("listall",True).split("\n")68    def load(self,playlist):69        return self.__runcmd("load",False,[playlist])70    def ls(self,directory=None):71        if directory != None:72            return self.__runcmd("ls",True,[directory])73        else:74            return self.__runcmd("ls",True)75    def lsplaylists(self,directory=None):76        return self.__runcmd("lsplaylists",True)77    def move(self,frm,to):78        return self.__runcmd("move",False,[frm,to])79    def next(self):80        return self.__runcmd("next")81    def playlist(self,playlist=None):82        if playlist != None:83            return self.__runcmd("playlist",True,[playlist])84        else:85            return self.__runcmd("playlist",True)86    def prev(self):87        return self.__runcmd("prev")88    def random(self,on=None):89        if on != None:90            return self.__runcmd("random",False,[onoff(om)])91        else:92            return self.__runcmd("random",False)93    def repeat(self,on=None):94        if on != None:95            return self.__runcmd("repeat",False,[onoff(om)])96        else:97            return self.__runcmd("repeat",False)98    def single(self,on=None):99        if on != None:100            return self.__runcmd("single",False,[onoff(om)])101        else:102            return self.__runcmd("single",False)103    def consume(self,on=None):104        if on != None:105            return self.__runcmd("consume",False,[onoff(om)])106        else:107            return self.__runcmd("consume",False)108    def rm(self,playlist):109        return self.__runcmd("rm",False,[playlist])110    def save(self,playlist):111        return self.__runcmd("save",False,[playlist])112    def shuffle(self):113        return self.__runcmd("shuffle")114    def stats(self):115        return self.__runcmd("stats",True)116    def stop(self):117        return self.__runcmd("stop")118    def toggle(self):119        return self.__runcmd("toggle")120    def add(self,file):121        file = '"'+file+'"'122        return self.__runtcp("add",False,[file])123    def insert(self,file):124        file = shlex.quote(file)125        return self.__runtcp("insert",False,[file])126    def update(self,wait=False,path=None):127        path = shlex.quote(path)128        args = []129        if wait:130            args.append("--wait")131        if path != None:132            args.append(path)133        return self.__runcmd("update",False,args)134    def version(self):...

Full Screen

Full Screen

para_4_normal.js

Source:para_4_normal.js Github

copy

Full Screen

1const { runCmd } = require('./cmd')2runCmd('start my para_4_normal_1 inpainting...',3    'cd ../generative_inpainting/test/para_4_normal && python test_1.py',4    'generative inpainting done!',5    1000)6runCmd('start my para_4_normal_2 inpainting...',7    'cd ../generative_inpainting/test/para_4_normal && python test_2.py',8    'generative inpainting done!',9    2000)10runCmd('start my para_4_normal_3 inpainting...',11    'cd ../generative_inpainting/test/para_4_normal && python test_3.py',12    'generative inpainting done!',13    3000)14runCmd('start my para_4_normal_4 inpainting...',15    'cd ../generative_inpainting/test/para_4_normal && python test_4.py',16    'generative inpainting done!',17    4000)18runCmd('start my para_4_normal_5 inpainting...',19    'cd ../generative_inpainting/test/para_4_normal && python test_5.py',20    'generative inpainting done!',21    5000)22runCmd('start my para_4_normal_6 inpainting...',23    'cd ../generative_inpainting/test/para_4_normal && python test_6.py',24    'generative inpainting done!',25    6000)26    runCmd('start my para_4_normal_7 inpainting...',27    'cd ../generative_inpainting/test/para_4_normal && python test_7.py',28    'generative inpainting done!',29    7000)30    runCmd('start my para_4_normal_8 inpainting...',31    'cd ../generative_inpainting/test/para_4_normal && python test_8.py',32    'generative inpainting done!',33    8000)34    runCmd('start my para_4_normal_9 inpainting...',35    'cd ../generative_inpainting/test/para_4_normal && python test_9.py',36    'generative inpainting done!',37    9000)38    runCmd('start my para_4_normal_10 inpainting...',39    'cd ../generative_inpainting/test/para_4_normal && python test_10.py',40    'generative inpainting done!',41    10000)42    runCmd('start my para_4_normal_11 inpainting...',43    'cd ../generative_inpainting/test/para_4_normal && python test_11.py',44    'generative inpainting done!',...

Full Screen

Full Screen

Using AI Code Generation

copy

Full Screen

1describe('My First Test', () => {2    it('Visits the Kitchen Sink', () => {3      cy.contains('type').click()4      cy.url().should('include', '/commands/actions')5      cy.get('.action-email')6        .type('

Full Screen

Using AI Code Generation

copy

Full Screen

1Cypress.Commands.add('runCmd', (cmd) => {2    cy.exec(cmd, {log: true}).its('stdout').then((stdout) => {3        cy.log(stdout)4    })5})6Cypress.Commands.add('runCmd', (cmd) => {7    cy.exec(cmd, {log: true}).its('stdout').then((stdout) => {8        cy.log(stdout)9    })10})11Cypress.Commands.add('runCmd', (cmd) => {12    cy.exec(cmd, {log: true}).its('stdout').then((stdout) => {13        cy.log(stdout)14    })15})16Cypress.Commands.add('runCmd', (cmd) => {17    cy.exec(cmd, {log: true}).its('stdout').then((stdout) => {18        cy.log(stdout)19    })20})21Cypress.Commands.add('runCmd', (cmd) => {22    cy.exec(cmd, {log: true}).its('stdout').then((stdout) => {23        cy.log(stdout)24    })25})26Cypress.Commands.add('runCmd', (cmd) => {27    cy.exec(cmd, {log: true}).its('stdout').then((stdout) => {28        cy.log(stdout)29    })30})31Cypress.Commands.add('runCmd', (cmd) => {32    cy.exec(cmd, {log: true}).its('stdout').then((stdout) => {33        cy.log(stdout)34    })35})36Cypress.Commands.add('runCmd', (cmd) => {37    cy.exec(cmd, {log: true}).its('stdout').then((stdout) => {38        cy.log(stdout)39    })40})41Cypress.Commands.add('runCmd', (cmd) => {42    cy.exec(cmd, {log: true}).its('stdout').then((stdout) => {43        cy.log(stdout)44    })45})

Full Screen

Using AI Code Generation

copy

Full Screen

1describe('Run Cypress commands', () => {2  it('Run Cypress commands', () => {3    cy.get('.action-email')4      .type('fake@email')5      .should('have.value', 'fake@email')6  })7})8describe('Run Cypress commands', () => {9  it('Run Cypress commands', () => {10    cy.get('.action-email')11      .type('fake@email')12      .should('have.value', 'fake@email')13  })14})15describe('Run Cypress commands', () => {16  it('Run Cypress commands', () => {17    cy.get('.action-email')18      .type('fake@email')19      .should('have.value', 'fake@email')20  })21})22describe('Run Cypress commands', () => {23  it('Run Cypress commands', () => {24    cy.get('.action-email')25      .type('fake@email')26      .should('have.value', 'fake@email')27  })28})29describe('Run Cypress commands', () => {30  it('Run Cypress commands', () => {31    cy.get('.action-email')32      .type('fake@email')33      .should('have.value', 'fake@email')34  })35})

Full Screen

Using AI Code Generation

copy

Full Screen

1describe('Cypress runCmd', () => {2    it('runCmd', () => {3        cy.runCmd('ls -la')4    })5})6describe('Cypress runCmd', () => {7    it('runCmd', () => {8        cy.runCmd('ls -la', {9        })10    })11})12describe('Cypress runCmd', () => {13    it('runCmd', () => {14        cy.runCmd('ls -la', {15        }).then((res) => {16            expect(res).to.be.not.null17        })18    })19})20describe('Cypress runCmd', () => {21    it('runCmd', () => {22        cy.runCmd('ls -la', {23        }).then((res) => {24            expect(res).to.be.not.null25            expect(res.code).to.be.equal(0)26            expect(res.stdout).to.be.not.null27            expect(res.stderr).to.be.null28        })29    })30})31describe('Cypress runCmd', () => {32    it('runCmd', () => {33        cy.runCmd('ls -la', {34        }).then((res) => {35            expect(res).to.be.not.null36            expect(res.code).to.be.equal(0)37            expect(res.stdout).to.be.not.null38            expect(res.stderr).to.be.null39            expect(res.stdout).to.include('test.js')40        })41    })42})43describe('Cypress runCmd', () => {44    it('runCmd', () => {45        cy.runCmd('ls -la', {46        }).then((res) => {47            expect(res

Full Screen

Cypress Tutorial

Cypress is a renowned Javascript-based open-source, easy-to-use end-to-end testing framework primarily used for testing web applications. Cypress is a relatively new player in the automation testing space and has been gaining much traction lately, as evidenced by the number of Forks (2.7K) and Stars (42.1K) for the project. LambdaTest’s Cypress Tutorial covers step-by-step guides that will help you learn from the basics till you run automation tests on LambdaTest.

Chapters:

  1. What is Cypress? -
  2. Why Cypress? - Learn why Cypress might be a good choice for testing your web applications.
  3. Features of Cypress Testing - Learn about features that make Cypress a powerful and flexible tool for testing web applications.
  4. Cypress Drawbacks - Although Cypress has many strengths, it has a few limitations that you should be aware of.
  5. Cypress Architecture - Learn more about Cypress architecture and how it is designed to be run directly in the browser, i.e., it does not have any additional servers.
  6. Browsers Supported by Cypress - Cypress is built on top of the Electron browser, supporting all modern web browsers. Learn browsers that support Cypress.
  7. Selenium vs Cypress: A Detailed Comparison - Compare and explore some key differences in terms of their design and features.
  8. Cypress Learning: Best Practices - Take a deep dive into some of the best practices you should use to avoid anti-patterns in your automation tests.
  9. How To Run Cypress Tests on LambdaTest? - Set up a LambdaTest account, and now you are all set to learn how to run Cypress tests.

Certification

You can elevate your expertise with end-to-end testing using the Cypress automation framework and stay one step ahead in your career by earning a Cypress certification. Check out our Cypress 101 Certification.

YouTube

Watch this 3 hours of complete tutorial to learn the basics of Cypress and various Cypress commands with the Cypress testing at LambdaTest.

Run Cypress automation tests on LambdaTest cloud grid

Perform automation testing on 3000+ real desktop and mobile devices online.

Try LambdaTest Now !!

Get 100 minutes of automation test minutes FREE!!

Next-Gen App & Browser Testing Cloud

Was this article helpful?

Helpful

NotHelpful