How to use runCmd method in Cypress

...25        self.assertEqual(checkvars, {}, 'Some variables not found: %s' % checkvars)26        for inherit in checkinherits:27            self.assertIn(inherit, inherits, 'Missing inherit of %s' % inherit)28    def _check_bbappend(self, testrecipe, recipefile, appenddir):29        result = runCmd('bitbake-layers show-appends', cwd=self.builddir)30        resultlines = result.output.splitlines()31        inrecipe = False32        bbappends = []33        bbappendfile = None34        for line in resultlines:35            if inrecipe:36                if line.startswith(' '):37                    bbappends.append(line.strip())38                else:39                    break40            elif line == '%s:' % os.path.basename(recipefile):41                inrecipe = True42        self.assertLessEqual(len(bbappends), 2, '%s recipe is being bbappended by another layer - bbappends found:\n  %s' % (testrecipe, '\n  '.join(bbappends)))43        for bbappend in bbappends:44            if bbappend.startswith(appenddir):45                bbappendfile = bbappend46                break47        else:48            self.assertTrue(False, 'bbappend for recipe %s does not seem to be created in test layer' % testrecipe)49        return bbappendfile50    def _create_temp_layer(self, templayerdir, addlayer, templayername, priority=999, recipepathspec='recipes-*/*'):51        create_temp_layer(templayerdir, templayername, priority, recipepathspec)52        if addlayer:53            self.add_command_to_tearDown('bitbake-layers remove-layer %s || true' % templayerdir)54            result = runCmd('bitbake-layers add-layer %s' % templayerdir, cwd=self.builddir)55    def _process_ls_output(self, output):56        """57        Convert ls -l output to a format we can reasonably compare from one context58        to another (e.g. from host to target)59        """60        filelist = []61        for line in output.splitlines():62            splitline = line.split()63            # Remove trailing . on perms64            splitline[0] = splitline[0].rstrip('.')65            # Remove leading . on paths66            splitline[-1] = splitline[-1].lstrip('.')67            # Drop fields we don't want to compare68            del splitline[7]69            del splitline[6]70            del splitline[5]71            del splitline[4]72            del splitline[1]73            filelist.append(' '.join(splitline))74        return filelist75class DevtoolTests(DevtoolBase):76    @testcase(1158)77    def test_create_workspace(self):78        # Check preconditions79        workspacedir = os.path.join(self.builddir, 'workspace')80        self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')81        result = runCmd('bitbake-layers show-layers')82        self.assertTrue('/workspace' not in result.output, 'This test cannot be run with a workspace layer in bblayers.conf')83        # Try creating a workspace layer with a specific path84        tempdir = tempfile.mkdtemp(prefix='devtoolqa')85        self.track_for_cleanup(tempdir)86        result = runCmd('devtool create-workspace %s' % tempdir)87        self.assertTrue(os.path.isfile(os.path.join(tempdir, 'conf', 'layer.conf')), msg = "No workspace created. devtool output: %s " % result.output)88        result = runCmd('bitbake-layers show-layers')89        self.assertIn(tempdir, result.output)90        # Try creating a workspace layer with the default path91        self.track_for_cleanup(workspacedir)92        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')93        result = runCmd('devtool create-workspace')94        self.assertTrue(os.path.isfile(os.path.join(workspacedir, 'conf', 'layer.conf')), msg = "No workspace created. devtool output: %s " % result.output)95        result = runCmd('bitbake-layers show-layers')96        self.assertNotIn(tempdir, result.output)97        self.assertIn(workspacedir, result.output)98    @testcase(1159)99    def test_devtool_add(self):100        # Check preconditions101        workspacedir = os.path.join(self.builddir, 'workspace')102        self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')103        # Fetch source104        tempdir = tempfile.mkdtemp(prefix='devtoolqa')105        self.track_for_cleanup(tempdir)106        url = 'http://www.ivarch.com/programs/sources/pv-1.5.3.tar.bz2'107        result = runCmd('wget %s' % url, cwd=tempdir)108        result = runCmd('tar xfv pv-1.5.3.tar.bz2', cwd=tempdir)109        srcdir = os.path.join(tempdir, 'pv-1.5.3')110        self.assertTrue(os.path.isfile(os.path.join(srcdir, 'configure')), 'Unable to find configure script in source directory')111        # Test devtool add112        self.track_for_cleanup(workspacedir)113        self.add_command_to_tearDown('bitbake -c cleansstate pv')114        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')115        result = runCmd('devtool add pv %s' % srcdir)116        self.assertTrue(os.path.exists(os.path.join(workspacedir, 'conf', 'layer.conf')), 'Workspace directory not created')117        # Test devtool status118        result = runCmd('devtool status')119        self.assertIn('pv', result.output)120        self.assertIn(srcdir, result.output)121        # Clean up anything in the workdir/sysroot/sstate cache (have to do this *after* devtool add since the recipe only exists then)122        bitbake('pv -c cleansstate')123        # Test devtool build124        result = runCmd('devtool build pv')125        installdir = get_bb_var('D', 'pv')126        self.assertTrue(installdir, 'Could not query installdir variable')127        bindir = get_bb_var('bindir', 'pv')128        self.assertTrue(bindir, 'Could not query bindir variable')129        if bindir[0] == '/':130            bindir = bindir[1:]131        self.assertTrue(os.path.isfile(os.path.join(installdir, bindir, 'pv')), 'pv binary not found in D')132    @testcase(1162)133    def test_devtool_add_library(self):134        # Check preconditions135        workspacedir = os.path.join(self.builddir, 'workspace')136        self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')137        # We don't have the ability to pick up this dependency automatically yet...138        bitbake('libusb1')139        # Fetch source140        tempdir = tempfile.mkdtemp(prefix='devtoolqa')141        self.track_for_cleanup(tempdir)142        url = 'http://www.intra2net.com/en/developer/libftdi/download/libftdi1-1.1.tar.bz2'143        result = runCmd('wget %s' % url, cwd=tempdir)144        result = runCmd('tar xfv libftdi1-1.1.tar.bz2', cwd=tempdir)145        srcdir = os.path.join(tempdir, 'libftdi1-1.1')146        self.assertTrue(os.path.isfile(os.path.join(srcdir, 'CMakeLists.txt')), 'Unable to find CMakeLists.txt in source directory')147        # Test devtool add (and use -V so we test that too)148        self.track_for_cleanup(workspacedir)149        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')150        result = runCmd('devtool add libftdi %s -V 1.1' % srcdir)151        self.assertTrue(os.path.exists(os.path.join(workspacedir, 'conf', 'layer.conf')), 'Workspace directory not created')152        # Test devtool status153        result = runCmd('devtool status')154        self.assertIn('libftdi', result.output)155        self.assertIn(srcdir, result.output)156        # Clean up anything in the workdir/sysroot/sstate cache (have to do this *after* devtool add since the recipe only exists then)157        bitbake('libftdi -c cleansstate')158        # Test devtool build159        result = runCmd('devtool build libftdi')160        self.add_command_to_tearDown('bitbake -c cleansstate libftdi')161        staging_libdir = get_bb_var('STAGING_LIBDIR', 'libftdi')162        self.assertTrue(staging_libdir, 'Could not query STAGING_LIBDIR variable')163        self.assertTrue(os.path.isfile(os.path.join(staging_libdir, 'libftdi1.so.2.1.0')), "libftdi binary not found in STAGING_LIBDIR. Output of devtool build libftdi %s" % result.output)164        # Test devtool reset165        stampprefix = get_bb_var('STAMP', 'libftdi')166        result = runCmd('devtool reset libftdi')167        result = runCmd('devtool status')168        self.assertNotIn('libftdi', result.output)169        self.assertTrue(stampprefix, 'Unable to get STAMP value for recipe libftdi')170        matches = glob.glob(stampprefix + '*')171        self.assertFalse(matches, 'Stamp files exist for recipe libftdi that should have been cleaned')172        self.assertFalse(os.path.isfile(os.path.join(staging_libdir, 'libftdi1.so.2.1.0')), 'libftdi binary still found in STAGING_LIBDIR after cleaning')173    @testcase(1160)174    def test_devtool_add_fetch(self):175        # Check preconditions176        workspacedir = os.path.join(self.builddir, 'workspace')177        self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')178        # Fetch source179        tempdir = tempfile.mkdtemp(prefix='devtoolqa')180        self.track_for_cleanup(tempdir)181        testver = '0.23'182        url = 'https://pypi.python.org/packages/source/M/MarkupSafe/MarkupSafe-%s.tar.gz' % testver183        testrecipe = 'python-markupsafe'184        srcdir = os.path.join(tempdir, testrecipe)185        # Test devtool add186        self.track_for_cleanup(workspacedir)187        self.add_command_to_tearDown('bitbake -c cleansstate %s' % testrecipe)188        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')189        result = runCmd('devtool add %s %s -f %s' % (testrecipe, srcdir, url))190        self.assertTrue(os.path.exists(os.path.join(workspacedir, 'conf', 'layer.conf')), 'Workspace directory not created. %s' % result.output)191        self.assertTrue(os.path.isfile(os.path.join(srcdir, 'setup.py')), 'Unable to find setup.py in source directory')192        # Test devtool status193        result = runCmd('devtool status')194        self.assertIn(testrecipe, result.output)195        self.assertIn(srcdir, result.output)196        # Check recipe197        recipefile = get_bb_var('FILE', testrecipe)198        self.assertIn('%s.bb' % testrecipe, recipefile, 'Recipe file incorrectly named')199        checkvars = {}200        checkvars['S'] = '${WORKDIR}/MarkupSafe-%s' % testver201        checkvars['SRC_URI'] = url202        self._test_recipe_contents(recipefile, checkvars, [])203        # Try with version specified204        result = runCmd('devtool reset -n %s' % testrecipe)205        shutil.rmtree(srcdir)206        result = runCmd('devtool add %s %s -f %s -V %s' % (testrecipe, srcdir, url, testver))207        self.assertTrue(os.path.isfile(os.path.join(srcdir, 'setup.py')), 'Unable to find setup.py in source directory')208        # Test devtool status209        result = runCmd('devtool status')210        self.assertIn(testrecipe, result.output)211        self.assertIn(srcdir, result.output)212        # Check recipe213        recipefile = get_bb_var('FILE', testrecipe)214        self.assertIn('%s_%s.bb' % (testrecipe, testver), recipefile, 'Recipe file incorrectly named')215        checkvars = {}216        checkvars['S'] = '${WORKDIR}/MarkupSafe-${PV}'217        checkvars['SRC_URI'] = url.replace(testver, '${PV}')218        self._test_recipe_contents(recipefile, checkvars, [])219    @testcase(1161)220    def test_devtool_add_fetch_git(self):221        # Check preconditions222        workspacedir = os.path.join(self.builddir, 'workspace')223        self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')224        # Fetch source225        tempdir = tempfile.mkdtemp(prefix='devtoolqa')226        self.track_for_cleanup(tempdir)227        url = 'git://git.yoctoproject.org/libmatchbox'228        checkrev = '462f0652055d89c648ddd54fd7b03f175c2c6973'229        testrecipe = 'libmatchbox2'230        srcdir = os.path.join(tempdir, testrecipe)231        # Test devtool add232        self.track_for_cleanup(workspacedir)233        self.add_command_to_tearDown('bitbake -c cleansstate %s' % testrecipe)234        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')235        result = runCmd('devtool add %s %s -f %s' % (testrecipe, srcdir, url))236        self.assertTrue(os.path.exists(os.path.join(workspacedir, 'conf', 'layer.conf')), 'Workspace directory not created: %s' % result.output)237        self.assertTrue(os.path.isfile(os.path.join(srcdir, 'configure.ac')), 'Unable to find configure.ac in source directory')238        # Test devtool status239        result = runCmd('devtool status')240        self.assertIn(testrecipe, result.output)241        self.assertIn(srcdir, result.output)242        # Check recipe243        recipefile = get_bb_var('FILE', testrecipe)244        self.assertIn('_git.bb', recipefile, 'Recipe file incorrectly named')245        checkvars = {}246        checkvars['S'] = '${WORKDIR}/git'247        checkvars['PV'] = '1.0+git${SRCPV}'248        checkvars['SRC_URI'] = url249        checkvars['SRCREV'] = '${AUTOREV}'250        self._test_recipe_contents(recipefile, checkvars, [])251        # Try with revision and version specified252        result = runCmd('devtool reset -n %s' % testrecipe)253        shutil.rmtree(srcdir)254        url_rev = '%s;rev=%s' % (url, checkrev)255        result = runCmd('devtool add %s %s -f "%s" -V 1.5' % (testrecipe, srcdir, url_rev))256        self.assertTrue(os.path.isfile(os.path.join(srcdir, 'configure.ac')), 'Unable to find configure.ac in source directory')257        # Test devtool status258        result = runCmd('devtool status')259        self.assertIn(testrecipe, result.output)260        self.assertIn(srcdir, result.output)261        # Check recipe262        recipefile = get_bb_var('FILE', testrecipe)263        self.assertIn('_git.bb', recipefile, 'Recipe file incorrectly named')264        checkvars = {}265        checkvars['S'] = '${WORKDIR}/git'266        checkvars['PV'] = '1.5+git${SRCPV}'267        checkvars['SRC_URI'] = url268        checkvars['SRCREV'] = checkrev269        self._test_recipe_contents(recipefile, checkvars, [])270    @testcase(1164)271    def test_devtool_modify(self):272        # Check preconditions273        workspacedir = os.path.join(self.builddir, 'workspace')274        self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')275        # Clean up anything in the workdir/sysroot/sstate cache276        bitbake('mdadm -c cleansstate')277        # Try modifying a recipe278        tempdir = tempfile.mkdtemp(prefix='devtoolqa')279        self.track_for_cleanup(tempdir)280        self.track_for_cleanup(workspacedir)281        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')282        self.add_command_to_tearDown('bitbake -c clean mdadm')283        result = runCmd('devtool modify mdadm -x %s' % tempdir)284        self.assertTrue(os.path.exists(os.path.join(tempdir, 'Makefile')), 'Extracted source could not be found')285        self.assertTrue(os.path.isdir(os.path.join(tempdir, '.git')), 'git repository for external source tree not found')286        self.assertTrue(os.path.exists(os.path.join(workspacedir, 'conf', 'layer.conf')), 'Workspace directory not created')287        matches = glob.glob(os.path.join(workspacedir, 'appends', 'mdadm_*.bbappend'))288        self.assertTrue(matches, 'bbappend not created %s' % result.output)289        # Test devtool status290        result = runCmd('devtool status')291        self.assertIn('mdadm', result.output)292        self.assertIn(tempdir, result.output)293        # Check git repo294        result = runCmd('git status --porcelain', cwd=tempdir)295        self.assertEqual(result.output.strip(), "", 'Created git repo is not clean')296        result = runCmd('git symbolic-ref HEAD', cwd=tempdir)297        self.assertEqual(result.output.strip(), "refs/heads/devtool", 'Wrong branch in git repo')298        # Try building299        bitbake('mdadm')300        # Try making (minor) modifications to the source301        result = runCmd("sed -i 's!^\.TH.*!.TH MDADM 8 \"\" v9.999-custom!' %s" % os.path.join(tempdir, 'mdadm.8.in'))302        bitbake('mdadm -c package')303        pkgd = get_bb_var('PKGD', 'mdadm')304        self.assertTrue(pkgd, 'Could not query PKGD variable')305        mandir = get_bb_var('mandir', 'mdadm')306        self.assertTrue(mandir, 'Could not query mandir variable')307        if mandir[0] == '/':308            mandir = mandir[1:]309        with open(os.path.join(pkgd, mandir, 'man8', 'mdadm.8'), 'r') as f:310            for line in f:311                if line.startswith('.TH'):312                    self.assertEqual(line.rstrip(), '.TH MDADM 8 "" v9.999-custom', 'man file not modified. man searched file path: %s' % os.path.join(pkgd, mandir, 'man8', 'mdadm.8'))313        # Test devtool reset314        stampprefix = get_bb_var('STAMP', 'mdadm')315        result = runCmd('devtool reset mdadm')316        result = runCmd('devtool status')317        self.assertNotIn('mdadm', result.output)318        self.assertTrue(stampprefix, 'Unable to get STAMP value for recipe mdadm')319        matches = glob.glob(stampprefix + '*')320        self.assertFalse(matches, 'Stamp files exist for recipe mdadm that should have been cleaned')321    @testcase(1166)322    def test_devtool_modify_invalid(self):323        # Check preconditions324        workspacedir = os.path.join(self.builddir, 'workspace')325        self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')326        # Try modifying some recipes327        tempdir = tempfile.mkdtemp(prefix='devtoolqa')328        self.track_for_cleanup(tempdir)329        self.track_for_cleanup(workspacedir)330        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')331        testrecipes = 'perf kernel-devsrc package-index core-image-minimal meta-toolchain packagegroup-core-sdk meta-ide-support'.split()332        # Find actual name of gcc-source since it now includes the version - crude, but good enough for this purpose333        result = runCmd('bitbake-layers show-recipes gcc-source*')334        reading = False335        for line in result.output.splitlines():336            if line.startswith('=='):337                reading = True338            elif reading and not line.startswith(' '):339                testrecipes.append(line.split(':')[0])340        for testrecipe in testrecipes:341            # Check it's a valid recipe342            bitbake('%s -e' % testrecipe)343            # devtool extract should fail344            result = runCmd('devtool extract %s %s' % (testrecipe, os.path.join(tempdir, testrecipe)), ignore_status=True)345            self.assertNotEqual(result.status, 0, 'devtool extract on %s should have failed. devtool output: %s' % (testrecipe, result.output))346            self.assertNotIn('Fetching ', result.output, 'devtool extract on %s should have errored out before trying to fetch' % testrecipe)347            self.assertIn('ERROR: ', result.output, 'devtool extract on %s should have given an ERROR' % testrecipe)348            # devtool modify should fail349            result = runCmd('devtool modify %s -x %s' % (testrecipe, os.path.join(tempdir, testrecipe)), ignore_status=True)350            self.assertNotEqual(result.status, 0, 'devtool modify on %s should have failed. devtool output: %s' %  (testrecipe, result.output))351            self.assertIn('ERROR: ', result.output, 'devtool modify on %s should have given an ERROR' % testrecipe)352    @testcase(1165)353    def test_devtool_modify_git(self):354        # Check preconditions355        workspacedir = os.path.join(self.builddir, 'workspace')356        self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')357        testrecipe = 'mkelfimage'358        src_uri = get_bb_var('SRC_URI', testrecipe)359        self.assertIn('git://', src_uri, 'This test expects the %s recipe to be a git recipe' % testrecipe)360        # Clean up anything in the workdir/sysroot/sstate cache361        bitbake('%s -c cleansstate' % testrecipe)362        # Try modifying a recipe363        tempdir = tempfile.mkdtemp(prefix='devtoolqa')364        self.track_for_cleanup(tempdir)365        self.track_for_cleanup(workspacedir)366        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')367        self.add_command_to_tearDown('bitbake -c clean %s' % testrecipe)368        result = runCmd('devtool modify %s -x %s' % (testrecipe, tempdir))369        self.assertTrue(os.path.exists(os.path.join(tempdir, 'Makefile')), 'Extracted source could not be found')370        self.assertTrue(os.path.isdir(os.path.join(tempdir, '.git')), 'git repository for external source tree not found')371        self.assertTrue(os.path.exists(os.path.join(workspacedir, 'conf', 'layer.conf')), 'Workspace directory not created. devtool output: %s' % result.output)372        matches = glob.glob(os.path.join(workspacedir, 'appends', 'mkelfimage_*.bbappend'))373        self.assertTrue(matches, 'bbappend not created')374        # Test devtool status375        result = runCmd('devtool status')376        self.assertIn(testrecipe, result.output)377        self.assertIn(tempdir, result.output)378        # Check git repo379        result = runCmd('git status --porcelain', cwd=tempdir)380        self.assertEqual(result.output.strip(), "", 'Created git repo is not clean')381        result = runCmd('git symbolic-ref HEAD', cwd=tempdir)382        self.assertEqual(result.output.strip(), "refs/heads/devtool", 'Wrong branch in git repo')383        # Try building384        bitbake(testrecipe)385    @testcase(1167)386    def test_devtool_modify_localfiles(self):387        # Check preconditions388        workspacedir = os.path.join(self.builddir, 'workspace')389        self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')390        testrecipe = 'lighttpd'391        src_uri = (get_bb_var('SRC_URI', testrecipe) or '').split()392        foundlocal = False393        for item in src_uri:394            if item.startswith('file://') and '.patch' not in item:395                foundlocal = True396                break397        self.assertTrue(foundlocal, 'This test expects the %s recipe to fetch local files and it seems that it no longer does' % testrecipe)398        # Clean up anything in the workdir/sysroot/sstate cache399        bitbake('%s -c cleansstate' % testrecipe)400        # Try modifying a recipe401        tempdir = tempfile.mkdtemp(prefix='devtoolqa')402        self.track_for_cleanup(tempdir)403        self.track_for_cleanup(workspacedir)404        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')405        self.add_command_to_tearDown('bitbake -c clean %s' % testrecipe)406        result = runCmd('devtool modify %s -x %s' % (testrecipe, tempdir))407        self.assertTrue(os.path.exists(os.path.join(tempdir, 'configure.ac')), 'Extracted source could not be found')408        self.assertTrue(os.path.exists(os.path.join(workspacedir, 'conf', 'layer.conf')), 'Workspace directory not created')409        matches = glob.glob(os.path.join(workspacedir, 'appends', '%s_*.bbappend' % testrecipe))410        self.assertTrue(matches, 'bbappend not created')411        # Test devtool status412        result = runCmd('devtool status')413        self.assertIn(testrecipe, result.output)414        self.assertIn(tempdir, result.output)415        # Try building416        bitbake(testrecipe)417    @testcase(1169)418    def test_devtool_update_recipe(self):419        # Check preconditions420        workspacedir = os.path.join(self.builddir, 'workspace')421        self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')422        testrecipe = 'minicom'423        recipefile = get_bb_var('FILE', testrecipe)424        src_uri = get_bb_var('SRC_URI', testrecipe)425        self.assertNotIn('git://', src_uri, 'This test expects the %s recipe to NOT be a git recipe' % testrecipe)426        result = runCmd('git status . --porcelain', cwd=os.path.dirname(recipefile))427        self.assertEqual(result.output.strip(), "", '%s recipe is not clean' % testrecipe)428        # First, modify a recipe429        tempdir = tempfile.mkdtemp(prefix='devtoolqa')430        self.track_for_cleanup(tempdir)431        self.track_for_cleanup(workspacedir)432        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')433        # (don't bother with cleaning the recipe on teardown, we won't be building it)434        result = runCmd('devtool modify %s -x %s' % (testrecipe, tempdir))435        # Check git repo436        self.assertTrue(os.path.isdir(os.path.join(tempdir, '.git')), 'git repository for external source tree not found')437        result = runCmd('git status --porcelain', cwd=tempdir)438        self.assertEqual(result.output.strip(), "", 'Created git repo is not clean')439        result = runCmd('git symbolic-ref HEAD', cwd=tempdir)440        self.assertEqual(result.output.strip(), "refs/heads/devtool", 'Wrong branch in git repo')441        # Add a couple of commits442        # FIXME: this only tests adding, need to also test update and remove443        result = runCmd('echo "Additional line" >> README', cwd=tempdir)444        result = runCmd('git commit -a -m "Change the README"', cwd=tempdir)445        result = runCmd('echo "A new file" > devtool-new-file', cwd=tempdir)446        result = runCmd('git add devtool-new-file', cwd=tempdir)447        result = runCmd('git commit -m "Add a new file"', cwd=tempdir)448        self.add_command_to_tearDown('cd %s; rm %s/*.patch; git checkout %s %s' % (os.path.dirname(recipefile), testrecipe, testrecipe, os.path.basename(recipefile)))449        result = runCmd('devtool update-recipe %s' % testrecipe)450        result = runCmd('git status . --porcelain', cwd=os.path.dirname(recipefile))451        self.assertNotEqual(result.output.strip(), "", '%s recipe should be modified' % testrecipe)452        status = result.output.splitlines()453        self.assertEqual(len(status), 3, 'Less/more files modified than expected. Entire status:\n%s' % result.output)454        for line in status:455            if line.endswith('0001-Change-the-README.patch'):456                self.assertEqual(line[:3], '?? ', 'Unexpected status in line: %s' % line)457            elif line.endswith('0002-Add-a-new-file.patch'):458                self.assertEqual(line[:3], '?? ', 'Unexpected status in line: %s' % line)459            elif re.search('%s_[^_]*.bb$' % testrecipe, line):460                self.assertEqual(line[:3], ' M ', 'Unexpected status in line: %s' % line)461            else:462                raise AssertionError('Unexpected modified file in status: %s' % line)463    @testcase(1172)464    def test_devtool_update_recipe_git(self):465        # Check preconditions466        workspacedir = os.path.join(self.builddir, 'workspace')467        self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')468        testrecipe = 'mtd-utils'469        recipefile = get_bb_var('FILE', testrecipe)470        src_uri = get_bb_var('SRC_URI', testrecipe)471        self.assertIn('git://', src_uri, 'This test expects the %s recipe to be a git recipe' % testrecipe)472        patches = []473        for entry in src_uri.split():474            if entry.startswith('file://') and entry.endswith('.patch'):475                patches.append(entry[7:].split(';')[0])476        self.assertGreater(len(patches), 0, 'The %s recipe does not appear to contain any patches, so this test will not be effective' % testrecipe)477        result = runCmd('git status . --porcelain', cwd=os.path.dirname(recipefile))478        self.assertEqual(result.output.strip(), "", '%s recipe is not clean' % testrecipe)479        # First, modify a recipe480        tempdir = tempfile.mkdtemp(prefix='devtoolqa')481        self.track_for_cleanup(tempdir)482        self.track_for_cleanup(workspacedir)483        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')484        # (don't bother with cleaning the recipe on teardown, we won't be building it)485        result = runCmd('devtool modify %s -x %s' % (testrecipe, tempdir))486        # Check git repo487        self.assertTrue(os.path.isdir(os.path.join(tempdir, '.git')), 'git repository for external source tree not found')488        result = runCmd('git status --porcelain', cwd=tempdir)489        self.assertEqual(result.output.strip(), "", 'Created git repo is not clean')490        result = runCmd('git symbolic-ref HEAD', cwd=tempdir)491        self.assertEqual(result.output.strip(), "refs/heads/devtool", 'Wrong branch in git repo')492        # Add a couple of commits493        # FIXME: this only tests adding, need to also test update and remove494        result = runCmd('echo "# Additional line" >> Makefile', cwd=tempdir)495        result = runCmd('git commit -a -m "Change the Makefile"', cwd=tempdir)496        result = runCmd('echo "A new file" > devtool-new-file', cwd=tempdir)497        result = runCmd('git add devtool-new-file', cwd=tempdir)498        result = runCmd('git commit -m "Add a new file"', cwd=tempdir)499        self.add_command_to_tearDown('cd %s; git checkout %s %s' % (os.path.dirname(recipefile), testrecipe, os.path.basename(recipefile)))500        result = runCmd('devtool update-recipe %s' % testrecipe)501        result = runCmd('git status . --porcelain', cwd=os.path.dirname(recipefile))502        self.assertNotEqual(result.output.strip(), "", '%s recipe should be modified' % testrecipe)503        status = result.output.splitlines()504        for line in status:505            for patch in patches:506                if line.endswith(patch):507                    self.assertEqual(line[:3], ' D ', 'Unexpected status in line: %s' % line)508                    break509            else:510                if re.search('%s_[^_]*.bb$' % testrecipe, line):511                    self.assertEqual(line[:3], ' M ', 'Unexpected status in line: %s' % line)512                else:513                    raise AssertionError('Unexpected modified file in status: %s' % line)514        result = runCmd('git diff %s' % os.path.basename(recipefile), cwd=os.path.dirname(recipefile))515        addlines = ['SRCREV = ".*"', 'SRC_URI = "git://git.infradead.org/mtd-utils.git"']516        srcurilines = src_uri.split()517        srcurilines[0] = 'SRC_URI = "' + srcurilines[0]518        srcurilines.append('"')519        removelines = ['SRCREV = ".*"'] + srcurilines520        for line in result.output.splitlines():521            if line.startswith('+++') or line.startswith('---'):522                continue523            elif line.startswith('+'):524                matched = False525                for item in addlines:526                    if re.match(item, line[1:].strip()):527                        matched = True528                        break529                self.assertTrue(matched, 'Unexpected diff add line: %s' % line)530            elif line.startswith('-'):531                matched = False532                for item in removelines:533                    if re.match(item, line[1:].strip()):534                        matched = True535                        break536                self.assertTrue(matched, 'Unexpected diff remove line: %s' % line)537    @testcase(1170)538    def test_devtool_update_recipe_append(self):539        # Check preconditions540        workspacedir = os.path.join(self.builddir, 'workspace')541        self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')542        testrecipe = 'mdadm'543        recipefile = get_bb_var('FILE', testrecipe)544        src_uri = get_bb_var('SRC_URI', testrecipe)545        self.assertNotIn('git://', src_uri, 'This test expects the %s recipe to NOT be a git recipe' % testrecipe)546        result = runCmd('git status . --porcelain', cwd=os.path.dirname(recipefile))547        self.assertEqual(result.output.strip(), "", '%s recipe is not clean' % testrecipe)548        # First, modify a recipe549        tempdir = tempfile.mkdtemp(prefix='devtoolqa')550        tempsrcdir = os.path.join(tempdir, 'source')551        templayerdir = os.path.join(tempdir, 'layer')552        self.track_for_cleanup(tempdir)553        self.track_for_cleanup(workspacedir)554        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')555        # (don't bother with cleaning the recipe on teardown, we won't be building it)556        result = runCmd('devtool modify %s -x %s' % (testrecipe, tempsrcdir))557        # Check git repo558        self.assertTrue(os.path.isdir(os.path.join(tempsrcdir, '.git')), 'git repository for external source tree not found')559        result = runCmd('git status --porcelain', cwd=tempsrcdir)560        self.assertEqual(result.output.strip(), "", 'Created git repo is not clean')561        result = runCmd('git symbolic-ref HEAD', cwd=tempsrcdir)562        self.assertEqual(result.output.strip(), "refs/heads/devtool", 'Wrong branch in git repo')563        # Add a commit564        result = runCmd("sed 's!\\(#define VERSION\\W*\"[^\"]*\\)\"!\\1-custom\"!' -i ReadMe.c", cwd=tempsrcdir)565        result = runCmd('git commit -a -m "Add our custom version"', cwd=tempsrcdir)566        self.add_command_to_tearDown('cd %s; rm -f %s/*.patch; git checkout .' % (os.path.dirname(recipefile), testrecipe))567        # Create a temporary layer and add it to bblayers.conf568        self._create_temp_layer(templayerdir, True, 'selftestupdaterecipe')569        # Create the bbappend570        result = runCmd('devtool update-recipe %s -a %s' % (testrecipe, templayerdir))571        self.assertNotIn('WARNING:', result.output)572        # Check recipe is still clean573        result = runCmd('git status . --porcelain', cwd=os.path.dirname(recipefile))574        self.assertEqual(result.output.strip(), "", '%s recipe is not clean' % testrecipe)575        # Check bbappend was created576        splitpath = os.path.dirname(recipefile).split(os.sep)577        appenddir = os.path.join(templayerdir, splitpath[-2], splitpath[-1])578        bbappendfile = self._check_bbappend(testrecipe, recipefile, appenddir)579        patchfile = os.path.join(appenddir, testrecipe, '0001-Add-our-custom-version.patch')580        self.assertTrue(os.path.exists(patchfile), 'Patch file not created')581        # Check bbappend contents582        expectedlines = ['FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n',583                         '\n',584                         'SRC_URI += "file://0001-Add-our-custom-version.patch"\n',585                         '\n']586        with open(bbappendfile, 'r') as f:587            self.assertEqual(expectedlines, f.readlines())588        # Check we can run it again and bbappend isn't modified589        result = runCmd('devtool update-recipe %s -a %s' % (testrecipe, templayerdir))590        with open(bbappendfile, 'r') as f:591            self.assertEqual(expectedlines, f.readlines())592        # Drop new commit and check patch gets deleted593        result = runCmd('git reset HEAD^', cwd=tempsrcdir)594        result = runCmd('devtool update-recipe %s -a %s' % (testrecipe, templayerdir))595        self.assertFalse(os.path.exists(patchfile), 'Patch file not deleted')596        expectedlines2 = ['FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n',597                         '\n']598        with open(bbappendfile, 'r') as f:599            self.assertEqual(expectedlines2, f.readlines())600        # Put commit back and check we can run it if layer isn't in bblayers.conf601        os.remove(bbappendfile)602        result = runCmd('git commit -a -m "Add our custom version"', cwd=tempsrcdir)603        result = runCmd('bitbake-layers remove-layer %s' % templayerdir, cwd=self.builddir)604        result = runCmd('devtool update-recipe %s -a %s' % (testrecipe, templayerdir))605        self.assertIn('WARNING: Specified layer is not currently enabled in bblayers.conf', result.output)606        self.assertTrue(os.path.exists(patchfile), 'Patch file not created (with disabled layer)')607        with open(bbappendfile, 'r') as f:608            self.assertEqual(expectedlines, f.readlines())609        # Deleting isn't expected to work under these circumstances610    @testcase(1171)611    def test_devtool_update_recipe_append_git(self):612        # Check preconditions613        workspacedir = os.path.join(self.builddir, 'workspace')614        self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')615        testrecipe = 'mtd-utils'616        recipefile = get_bb_var('FILE', testrecipe)617        src_uri = get_bb_var('SRC_URI', testrecipe)618        self.assertIn('git://', src_uri, 'This test expects the %s recipe to be a git recipe' % testrecipe)619        for entry in src_uri.split():620            if entry.startswith('git://'):621                git_uri = entry622                break623        result = runCmd('git status . --porcelain', cwd=os.path.dirname(recipefile))624        self.assertEqual(result.output.strip(), "", '%s recipe is not clean' % testrecipe)625        # First, modify a recipe626        tempdir = tempfile.mkdtemp(prefix='devtoolqa')627        tempsrcdir = os.path.join(tempdir, 'source')628        templayerdir = os.path.join(tempdir, 'layer')629        self.track_for_cleanup(tempdir)630        self.track_for_cleanup(workspacedir)631        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')632        # (don't bother with cleaning the recipe on teardown, we won't be building it)633        result = runCmd('devtool modify %s -x %s' % (testrecipe, tempsrcdir))634        # Check git repo635        self.assertTrue(os.path.isdir(os.path.join(tempsrcdir, '.git')), 'git repository for external source tree not found')636        result = runCmd('git status --porcelain', cwd=tempsrcdir)637        self.assertEqual(result.output.strip(), "", 'Created git repo is not clean')638        result = runCmd('git symbolic-ref HEAD', cwd=tempsrcdir)639        self.assertEqual(result.output.strip(), "refs/heads/devtool", 'Wrong branch in git repo')640        # Add a commit641        result = runCmd('echo "# Additional line" >> Makefile', cwd=tempsrcdir)642        result = runCmd('git commit -a -m "Change the Makefile"', cwd=tempsrcdir)643        self.add_command_to_tearDown('cd %s; rm -f %s/*.patch; git checkout .' % (os.path.dirname(recipefile), testrecipe))644        # Create a temporary layer645        os.makedirs(os.path.join(templayerdir, 'conf'))646        with open(os.path.join(templayerdir, 'conf', 'layer.conf'), 'w') as f:647            f.write('BBPATH .= ":${LAYERDIR}"\n')648            f.write('BBFILES += "${LAYERDIR}/recipes-*/*/*.bbappend"\n')649            f.write('BBFILE_COLLECTIONS += "oeselftesttemplayer"\n')650            f.write('BBFILE_PATTERN_oeselftesttemplayer = "^${LAYERDIR}/"\n')651            f.write('BBFILE_PRIORITY_oeselftesttemplayer = "999"\n')652            f.write('BBFILE_PATTERN_IGNORE_EMPTY_oeselftesttemplayer = "1"\n')653        self.add_command_to_tearDown('bitbake-layers remove-layer %s || true' % templayerdir)654        result = runCmd('bitbake-layers add-layer %s' % templayerdir, cwd=self.builddir)655        # Create the bbappend656        result = runCmd('devtool update-recipe %s -a %s' % (testrecipe, templayerdir))657        self.assertNotIn('WARNING:', result.output)658        # Check recipe is still clean659        result = runCmd('git status . --porcelain', cwd=os.path.dirname(recipefile))660        self.assertEqual(result.output.strip(), "", '%s recipe is not clean' % testrecipe)661        # Check bbappend was created662        splitpath = os.path.dirname(recipefile).split(os.sep)663        appenddir = os.path.join(templayerdir, splitpath[-2], splitpath[-1])664        bbappendfile = self._check_bbappend(testrecipe, recipefile, appenddir)665        self.assertFalse(os.path.exists(os.path.join(appenddir, testrecipe)), 'Patch directory should not be created')666        # Check bbappend contents667        result = runCmd('git rev-parse HEAD', cwd=tempsrcdir)668        expectedlines = ['SRCREV = "%s"\n' % result.output,669                         '\n',670                         'SRC_URI = "%s"\n' % git_uri,671                         '\n']672        with open(bbappendfile, 'r') as f:673            self.assertEqual(expectedlines, f.readlines())674        # Check we can run it again and bbappend isn't modified675        result = runCmd('devtool update-recipe %s -a %s' % (testrecipe, templayerdir))676        with open(bbappendfile, 'r') as f:677            self.assertEqual(expectedlines, f.readlines())678        # Drop new commit and check SRCREV changes679        result = runCmd('git reset HEAD^', cwd=tempsrcdir)680        result = runCmd('devtool update-recipe %s -a %s' % (testrecipe, templayerdir))681        self.assertFalse(os.path.exists(os.path.join(appenddir, testrecipe)), 'Patch directory should not be created')682        result = runCmd('git rev-parse HEAD', cwd=tempsrcdir)683        expectedlines = ['SRCREV = "%s"\n' % result.output,684                         '\n',685                         'SRC_URI = "%s"\n' % git_uri,686                         '\n']687        with open(bbappendfile, 'r') as f:688            self.assertEqual(expectedlines, f.readlines())689        # Put commit back and check we can run it if layer isn't in bblayers.conf690        os.remove(bbappendfile)691        result = runCmd('git commit -a -m "Change the Makefile"', cwd=tempsrcdir)692        result = runCmd('bitbake-layers remove-layer %s' % templayerdir, cwd=self.builddir)693        result = runCmd('devtool update-recipe %s -a %s' % (testrecipe, templayerdir))694        self.assertIn('WARNING: Specified layer is not currently enabled in bblayers.conf', result.output)695        self.assertFalse(os.path.exists(os.path.join(appenddir, testrecipe)), 'Patch directory should not be created')696        result = runCmd('git rev-parse HEAD', cwd=tempsrcdir)697        expectedlines = ['SRCREV = "%s"\n' % result.output,698                         '\n',699                         'SRC_URI = "%s"\n' % git_uri,700                         '\n']701        with open(bbappendfile, 'r') as f:702            self.assertEqual(expectedlines, f.readlines())703        # Deleting isn't expected to work under these circumstances704    @testcase(1163)705    def test_devtool_extract(self):706        # Check preconditions707        workspacedir = os.path.join(self.builddir, 'workspace')708        self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')709        tempdir = tempfile.mkdtemp(prefix='devtoolqa')710        # Try devtool extract711        self.track_for_cleanup(tempdir)712        self.track_for_cleanup(workspacedir)713        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')714        result = runCmd('devtool extract remake %s' % tempdir)715        self.assertTrue(os.path.exists(os.path.join(tempdir, 'Makefile.am')), 'Extracted source could not be found')716        self.assertTrue(os.path.isdir(os.path.join(tempdir, '.git')), 'git repository for external source tree not found')717    @testcase(1168)718    def test_devtool_reset_all(self):719        # Check preconditions720        workspacedir = os.path.join(self.builddir, 'workspace')721        self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')722        tempdir = tempfile.mkdtemp(prefix='devtoolqa')723        self.track_for_cleanup(tempdir)724        self.track_for_cleanup(workspacedir)725        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')726        testrecipe1 = 'mdadm'727        testrecipe2 = 'cronie'728        result = runCmd('devtool modify -x %s %s' % (testrecipe1, os.path.join(tempdir, testrecipe1)))729        result = runCmd('devtool modify -x %s %s' % (testrecipe2, os.path.join(tempdir, testrecipe2)))730        result = runCmd('devtool build %s' % testrecipe1)731        result = runCmd('devtool build %s' % testrecipe2)732        stampprefix1 = get_bb_var('STAMP', testrecipe1)733        self.assertTrue(stampprefix1, 'Unable to get STAMP value for recipe %s' % testrecipe1)734        stampprefix2 = get_bb_var('STAMP', testrecipe2)735        self.assertTrue(stampprefix2, 'Unable to get STAMP value for recipe %s' % testrecipe2)736        result = runCmd('devtool reset -a')737        self.assertIn(testrecipe1, result.output)738        self.assertIn(testrecipe2, result.output)739        result = runCmd('devtool status')740        self.assertNotIn(testrecipe1, result.output)741        self.assertNotIn(testrecipe2, result.output)742        matches1 = glob.glob(stampprefix1 + '*')743        self.assertFalse(matches1, 'Stamp files exist for recipe %s that should have been cleaned' % testrecipe1)744        matches2 = glob.glob(stampprefix2 + '*')745        self.assertFalse(matches2, 'Stamp files exist for recipe %s that should have been cleaned' % testrecipe2)746    def test_devtool_deploy_target(self):747        # NOTE: Whilst this test would seemingly be better placed as a runtime test,748        # unfortunately the runtime tests run under bitbake and you can't run749        # devtool within bitbake (since devtool needs to run bitbake itself).750        # Additionally we are testing build-time functionality as well, so751        # really this has to be done as an oe-selftest test.752        #753        # Check preconditions754        machine = get_bb_var('MACHINE')755        if not machine.startswith('qemu'):756            self.skipTest('This test only works with qemu machines')757        if not os.path.exists('/etc/runqemu-nosudo'):758            self.skipTest('You must set up tap devices with scripts/runqemu-gen-tapdevs before running this test')759        result = runCmd('PATH="$PATH:/sbin:/usr/sbin" ip tuntap show', ignore_status=True)760        if result.status != 0:761            result = runCmd('PATH="$PATH:/sbin:/usr/sbin" ifconfig -a', ignore_status=True)762            if result.status != 0:763                self.skipTest('Failed to determine if tap devices exist with ifconfig or ip: %s' % result.output)764        for line in result.output.splitlines():765            if line.startswith('tap'):766                break767        else:768            self.skipTest('No tap devices found - you must set up tap devices with scripts/runqemu-gen-tapdevs before running this test')769        workspacedir = os.path.join(self.builddir, 'workspace')770        self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory')771        import pexpect772        # Definitions773        testrecipe = 'mdadm'774        testfile = '/sbin/mdadm'775        testimage = 'oe-selftest-image'776        testhost = '192.168.7.2'777        testcommand = '/sbin/mdadm --help'778        # Build an image to run779        bitbake("%s qemu-native qemu-helper-native" % testimage)780        deploy_dir_image = get_bb_var('DEPLOY_DIR_IMAGE')781        self.add_command_to_tearDown('bitbake -c clean %s' % testimage)782        self.add_command_to_tearDown('rm -f %s/%s*' % (deploy_dir_image, testimage))783        # Clean recipe so the first deploy will fail784        bitbake("%s -c clean" % testrecipe)785        # Try devtool modify786        tempdir = tempfile.mkdtemp(prefix='devtoolqa')787        self.track_for_cleanup(tempdir)788        self.track_for_cleanup(workspacedir)789        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')790        self.add_command_to_tearDown('bitbake -c clean %s' % testrecipe)791        result = runCmd('devtool modify %s -x %s' % (testrecipe, tempdir))792        # Test that deploy-target at this point fails (properly)793        result = runCmd('devtool deploy-target -n %s root@%s' % (testrecipe, testhost), ignore_status=True)794        self.assertNotEqual(result.output, 0, 'devtool deploy-target should have failed, output: %s' % result.output)795        self.assertNotIn(result.output, 'Traceback', 'devtool deploy-target should have failed with a proper error not a traceback, output: %s' % result.output)796        result = runCmd('devtool build %s' % testrecipe)797        # First try a dry-run of deploy-target798        result = runCmd('devtool deploy-target -n %s root@%s' % (testrecipe, testhost))799        self.assertIn('  %s' % testfile, result.output)800        # Boot the image801        console = pexpect.spawn('runqemu %s %s qemuparams="-snapshot" nographic' % (machine, testimage))802        console.expect("login:", timeout=120)803        # Now really test deploy-target804        result = runCmd('devtool deploy-target -c %s root@%s' % (testrecipe, testhost))805        # Run a test command to see if it was installed properly806        sshargs = '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no'807        result = runCmd('ssh %s root@%s %s' % (sshargs, testhost, testcommand))808        # Check if it deployed all of the files with the right ownership/perms809        # First look on the host - need to do this under pseudo to get the correct ownership/perms810        installdir = get_bb_var('D', testrecipe)811        fakerootenv = get_bb_var('FAKEROOTENV', testrecipe)812        fakerootcmd = get_bb_var('FAKEROOTCMD', testrecipe)813        result = runCmd('%s %s find . -type f -exec ls -l {} \;' % (fakerootenv, fakerootcmd), cwd=installdir)814        filelist1 = self._process_ls_output(result.output)815        # Now look on the target816        tempdir2 = tempfile.mkdtemp(prefix='devtoolqa')817        self.track_for_cleanup(tempdir2)818        tmpfilelist = os.path.join(tempdir2, 'files.txt')819        with open(tmpfilelist, 'w') as f:820            for line in filelist1:821                splitline = line.split()822                f.write(splitline[-1] + '\n')823        result = runCmd('cat %s | ssh -q %s root@%s \'xargs ls -l\'' % (tmpfilelist, sshargs, testhost))824        filelist2 = self._process_ls_output(result.output)825        filelist1.sort(key=lambda item: item.split()[-1])826        filelist2.sort(key=lambda item: item.split()[-1])827        self.assertEqual(filelist1, filelist2)828        # Test undeploy-target829        result = runCmd('devtool undeploy-target -c %s root@%s' % (testrecipe, testhost))830        result = runCmd('ssh %s root@%s %s' % (sshargs, testhost, testcommand), ignore_status=True)831        self.assertNotEqual(result, 0, 'undeploy-target did not remove command as it should have')...

1#! /usr/bin/env python32"""3Sets up an Ubuntu 14.04 x64 server to be a Counterblock Federated Node.4NOTE: The system should be properly secured before running this script.5TODO: This is admittedly a (bit of a) hack. In the future, take this kind of functionality out to a .deb with6      a postinst script to do all of this, possibly.7"""8import os9import sys10import re11import time12import getopt13import logging14import shutil15import socket16import urllib17import zipfile18import platform19import collections20import tempfile21import subprocess22import stat23import string24import random25try: #ignore import errors on windows26    import pwd27    import grp28except ImportError:29    pass30USERNAME = "xch"31DAEMON_USERNAME = "xchd"32REPO_COUNTERPARTYD_BUILD = "https://github.com/ClearingHouse/clearinghoused_build.git"33REPO_COUNTERWALLET = "https://github.com/ClearingHouse/clearwallet.git"34def pass_generator(size=14, chars=string.ascii_uppercase + string.ascii_lowercase + string.digits):35    return ''.join(random.choice(chars) for x in range(size))36def runcmd(command, abort_on_failure=True):37    logging.debug("RUNNING COMMAND: %s" % command)38    ret = os.system(command)39    if abort_on_failure and ret != 0:40        logging.error("Command failed: '%s'" % command)41        sys.exit(1)42def modify_config(param_re, content_to_add, filenames, replace_if_exists=True, dotall=False, add_newline=True):43    if not isinstance(filenames, (list, tuple)):44        filenames = [filenames,]45    re_flags = re.MULTILINE | re.DOTALL if dotall else re.MULTILINE46    if add_newline and not content_to_add.endswith('\n'):47        content_to_add += '\n'48    for filename in filenames:49        f = open(filename, 'r')50        content = f.read()51        f.close()52        if content[-1] != '\n':53            content += '\n'54        if not re.search(param_re, content, re_flags): #missing; add to config55            content += content_to_add56        elif replace_if_exists: #replace in config57            content = re.sub(param_re, content_to_add, content, flags=re_flags)58        f = open(filename, 'w')59        f.write(content)60        f.close()61def modify_cp_config(param_re, content_to_add, replace_if_exists=True, config='clearinghoused', net='both'):62    assert config in ('clearinghoused', 'clearblockd', 'both')63    assert net in ('mainnet', 'testnet', 'both')64    cfg_filenames = []65    if config in ('clearinghoused', 'both'):66        if net in ('mainnet', 'both'):67            cfg_filenames.append(os.path.join(os.path.expanduser('~'+USERNAME), ".config", "clearinghoused", "clearinghoused.conf"))68        if net in ('testnet', 'both'):69            cfg_filenames.append(os.path.join(os.path.expanduser('~'+USERNAME), ".config", "clearinghoused-testnet", "clearinghoused.conf"))70    if config in ('clearblockd', 'both'):71        if net in ('mainnet', 'both'):72            cfg_filenames.append(os.path.join(os.path.expanduser('~'+USERNAME), ".config", "clearblockd", "clearblockd.conf"))73        if net in ('testnet', 'both'):74            cfg_filenames.append(os.path.join(os.path.expanduser('~'+USERNAME), ".config", "clearblockd-testnet", "clearblockd.conf"))75    modify_config(param_re, content_to_add, cfg_filenames, replace_if_exists=replace_if_exists)76def ask_question(question, options, default_option):77    assert isinstance(options, (list, tuple))78    assert default_option in options79    answer = None80    while True:81        answer = input(question + ": ")82        answer = answer.lower()83        if answer and answer not in options:84            logging.error("Please enter one of: " + ', '.join(options))85        else:86            if answer == '': answer = default_option87            break88    return answer89def git_repo_clone(branch, repo_dir, repo_url, run_as_user, hash=None):90    if branch == 'AUTO':91        try:92            branch = subprocess.check_output("cd %s && git rev-parse --abbrev-ref HEAD" % (93                os.path.expanduser("~%s/%s" % (USERNAME, repo_dir))), shell=True).strip().decode('utf-8')94        except:95            raise Exception("Cannot get current get branch for %s." % repo_dir)96    logging.info("Checking out/updating %s:%s from git..." % (repo_dir, branch))97    if os.path.exists(os.path.expanduser("~%s/%s" % (USERNAME, repo_dir))):98        runcmd("cd ~%s/%s && git pull origin %s" % (USERNAME, repo_dir, branch))99    else:100        runcmd("git clone -b %s %s ~%s/%s" % (branch, repo_url, USERNAME, repo_dir))101    if hash:102        runcmd("cd ~%s/%s && git reset --hard %s" % (USERNAME, repo_dir, hash))103    runcmd("cd ~%s/%s && git config core.sharedRepository group && find ~%s/%s -type d -print0 | xargs -0 chmod g+s" % (104        USERNAME, repo_dir, USERNAME, repo_dir)) #to allow for group git actions105    runcmd("chown -R %s:%s ~%s/%s" % (USERNAME, USERNAME, USERNAME, repo_dir))106    runcmd("chmod -R u+rw,g+rw,o+r,o-w ~%s/%s" % (USERNAME, repo_dir)) #just in case107def do_prerun_checks():108    #make sure this is running on a supported OS109    if os.name != "posix" or platform.dist()[0] != "Ubuntu" or platform.architecture()[0] != '64bit':110        logging.error("Only 64bit Ubuntu Linux is supported at this time")111        sys.exit(1)112    ubuntu_release = platform.linux_distribution()[1]113    if ubuntu_release != "14.04":114        logging.error("Only Ubuntu 14.04 supported for Clearblock Federated Node install.")115        sys.exit(1)116    #script must be run as root117    if os.geteuid() != 0:118        logging.error("This script must be run as root (use 'sudo' to run)")119        sys.exit(1)120    if os.name == "posix" and "SUDO_USER" not in os.environ:121        logging.error("Please use `sudo` to run this script.")122        sys.exit(1)123def do_base_setup(run_as_user, branch, base_path, dist_path):124    """This creates the xch and xchd users and checks out the counterpartyd_build system from git"""125    #change time to UTC126    runcmd("ln -sf /usr/share/zoneinfo/UTC /etc/localtime")127    #install some necessary base deps128    runcmd("apt-get update")129    runcmd("apt-get -y install git-core software-properties-common python-software-properties build-essential ssl-cert ntp")130    runcmd("apt-get update")131    #node-gyp building for insight has ...issues out of the box on Ubuntu... use Chris Lea's nodejs build instead, which is newer132    runcmd("apt-get -y remove nodejs npm gyp")133    runcmd("add-apt-repository -y ppa:chris-lea/node.js")134    runcmd("apt-get update")135    runcmd("apt-get -y install nodejs") #includes npm136    #Create xcp user, under which the files will be stored, and who will own the files, etc137    try:138        pwd.getpwnam(USERNAME)139    except:140        logging.info("Creating user '%s' ..." % USERNAME)141        runcmd("adduser --system --disabled-password --shell /bin/false --group %s" % USERNAME)142    #Create xcpd user (to run counterpartyd, counterblockd, insight, bitcoind, nginx) if not already made143    try:144        pwd.getpwnam(DAEMON_USERNAME)145    except:146        logging.info("Creating user '%s' ..." % DAEMON_USERNAME)147        user_homedir = os.path.expanduser("~" + USERNAME)148        runcmd("adduser --system --disabled-password --shell /bin/false --ingroup nogroup --home %s %s" % (user_homedir, DAEMON_USERNAME))149    #add the run_as_user to the xcp group150    runcmd("adduser %s %s" % (run_as_user, USERNAME))151    #Check out counterpartyd-build repo under this user's home dir and use that for the build152    git_repo_clone(branch, "clearinghoused_build", REPO_COUNTERPARTYD_BUILD, run_as_user)153    #enhance fd limits for the xcpd user154    runcmd("cp -af %s/linux/other/xchd_security_limits.conf /etc/security/limits.d/" % dist_path)155def do_security_setup(run_as_user, branch, base_path, dist_path):156    """Some helpful security-related tasks, to tighten up the box"""157    #modify host.conf158    modify_config(r'^nospoof on$', 'nospoof on', '/etc/host.conf')159    #enable automatic security updates160    runcmd("apt-get -y install unattended-upgrades")161    runcmd('''bash -c "echo -e 'APT::Periodic::Update-Package-Lists "1";\nAPT::Periodic::Unattended-Upgrade "1";' > /etc/apt/apt.conf.d/20auto-upgrades" ''')162    runcmd("dpkg-reconfigure -fnoninteractive -plow unattended-upgrades")163    #sysctl164    runcmd("install -m 0644 -o root -g root -D %s/linux/other/sysctl_rules.conf /etc/sysctl.d/60-tweaks.conf" % dist_path)165    #set up fail2ban166    runcmd("apt-get -y install fail2ban")167    runcmd("install -m 0644 -o root -g root -D %s/linux/other/fail2ban.jail.conf /etc/fail2ban/jail.d/clearblock.conf" % dist_path)168    runcmd("service fail2ban restart")169    #set up psad170    runcmd("apt-get -y install psad")171    modify_config(r'^ENABLE_AUTO_IDS\s+?N;$', 'ENABLE_AUTO_IDS\tY;', '/etc/psad/psad.conf')172    modify_config(r'^ENABLE_AUTO_IDS_EMAILS\s+?Y;$', 'ENABLE_AUTO_IDS_EMAILS\tN;', '/etc/psad/psad.conf')173    for f in ['/etc/ufw/before.rules', '/etc/ufw/before6.rules']:174        modify_config(r'^# End required lines.*?# allow all on loopback$',175            '# End required lines\n\n#CUSTOM: for psad\n-A INPUT -j LOG\n-A FORWARD -j LOG\n\n# allow all on loopback',176            f, dotall=True, add_newline=False)177    runcmd("psad -R && psad --sig-update")178    runcmd("service ufw restart")179    runcmd("service psad restart")180    #set up chkrootkit, rkhunter181    runcmd("apt-get -y install rkhunter chkrootkit")182    runcmd('bash -c "rkhunter --update; exit 0"')183    runcmd("rkhunter --propupd")184    runcmd('bash -c "rkhunter --check --sk; exit 0"')185    runcmd("rkhunter --propupd")186    #logwatch187    runcmd("apt-get -y install logwatch libdate-manip-perl")188    #apparmor189    runcmd("apt-get -y install apparmor apparmor-profiles")190    #auditd191    #note that auditd will need a reboot to fully apply the rules, due to it operating in "immutable mode" by default192    runcmd("apt-get -y install auditd audispd-plugins")193    runcmd("install -m 0640 -o root -g root -D %s/linux/other/audit.rules /etc/audit/rules.d/clearblock.rules" % dist_path)194    runcmd("service auditd restart")195    #iwatch196    runcmd("apt-get -y install iwatch")197    modify_config(r'^START_DAEMON=.*?$', 'START_DAEMON=true', '/etc/default/iwatch')198    runcmd("install -m 0644 -o root -g root -D %s/linux/other/iwatch.xml /etc/iwatch/iwatch.xml" % dist_path)199    modify_config(r'guard email="root@localhost"', 'guard email="noreply@%s"' % socket.gethostname(), '/etc/iwatch/iwatch.xml')200    runcmd("service iwatch restart")201def do_bitcoind_setup(run_as_user, branch, base_path, dist_path, run_mode):202    """Installs and configures bitcoind"""203    user_homedir = os.path.expanduser("~" + USERNAME)204    bitcoind_rpc_password = pass_generator()205    bitcoind_rpc_password_testnet = pass_generator()206    #Install bitcoind207    BITCOIND_VER = "0.10.8"208    runcmd("rm -rf /tmp/viacoind.tar.gz /tmp/viacoin-%s-linux" % BITCOIND_VER)209    runcmd("wget -O /tmp/viacoind.tar.gz https://github.com/viacoin/viacoin/releases/download/v%s/viacoin-%s-linux64.tar.gz" % (BITCOIND_VER, BITCOIND_VER))210    runcmd("tar -C /tmp -zxvf /tmp/viacoind.tar.gz")211    runcmd("cp -af /tmp/viacoin-%s/bin/viacoind /usr/local/bin" % BITCOIND_VER)212    runcmd("cp -af /tmp/viacoin-%s/bin/viacoin-cli /usr/local/bin" % BITCOIND_VER)213    runcmd("rm -rf /tmp/viacoind.tar.gz /tmp/viacoin-%s-linux" % BITCOIND_VER)214    #Do basic inital bitcoin config (for both testnet and mainnet)215    runcmd("mkdir -p ~%s/.viacoin ~%s/.viacoin-testnet" % (USERNAME, USERNAME))216    if not os.path.exists(os.path.join(user_homedir, '.viacoin', 'viacoin.conf')):217        runcmd(r"""bash -c 'echo -e "rpcuser=rpc\nrpcpassword=%s\nserver=1\ndaemon=1\ntxindex=1\naddrindex=1\nrpcthreads=1000\nrpctimeout=300" > ~%s/.viacoin/viacoin.conf'""" % (218            bitcoind_rpc_password, USERNAME))219    else: #grab the existing RPC password220        bitcoind_rpc_password = subprocess.check_output(221            r"""bash -c "cat ~%s/.viacoin/viacoin.conf | sed -n 's/.*rpcpassword=\([^ \n]*\).*/\1/p'" """ % USERNAME, shell=True).strip().decode('utf-8')222    if not os.path.exists(os.path.join(user_homedir, '.viacoin-testnet', 'viacoin.conf')):223        runcmd(r"""bash -c 'echo -e "rpcuser=rpc\nrpcpassword=%s\nserver=1\ndaemon=1\ntxindex=1\naddrindex=1\ntestnet=1\nrpcthreads=1000\nrpctimeout=300" > ~%s/.viacoin-testnet/viacoin.conf'""" % (224            bitcoind_rpc_password_testnet, USERNAME))225    else:226        bitcoind_rpc_password_testnet = subprocess.check_output(227            r"""bash -c "cat ~%s/.viacoin-testnet/viacoin.conf | sed -n 's/.*rpcpassword=\([^ \n]*\).*/\1/p'" """228            % USERNAME, shell=True).strip().decode('utf-8')229    #Set up bitcoind startup scripts (will be disabled later from autostarting on system startup if necessary)230    runcmd("rm -f /etc/init/viacoin.conf /etc/init/viacoin-testnet.conf")231    runcmd("cp -af %s/linux/init/viacoind.conf.template /etc/init/viacoind.conf" % dist_path)232    runcmd("sed -ri \"s/\!RUN_AS_USER\!/%s/g\" /etc/init/viacoind.conf" % DAEMON_USERNAME)233    runcmd("sed -ri \"s/\!USER_HOMEDIR\!/%s/g\" /etc/init/viacoind.conf" % user_homedir.replace('/', '\/'))234    runcmd("cp -af %s/linux/init/viacoind-testnet.conf.template /etc/init/viacoind-testnet.conf" % dist_path)235    runcmd("sed -ri \"s/\!RUN_AS_USER\!/%s/g\" /etc/init/viacoind-testnet.conf" % DAEMON_USERNAME)236    runcmd("sed -ri \"s/\!USER_HOMEDIR\!/%s/g\" /etc/init/viacoind-testnet.conf" % user_homedir.replace('/', '\/'))237    #install logrotate file238    runcmd("cp -af %s/linux/logrotate/viacoind /etc/logrotate.d/viacoind" % dist_path)239    runcmd("sed -ri \"s/\!USER_HOMEDIR\!/%s/g\" /etc/logrotate.d/viacoind" % user_homedir.replace('/', '\/'))240    #disable upstart scripts from autostarting on system boot if necessary241    if run_mode == 't': #disable mainnet daemons from autostarting242        runcmd(r"""bash -c "echo 'manual' >> /etc/init/viacoind.override" """)243    else:244        runcmd("rm -f /etc/init/viacoind.override")245    if run_mode == 'm': #disable testnet daemons from autostarting246        runcmd(r"""bash -c "echo 'manual' >> /etc/init/viacoind-testnet.override" """)247    else:248        runcmd("rm -f /etc/init/viacoind-testnet.override")249    runcmd("chown -R %s:%s ~%s/.viacoin ~%s/.viacoin-testnet" % (DAEMON_USERNAME, USERNAME, USERNAME, USERNAME))250    return bitcoind_rpc_password, bitcoind_rpc_password_testnet251def do_counterparty_setup(role, run_as_user, branch, base_path, dist_path, run_mode, bitcoind_rpc_password,252bitcoind_rpc_password_testnet, clearinghoused_public, clearwallet_support_email):253    """Installs and configures counterpartyd and counterblockd"""254    user_homedir = os.path.expanduser("~" + USERNAME)255    counterpartyd_rpc_password = '1234' if role == 'clearinghoused_only' and clearinghoused_public == 'y' else pass_generator()256    counterpartyd_rpc_password_testnet = '1234' if role == 'clearinghoused_only' and clearinghoused_public == 'y' else pass_generator()257    #Run setup.py (as the XCP user, who runs it sudoed) to install and set up counterpartyd, counterblockd258    # as -y is specified, this will auto install counterblockd full node (mongo and redis) as well as setting259    # counterpartyd/counterblockd to start up at startup for both mainnet and testnet (we will override this as necessary260    # based on run_mode later in this function)261    runcmd("~%s/clearinghoused_build/setup.py -y %s --with-testnet --for-user=%s" % (262        USERNAME, '--with-clearblockd' if role != 'clearinghoused_only' else '', USERNAME))263    runcmd("cd ~%s/clearinghoused_build && git config core.sharedRepository group && find ~%s/clearinghoused_build -type d -print0 | xargs -0 chmod g+s" % (264        USERNAME, USERNAME)) #to allow for group git actions265    runcmd("chown -R %s:%s ~%s/clearinghoused_build" % (USERNAME, USERNAME, USERNAME)) #just in case266    runcmd("chmod -R u+rw,g+rw,o+r,o-w ~%s/clearinghoused_build" % USERNAME) #just in case267    #now change the counterpartyd directories to be owned by the xcpd user (and the xcp group),268    # so that the xcpd account can write to the database, saved image files (counterblockd), log files, etc269    runcmd("mkdir -p ~%s/.config/clearinghoused ~%s/.config/clearinghoused-testnet" % (USERNAME, USERNAME))270    runcmd("chown -R %s:%s ~%s/.config/clearinghoused ~%s/.config/clearinghoused-testnet" % (271        DAEMON_USERNAME, USERNAME, USERNAME, USERNAME))272    runcmd("sed -ri \"s/USER=%s/USER=%s/g\" /etc/init/clearinghoused.conf /etc/init/clearinghoused-testnet.conf" % (273        USERNAME, DAEMON_USERNAME))274    #modify the default stored bitcoind passwords in counterpartyd.conf275    modify_cp_config(r'^(backend|bitcoind)\-rpc\-password=.*?$',276        'backend-rpc-password=%s' % bitcoind_rpc_password, config='clearinghoused', net='mainnet')277    modify_cp_config(r'^(backend|bitcoind)\-rpc\-password=.*?$',278        'backend-rpc-password=%s' % bitcoind_rpc_password_testnet, config='clearinghoused', net='testnet')279    #modify the counterpartyd API rpc password in counterpartyd.conf280    modify_cp_config(r'^rpc\-password=.*?$', 'rpc-password=%s' % counterpartyd_rpc_password,281        config='clearinghoused', net='mainnet')282    modify_cp_config(r'^rpc\-password=.*?$', 'rpc-password=%s' % counterpartyd_rpc_password_testnet,283        config='clearinghoused', net='testnet')284    if role == 'clearinghoused_only' and clearinghoused_public == 'y':285        modify_cp_config(r'^rpc\-host=.*?$', 'rpc-host=0.0.0.0', config='clearinghoused')286    #disable upstart scripts from autostarting on system boot if necessary287    if run_mode == 't': #disable mainnet daemons from autostarting288        runcmd(r"""bash -c "echo 'manual' >> /etc/init/clearinghoused.override" """)289    else:290        runcmd("rm -f /etc/init/clearinghoused.override")291    if run_mode == 'm': #disable testnet daemons from autostarting292        runcmd(r"""bash -c "echo 'manual' >> /etc/init/clearinghoused-testnet.override" """)293    else:294        runcmd("rm -f /etc/init/clearinghoused-testnet.override")295    if role != 'clearinghoused_only':296        #now change the counterblockd directories to be owned by the xcpd user (and the xcp group),297        runcmd("mkdir -p ~%s/.config/clearblockd ~%s/.config/clearblockd-testnet" % (USERNAME, USERNAME))298        runcmd("chown -R %s:%s ~%s/.config/clearblockd ~%s/.config/clearblockd-testnet" % (299            DAEMON_USERNAME, USERNAME, USERNAME, USERNAME))300        runcmd("sed -ri \"s/USER=%s/USER=%s/g\" /etc/init/clearblockd.conf /etc/init/clearblockd-testnet.conf" % (301            USERNAME, DAEMON_USERNAME))302        #modify the default stored bitcoind passwords in counterblockd.conf303        modify_cp_config(r'^(backend|bitcoind)\-rpc\-password=.*?$',304            'backend-rpc-password=%s' % bitcoind_rpc_password, config='clearblockd', net='mainnet')305        modify_cp_config(r'^(backend|bitcoind)\-rpc\-password=.*?$',306            'backend-rpc-password=%s' % bitcoind_rpc_password_testnet, config='clearblockd', net='testnet')307        #modify the counterpartyd API rpc password in counterblockd.conf308        modify_cp_config(r'^clearinghoused\-rpc\-password=.*?$',309            'clearinghoused-rpc-password=%s' % counterpartyd_rpc_password, config='clearblockd', net='mainnet')310        modify_cp_config(r'^clearinghoused\-rpc\-password=.*?$',311            'clearinghoused-rpc-password=%s' % counterpartyd_rpc_password_testnet, config='clearblockd', net='testnet')312        #role-specific counterblockd.conf values313        modify_cp_config(r'^auto\-btc\-escrow\-enable=.*?$', 'auto-btc-escrow-enable=%s' %314            '1' if role == 'btcpayescrow' else '0', config='clearblockd')315        modify_cp_config(r'^armory\-utxsvr\-enable=.*?$', 'armory-utxsvr-enable=%s' %316            '1' if role == 'counterwallet' else '0', config='clearblockd')317        if role == 'counterwallet':318            modify_cp_config(r'^support\-email=.*?$', 'support-email=%s' % clearwallet_support_email,319                config='clearblockd') #may be blank string320        #disable upstart scripts from autostarting on system boot if necessary321        if run_mode == 't': #disable mainnet daemons from autostarting322            runcmd(r"""bash -c "echo 'manual' >> /etc/init/clearblockd.override" """)323        else:324            runcmd("rm -f /etc/init/clearblockd.override")325        if run_mode == 'm': #disable testnet daemons from autostarting326            runcmd(r"""bash -c "echo 'manual' >> /etc/init/clearblockd-testnet.override" """)327        else:328            runcmd("rm -f /etc/init/clearblockd-testnet.override")329def do_blockchain_service_setup(run_as_user, base_path, dist_path, run_mode, blockchain_service, bitcoin_rpc_password, bitcoin_rpc_password_testnet):330    def do_insight_setup():331        """This installs and configures insight"""332        assert blockchain_service333        user_homedir = os.path.expanduser("~" + USERNAME)334        gypdir = None335        try:336            import gyp337            gypdir = os.path.dirname(gyp.__file__)338        except:339            pass340        else:341            runcmd("mv %s %s_bkup" % (gypdir, gypdir))342            #^ fix for https://github.com/TooTallNate/node-gyp/issues/363343        git_repo_clone("master", "insight-api", "https://github.com/viacoin/insight-api.git",344            run_as_user, hash="master") #insight 0.2.7345        runcmd("rm -rf ~%s/insight-api/node-modules && cd ~%s/insight-api && npm install" % (USERNAME, USERNAME))346        #Set up insight startup scripts (will be disabled later from autostarting on system startup if necessary)347        runcmd("rm -f /etc/init/insight.conf /etc/init/insight-testnet.conf")348        runcmd("cp -af %s/linux/init/insight.conf.template /etc/init/insight.conf" % dist_path)349        runcmd("sed -ri \"s/\!RUN_AS_USER\!/%s/g\" /etc/init/insight.conf" % DAEMON_USERNAME)350        runcmd("sed -ri \"s/\!USER_HOMEDIR\!/%s/g\" /etc/init/insight.conf" % user_homedir.replace('/', '\/'))351        runcmd("cp -af %s/linux/init/insight-testnet.conf.template /etc/init/insight-testnet.conf" % dist_path)352        runcmd("sed -ri \"s/\!RUN_AS_USER\!/%s/g\" /etc/init/insight-testnet.conf" % DAEMON_USERNAME)353        runcmd("sed -ri \"s/\!USER_HOMEDIR\!/%s/g\" /etc/init/insight-testnet.conf" % user_homedir.replace('/', '\/'))354        #install logrotate file355        runcmd("cp -af %s/linux/logrotate/insight /etc/logrotate.d/insight" % dist_path)356        runcmd("sed -ri \"s/\!RUN_AS_USER\!/%s/g\" /etc/logrotate.d/insight" % DAEMON_USERNAME)357        runcmd("sed -ri \"s/\!USER_HOMEDIR\!/%s/g\" /etc/logrotate.d/insight" % user_homedir.replace('/', '\/'))358        runcmd("mkdir -p ~%s/insight-api/db" % USERNAME)359        runcmd("chown -R %s:%s ~%s/insight-api" % (USERNAME, USERNAME, USERNAME))360        runcmd("chown -R %s:%s ~%s/insight-api/db" % (DAEMON_USERNAME, USERNAME, USERNAME))361        modify_cp_config(r'^blockchain\-service\-name=.*?$', 'blockchain-service-name=insight', config='both')362    def do_blockr_setup():363        modify_cp_config(r'^blockchain\-service\-name=.*?$', 'blockchain-service-name=blockr', config='both')364    def do_addrindex_setup():365        modify_cp_config(r'^insight\-enable=.*?$', 'insight-enable=0', config='both')366        modify_cp_config(r'^blockchain\-service\-name=.*?$', 'blockchain-service-name=addrindex', config='both')367        modify_cp_config(r'^blockchain\-service\-connect=.*?$', 'blockchain-service-connect=http://rpc:%s@localhost:5222' % bitcoin_rpc_password, config='clearblockd', net='mainnet')368        modify_cp_config(r'^blockchain\-service\-connect=.*?$', 'blockchain-service-connect=http://rpc:%s@localhost:25222' % bitcoin_rpc_password_testnet, config='clearblockd', net='testnet')369    #disable upstart scripts from autostarting on system boot if necessary370    if blockchain_service == 'i':371        do_insight_setup()372        if run_mode == 't': #disable mainnet daemons from autostarting373            runcmd(r"""bash -c "echo 'manual' >> /etc/init/insight.override" """)374        else:375            runcmd("rm -f /etc/init/insight.override")376        if run_mode == 'm': #disable testnet daemons from autostarting377            runcmd(r"""bash -c "echo 'manual' >> /etc/init/insight-testnet.override" """)378        else:379            runcmd("rm -f /etc/init/insight-testnet.override")380    if blockchain_service == 'a':381        runcmd("rm -f /etc/init/insight.override /etc/init/insight-testnet.override")382        do_addrindex_setup()383    else: #insight not being used as blockchain service384        runcmd("rm -f /etc/init/insight.override /etc/init/insight-testnet.override")385        #^ so insight doesn't start if it was in use before386        do_blockr_setup()387def do_nginx_setup(run_as_user, base_path, dist_path):388    #Build and install nginx (openresty) on Ubuntu389    #Most of these build commands from http://brian.akins.org/blog/2013/03/19/building-openresty-on-ubuntu/390    OPENRESTY_VER = "1.7.0.1"391    #install deps392    runcmd("apt-get -y install make ruby1.9.1 ruby1.9.1-dev git-core libpcre3-dev libxslt1-dev libgd2-xpm-dev libgeoip-dev unzip zip build-essential libssl-dev")393    runcmd("gem install fpm")394    #grab openresty and compile395    runcmd("rm -rf /tmp/openresty /tmp/ngx_openresty-* /tmp/nginx-openresty.tar.gz /tmp/nginx-openresty*.deb")396    runcmd('''wget -O /tmp/nginx-openresty.tar.gz http://openresty.org/download/ngx_openresty-%s.tar.gz''' % OPENRESTY_VER)397    runcmd("tar -C /tmp -zxvf /tmp/nginx-openresty.tar.gz")398    runcmd('''cd /tmp/ngx_openresty-%s && ./configure \399--with-luajit \400--sbin-path=/usr/sbin/nginx \401--conf-path=/etc/nginx/nginx.conf \402--error-log-path=/var/log/nginx/error.log \403--http-client-body-temp-path=/var/lib/nginx/body \404--http-fastcgi-temp-path=/var/lib/nginx/fastcgi \405--http-log-path=/var/log/nginx/access.log \406--http-proxy-temp-path=/var/lib/nginx/proxy \407--http-scgi-temp-path=/var/lib/nginx/scgi \408--http-uwsgi-temp-path=/var/lib/nginx/uwsgi \409--lock-path=/var/lock/nginx.lock \410--pid-path=/var/run/nginx.pid \411--with-http_geoip_module \412--with-http_gzip_static_module \413--with-http_realip_module \414--with-http_ssl_module \415--with-http_sub_module \416--with-http_xslt_module \417--with-ipv6 \418--with-sha1=/usr/include/openssl \419--with-md5=/usr/include/openssl \420--with-http_stub_status_module \421--with-http_secure_link_module \422--with-http_sub_module && make''' % OPENRESTY_VER)423    #set up the build environment424    runcmd('''cd /tmp/ngx_openresty-%s && make install DESTDIR=/tmp/openresty \425&& mkdir -p /tmp/openresty/var/lib/nginx \426&& install -m 0755 -D %s/linux/nginx/nginx.init /tmp/openresty/etc/init.d/nginx \427&& install -m 0755 -D %s/linux/nginx/nginx.conf /tmp/openresty/etc/nginx/nginx.conf \428&& install -m 0755 -D %s/linux/nginx/clearblock.conf /tmp/openresty/etc/nginx/sites-enabled/clearblock.conf \429&& install -m 0755 -D %s/linux/nginx/clearblock_api.inc /tmp/openresty/etc/nginx/sites-enabled/clearblock_api.inc \430&& install -m 0755 -D %s/linux/nginx/clearblock_api_cache.inc /tmp/openresty/etc/nginx/sites-enabled/clearblock_api_cache.inc \431&& install -m 0755 -D %s/linux/nginx/clearblock_socketio.inc /tmp/openresty/etc/nginx/sites-enabled/clearblock_socketio.inc \432&& install -m 0755 -D %s/linux/logrotate/nginx /tmp/openresty/etc/logrotate.d/nginx''' % (433    OPENRESTY_VER, dist_path, dist_path, dist_path, dist_path, dist_path, dist_path, dist_path))434    #package it up using fpm435    runcmd('''cd /tmp && fpm -s dir -t deb -n nginx-openresty -v %s --iteration 1 -C /tmp/openresty \436--description "openresty %s" \437--conflicts nginx \438--conflicts nginx-common \439-d libxslt1.1 \440-d libgeoip1 \441-d geoip-database \442-d libpcre3 \443--config-files /etc/nginx/nginx.conf \444--config-files /etc/nginx/sites-enabled/clearblock.conf \445--config-files /etc/nginx/fastcgi.conf.default \446--config-files /etc/nginx/win-utf \447--config-files /etc/nginx/fastcgi_params \448--config-files /etc/nginx/nginx.conf \449--config-files /etc/nginx/koi-win \450--config-files /etc/nginx/nginx.conf.default \451--config-files /etc/nginx/mime.types.default \452--config-files /etc/nginx/koi-utf \453--config-files /etc/nginx/uwsgi_params \454--config-files /etc/nginx/uwsgi_params.default \455--config-files /etc/nginx/fastcgi_params.default \456--config-files /etc/nginx/mime.types \457--config-files /etc/nginx/scgi_params.default \458--config-files /etc/nginx/scgi_params \459--config-files /etc/nginx/fastcgi.conf \460etc usr var''' % (OPENRESTY_VER, OPENRESTY_VER))461    #now install the .deb package that was created (along with its deps)462    runcmd("apt-get -y install libxslt1.1 libgeoip1 geoip-database libpcre3")463    runcmd("dpkg -i /tmp/nginx-openresty_%s-1_amd64.deb" % OPENRESTY_VER)464    #remove any .dpkg-old or .dpkg-dist files that might have been installed out of the nginx config dir465    runcmd("rm -f /etc/nginx/sites-enabled/*.dpkg-old /etc/nginx/sites-enabled/*.dpkg-dist")466    #clean up after ourselves467    runcmd("rm -rf /tmp/openresty /tmp/ngx_openresty-* /tmp/nginx-openresty.tar.gz /tmp/nginx-openresty*.deb")468    runcmd("update-rc.d nginx defaults")469def do_armory_utxsvr_setup(run_as_user, base_path, dist_path, run_mode, enable=True):470    user_homedir = os.path.expanduser("~" + USERNAME)471    runcmd("apt-get -y install xvfb python-qt4 python-twisted python-psutil xdg-utils")472    runcmd("rm -f /tmp/armory.deb")473    runcmd("wget -O /tmp/armory.deb https://s3.amazonaws.com/bitcoinarmory-releases/armory_0.92.1_ubuntu-64bit.deb")474    runcmd("mkdir -p /usr/share/desktop-directories/") #bug fix (see http://askubuntu.com/a/406015)475    runcmd("dpkg -i /tmp/armory.deb")476    runcmd("rm -f /tmp/armory.deb")477    runcmd("mkdir -p ~%s/.armory" % USERNAME)478    runcmd("chown -R %s:%s ~%s/.armory" % (DAEMON_USERNAME, USERNAME, USERNAME))479    runcmd("sudo ln -sf ~%s/.viacoin-testnet/testnet3 ~%s/.viacoin/" % (USERNAME, USERNAME))480    #^ ghetto hack, as armory has hardcoded dir settings in certain place481    #make a short script to launch armory_utxsvr482    f = open("/usr/local/bin/armory_utxsvr", 'w')483    f.write("#!/bin/sh\n%s/run.py armory_utxsvr \"$@\"" % base_path)484    f.close()485    runcmd("chmod +x /usr/local/bin/armory_utxsvr")486    #Set up upstart scripts (will be disabled later from autostarting on system startup if necessary)487    if enable:488        runcmd("rm -f /etc/init/armory_utxsvr.conf /etc/init/armory_utxsvr-testnet.conf")489        runcmd("cp -af %s/linux/init/armory_utxsvr.conf.template /etc/init/armory_utxsvr.conf" % dist_path)490        runcmd("sed -ri \"s/\!RUN_AS_USER\!/%s/g\" /etc/init/armory_utxsvr.conf" % DAEMON_USERNAME)491        runcmd("sed -ri \"s/\!USER_HOMEDIR\!/%s/g\" /etc/init/armory_utxsvr.conf" % user_homedir.replace('/', '\/'))492        runcmd("cp -af %s/linux/init/armory_utxsvr-testnet.conf.template /etc/init/armory_utxsvr-testnet.conf" % dist_path)493        runcmd("sed -ri \"s/\!RUN_AS_USER\!/%s/g\" /etc/init/armory_utxsvr-testnet.conf" % DAEMON_USERNAME)494        runcmd("sed -ri \"s/\!USER_HOMEDIR\!/%s/g\" /etc/init/armory_utxsvr-testnet.conf" % user_homedir.replace('/', '\/'))495    else: #disable496        runcmd("rm -f /etc/init/armory_utxsvr.conf /etc/init/armory_utxsvr-testnet.conf")497    #disable upstart scripts from autostarting on system boot if necessary498    if run_mode == 't': #disable mainnet daemons from autostarting499        runcmd(r"""bash -c "echo 'manual' >> /etc/init/armory_utxsvr.override" """)500    else:501        runcmd("rm -f /etc/init/armory_utxsvr.override")502    if run_mode == 'm': #disable testnet daemons from autostarting503        runcmd(r"""bash -c "echo 'manual' >> /etc/init/armory_utxsvr-testnet.override" """)504    else:505        runcmd("rm -f /etc/init/armory_utxsvr-testnet.override")506def do_counterwallet_setup(run_as_user, branch, updateOnly=False):507    #check out counterwallet from git508    git_repo_clone(branch, "clearwallet", REPO_COUNTERWALLET, run_as_user)509    if not updateOnly:510        runcmd("npm install -g grunt-cli bower")511    runcmd("cd ~%s/clearwallet/src && bower --allow-root --config.interactive=false install" % USERNAME)512    runcmd("cd ~%s/clearwallet && npm install" % USERNAME)513    runcmd("cd ~%s/clearwallet && grunt build" % USERNAME) #will generate the minified site514    runcmd("chown -R %s:%s ~%s/clearwallet" % (USERNAME, USERNAME, USERNAME)) #just in case515    runcmd("chmod -R u+rw,g+rw,o+r,o-w ~%s/clearwallet" % USERNAME) #just in case516def do_newrelic_setup(run_as_user, base_path, dist_path, run_mode):517    ##518    ## NOTE: NEW RELIC DAEMON CAUSES ISSUES WITH COUNTERBLOCKD (SOCKETS STICKING IN CLOSE_WAIT STATE)519    ##  -- DO NOT USE IT FOR COUNTERBLOCKD MONITORING520    ##521    NR_PREFS_LICENSE_KEY_PATH = "/etc/newrelic/LICENSE_KEY"522    NR_PREFS_HOSTNAME_PATH = "/etc/newrelic/HOSTNAME"523    runcmd("mkdir -p /etc/newrelic /var/log/newrelic /var/run/newrelic")524    runcmd("chown %s:%s /etc/newrelic /var/log/newrelic /var/run/newrelic" % (DAEMON_USERNAME, USERNAME))525    #try to find existing license key526    nr_license_key = None527    if os.path.exists(NR_PREFS_LICENSE_KEY_PATH):528        nr_license_key = open(NR_PREFS_LICENSE_KEY_PATH).read().strip()529    else:530        while True:531            nr_license_key = input("Enter New Relic license key (or blank to not setup New Relic): ") #gather license key532            nr_license_key = nr_license_key.strip()533            if not nr_license_key:534                return #skipping new relic535            nr_license_key_confirm = ask_question("You entererd '%s', is that right? (Y/n): " % nr_license_key, ('y', 'n'), 'y')536            if nr_license_key_confirm == 'y': break537        open(NR_PREFS_LICENSE_KEY_PATH, 'w').write(nr_license_key)538    assert nr_license_key539    logging.info("NewRelic license key: %s" % nr_license_key)540    #try to find existing app prefix541    nr_hostname = None542    if os.path.exists(NR_PREFS_HOSTNAME_PATH):543        nr_hostname = open(NR_PREFS_HOSTNAME_PATH).read().strip()544    else:545        while True:546            nr_hostname = input("Enter newrelic hostname/app prefix (e.g. 'cw01'): ") #gather app prefix547            nr_hostname = nr_hostname.strip()548            nr_hostname_confirm = ask_question("You entererd '%s', is that right? (Y/n): " % nr_hostname, ('y', 'n'), 'y')549            if nr_hostname_confirm == 'y': break550        open(NR_PREFS_HOSTNAME_PATH, 'w').write(nr_hostname)551    assert nr_hostname552    logging.info("NewRelic hostname/app prefix: %s" % nr_hostname)553    #install some deps...554    runcmd("sudo apt-get -y install libyaml-dev")555    #install/setup python agent for both counterpartyd and counterblockd556    #counterpartyd557    runcmd("%s/env/bin/pip install newrelic" % base_path)558    runcmd("cp -af %s/linux/newrelic/nr_clearinghoused.ini.template /etc/newrelic/nr_clearinghoused.ini" % dist_path)559    runcmd("sed -ri \"s/\!LICENSE_KEY\!/%s/g\" /etc/newrelic/nr_clearinghoused.ini" % nr_license_key)560    runcmd("sed -ri \"s/\!HOSTNAME\!/%s/g\" /etc/newrelic/nr_clearinghoused.ini" % nr_hostname)561    #counterblockd562    runcmd("%s/env.clearblockd/bin/pip install newrelic" % base_path)563    runcmd("cp -af %s/linux/newrelic/nr_clearblockd.ini.template /etc/newrelic/nr_clearblockd.ini" % dist_path)564    runcmd("sed -ri \"s/\!LICENSE_KEY\!/%s/g\" /etc/newrelic/nr_clearblockd.ini" % nr_license_key)565    runcmd("sed -ri \"s/\!HOSTNAME\!/%s/g\" /etc/newrelic/nr_clearblockd.ini" % nr_hostname)566    #install init scripts (overwrite the existing ones for now at least)567    runcmd("cp -af %s/linux/newrelic/init/nr-clearinghoused.conf /etc/init/clearinghoused.conf" % dist_path) #overwrite568    #runcmd("cp -af %s/linux/newrelic/init/nr-clearblockd.conf /etc/init/clearblockd.conf" % dist_path) #overwrite569    runcmd("cp -af %s/linux/newrelic/init/nr-clearinghoused-testnet.conf /etc/init/clearinghoused-testnet.conf" % dist_path) #overwrite570    #runcmd("cp -af %s/linux/newrelic/init/nr-clearblockd-testnet.conf /etc/init/clearblockd-testnet.conf" % dist_path) #overwrite571    #upstart enablement (overrides) should be fine as established in do_counterparty_setup...572    #install/setup server agent573    runcmd("add-apt-repository \"deb http://apt.newrelic.com/debian/ newrelic non-free\"")574    runcmd("wget -O- https://download.newrelic.com/548C16BF.gpg | apt-key add -")575    runcmd("apt-get update")576    runcmd("apt-get -y install newrelic-sysmond")577    runcmd("cp -af %s/linux/newrelic/nrsysmond.cfg.template /etc/newrelic/nrsysmond.cfg" % dist_path)578    runcmd("sed -ri \"s/\!LICENSE_KEY\!/%s/g\" /etc/newrelic/nrsysmond.cfg" % nr_license_key)579    runcmd("sed -ri \"s/\!HOSTNAME\!/%s/g\" /etc/newrelic/nrsysmond.cfg" % nr_hostname)580    runcmd("/etc/init.d/newrelic-sysmond restart")581    #install/setup meetme agent (mongo, redis)582    runcmd("pip install newrelic_plugin_agent pymongo")583    runcmd("cp -af %s/linux/newrelic/newrelic-plugin-agent.cfg.template /etc/newrelic/newrelic-plugin-agent.cfg" % dist_path)584    runcmd("sed -ri \"s/\!LICENSE_KEY\!/%s/g\" /etc/newrelic/newrelic-plugin-agent.cfg" % nr_license_key)585    runcmd("sed -ri \"s/\!HOSTNAME\!/%s/g\" /etc/newrelic/newrelic-plugin-agent.cfg" % nr_hostname)586    runcmd("ln -sf %s/linux/newrelic/init/newrelic-plugin-agent /etc/init.d/newrelic-plugin-agent" % dist_path)587    runcmd("update-rc.d newrelic-plugin-agent defaults")588    runcmd("/etc/init.d/newrelic-plugin-agent restart")589    #install/setup nginx agent590    runcmd("sudo apt-get -y install ruby ruby-bundler")591    runcmd("rm -rf /tmp/newrelic_nginx_agent.tar.gz /opt/newrelic_nginx_agent")592    #runcmd("wget -O /tmp/newrelic_nginx_agent.tar.gz http://nginx.com/download/newrelic/newrelic_nginx_agent.tar.gz")593    #runcmd("tar -C /opt -zxvf /tmp/newrelic_nginx_agent.tar.gz")594    runcmd("git clone https://github.com/crowdlab-uk/newrelic-nginx-agent.git /opt/newrelic_nginx_agent")595    runcmd("cd /opt/newrelic_nginx_agent && bundle install")596    runcmd("cp -af %s/linux/newrelic/newrelic_nginx_plugin.yml.template /opt/newrelic_nginx_agent/config/newrelic_plugin.yml" % dist_path)597    runcmd("sed -ri \"s/\!LICENSE_KEY\!/%s/g\" /opt/newrelic_nginx_agent/config/newrelic_plugin.yml" % nr_license_key)598    runcmd("sed -ri \"s/\!HOSTNAME\!/%s/g\" /opt/newrelic_nginx_agent/config/newrelic_plugin.yml" % nr_hostname)599    runcmd("ln -sf /opt/newrelic_nginx_agent/newrelic_nginx_agent.daemon /etc/init.d/newrelic_nginx_agent")600    runcmd("update-rc.d newrelic_nginx_agent defaults")601    runcmd("/etc/init.d/newrelic_nginx_agent restart")602def command_services(command, prompt=False):603    assert command in ("stop", "restart")604    if prompt:605        confirmation = ask_question("%s services? (y/N)" % command.capitalize(), ('y', 'n',), 'n')606        if confirmation == 'n':607            return False608    #restart/shutdown services if they may be running on the box609    if os.path.exists("/etc/init.d/iwatch"):610        runcmd("service iwatch %s" % command, abort_on_failure=False)611    if os.path.exists("/etc/init/clearinghoused.conf"):612        logging.warn("STOPPING SERVICES" if command == 'stop' else "RESTARTING SERVICES")613        runcmd("service viacoind %s" % command, abort_on_failure=False)614        runcmd("service viacoind-testnet %s" % command, abort_on_failure=False)615        if os.path.exists("/etc/init/insight.conf"):616            if command == "restart":617                logging.info("Waiting 45 seconds before starting insight, to allow viacoind to fully initialize...")618                time.sleep(45)619            runcmd("service insight %s" % command, abort_on_failure=False)620            runcmd("service insight-testnet %s" % command, abort_on_failure=False)621        runcmd("service clearinghoused %s" % command, abort_on_failure=False)622        runcmd("service clearinghoused-testnet %s" % command, abort_on_failure=False)623        runcmd("service clearblockd %s" % command, abort_on_failure=False)624        runcmd("service clearblockd-testnet %s" % command, abort_on_failure=False)625        if os.path.exists("/etc/init/armory_utxsvr.conf"):626            runcmd("service armory_utxsvr %s" % command, abort_on_failure=False)627            runcmd("service armory_utxsvr-testnet %s" % command, abort_on_failure=False)628        return True629QUESTION_FLAGS = collections.OrderedDict({630    "op": ('u', 'r'),631    "role": ('clearwallet', 'vendingmachine', 'blockexplorer', 'clearinghoused_only', 'btcpayescrow'),632    "branch": ('master', 'develop'),633    "run_mode": ('t', 'm', 'b'),634    "blockchain_service": ('b', 'i', 'a'),635    "security_hardening": ('y', 'n'),636    "clearinghoused_public": ('y', 'n'),637    "clearwallet_support_email": None638})639def gather_build_questions(answered_questions):640    if 'role' not in answered_questions:641        role = ask_question("Enter the number for the role you want to build:\n"642                + "\t1: Clearwallet server\n\t2: Vending machine\n\t3: Blockexplorer server\n"643                + "\t4: clearinghoused-only\n\t5: VIAPay Escrow Server\n"644                + "Your choice",645            ('1', '2', '3', '4', '5'), '1')646        if role == '1':647            role = 'clearwallet'648            role_desc = "Clearwallet server"649        elif role == '2':650            role = 'vendingmachine'651            role_desc = "Vending machine/gateway"652        elif role == '3':653            role = 'blockexplorer'654            role_desc = "Block explorer server"655        elif role == '4':656            role = 'clearinghoused_only'657            role_desc = "Counterpartyd server"658        elif role == '5':659            role = 'btcpayescrow'660            role_desc = "BTCPay escrow server"661        print("\tBuilding a %s" % role_desc)662        answered_questions['role'] = role663    assert answered_questions['role'] in QUESTION_FLAGS['role']664    if answered_questions['role'] in ('vendingmachine', 'blockexplorer'):665        raise NotImplementedError("This role not implemented yet...")666    if 'branch' not in answered_questions:667        branch = ask_question("Build from branch (M)aster or (d)evelop? (M/d)", ('m', 'd'), 'm')668        if branch == 'm': branch = 'master'669        elif branch == 'd': branch = 'develop'670        print("\tWorking with branch: %s" % branch)671        answered_questions['branch'] = branch672    assert answered_questions['branch'] in QUESTION_FLAGS['branch']673    if 'run_mode' not in answered_questions:674        answered_questions['run_mode'] = ask_question("Run as (t)estnet node, (m)ainnet node, or (B)oth? (t/m/B)", ('t', 'm', 'b'), 'b')675        print("\tSetting up to run on %s" % ('testnet' if answered_questions['run_mode'].lower() == 't'676            else ('mainnet' if answered_questions['run_mode'].lower() == 'm' else 'testnet and mainnet')))677    assert answered_questions['run_mode'] in QUESTION_FLAGS['run_mode']678    if 'blockchain_service' not in answered_questions:679        answered_questions['blockchain_service'] = ask_question("Blockchain services, use (B)lockr.io (remote), (i)nsight (local), (a)ddrindex? (b/i/A)", ('b', 'i', 'a'), 'a')680        if answered_questions['blockchain_service'] == 'b':681            answer = 'blockr.io'682        elif answered_questions['blockchain_service'] == 'a':683            answer = 'addrindex'684        else:685            answer = 'insight'686        print("\tUsing %s" % answer)687    assert answered_questions['blockchain_service'] in QUESTION_FLAGS['blockchain_service']688    if role == 'clearinghoused_only' and 'clearinghoused_public' not in answered_questions:689        answered_questions['clearinghoused_public'] = ask_question(690            "Enable public Clearinghoused setup (listen on all hosts w/ rpc/1234 user) (Y/n)", ('y', 'n'), 'y')691    else:692        answered_questions['clearinghoused_public'] = answered_questions.get('clearinghoused_public', 'n') #does not apply693    assert answered_questions['clearinghoused_public'] in QUESTION_FLAGS['clearinghoused_public']694    counterwallet_support_email = None695    if role == 'clearwallet' and 'clearwallet_support_email' not in answered_questions:696        while True:697            counterwallet_support_email = input("Email address where support cases should go (blank to disable): ")698            counterwallet_support_email = counterwallet_support_email.strip()699            if counterwallet_support_email:700                counterwallet_support_email_confirm = ask_question(701                    "You entered '%s', is that right? (Y/n): " % counterwallet_support_email, ('y', 'n'), 'y')702                if counterwallet_support_email_confirm == 'y': break703            else: break704        answered_questions['clearwallet_support_email'] = counterwallet_support_email705    else:706        answered_questions['clearwallet_support_email'] = answered_questions.get('clearwallet_support_email', '')707    if 'security_hardening' not in answered_questions:708        answered_questions['security_hardening'] = ask_question("Set up security hardening? (Y/n)", ('y', 'n'), 'y')709    assert answered_questions['security_hardening'] in QUESTION_FLAGS['security_hardening']710    return answered_questions711def usage():712    print("SYNTAX: %s [-h] %s" % (sys.argv[0], ' '.join([('[-%s=%s]' % (q, '|'.join(v) if v else '')) for q, v in QUESTION_FLAGS.items()])))713def main():714    logging.basicConfig(level=logging.DEBUG, format='%(asctime)s|%(levelname)s: %(message)s')715    do_prerun_checks()716    run_as_user = os.environ["SUDO_USER"]717    assert run_as_user718    answered_questions = {}719    #parse any command line objects720    branch = "master"721    try:722        opts, args = getopt.getopt(sys.argv[1:], "h", ["help",] + ['%s=' % q for q in QUESTION_FLAGS.keys()])723    except getopt.GetoptError as err:724        usage()725        sys.exit(2)726    for o, a in opts:727        if o in ("-h", "--help"):728            usage()729            sys.exit()730        elif o in ['--%s' % q for q in QUESTION_FLAGS.keys()]: #process flags for non-interactivity731            answered_questions[o.lstrip('-')] = a732        else:733            assert False, "Unhandled or unimplemented switch or option"734    base_path = os.path.expanduser("~%s/clearinghoused_build" % USERNAME)735    dist_path = os.path.join(base_path, "dist")736    #Detect if we should ask the user if they just want to update the source and not do a rebuild737    do_rebuild = 'r'738    try:739        pwd.getpwnam(USERNAME) #hacky check ...as this user is created by the script740    except:741        answered_questions['op'] = 'r' #do a build742    else: #setup has already been run at least once743        if not (answered_questions.get('op', None) in QUESTION_FLAGS['op']):744            answered_questions['op'] = ask_question(745                "It appears this setup has been run already. (r)ebuild node, or just (U)pdate from git? (r/U)",746                ('r', 'u'), 'u')747            assert answered_questions['op'] in QUESTION_FLAGS['op']748    if answered_questions['op'] == 'u': #just refresh counterpartyd, counterblockd, and counterwallet, etc. from github749        if os.path.exists("/etc/init.d/iwatch"):750            runcmd("service iwatch stop", abort_on_failure=False)751        #refresh counterpartyd_build, counterpartyd and counterblockd (if available)752        runcmd("%s/setup.py %s --for-user=xch update" % (base_path,753            '--with-clearblockd' if os.path.exists(os.path.join(dist_path, "clearblockd")) else ''))754        #refresh counterwallet (if available)755        if os.path.exists(os.path.exists(os.path.expanduser("~%s/clearwallet" % USERNAME))):756            do_counterwallet_setup(run_as_user, "AUTO", updateOnly=True)757        #offer to restart services758        restarted = command_services("restart", prompt=True)759        if not restarted and os.path.exists("/etc/init.d/iwatch"):760            runcmd("service iwatch start", abort_on_failure=False)761        sys.exit(0) #all done762    #If here, a) federated node has not been set up yet or b) the user wants a rebuild763    answered_questions = gather_build_questions(answered_questions)764    command_services("stop")765    do_base_setup(run_as_user, answered_questions['branch'], base_path, dist_path)766    bitcoind_rpc_password, bitcoind_rpc_password_testnet \767        = do_bitcoind_setup(run_as_user, answered_questions['branch'], base_path, dist_path, answered_questions['run_mode'])768    do_counterparty_setup(answered_questions['role'], run_as_user, answered_questions['branch'], base_path, dist_path, answered_questions['run_mode'],769        bitcoind_rpc_password, bitcoind_rpc_password_testnet,770        answered_questions['clearinghoused_public'], answered_questions['clearwallet_support_email'])771    do_blockchain_service_setup(run_as_user, base_path, dist_path,772        answered_questions['run_mode'], answered_questions['blockchain_service'], bitcoind_rpc_password, bitcoind_rpc_password_testnet)773    if answered_questions['role'] != "clearinghoused_only":774        do_nginx_setup(run_as_user, base_path, dist_path)775    do_armory_utxsvr_setup(run_as_user, base_path, dist_path,776        answered_questions['run_mode'], enable=answered_questions['role']=='clearwallet')777    if answered_questions['role'] == 'clearwallet':778        do_counterwallet_setup(run_as_user, answered_questions['branch'])779    do_newrelic_setup(run_as_user, base_path, dist_path, answered_questions['run_mode']) #optional780    if answered_questions['security_hardening']:781        do_security_setup(run_as_user, answered_questions['branch'], base_path, dist_path)782    logging.info("Clearblock Federated Node Build Complete (whew).")783    command_services("restart", prompt=True)784if __name__ == "__main__":...

...16        bitbake('glibc busybox zlib bash')17    @testcase(1203)18    def test_lookup_pkg(self):19        # Forward tests20        result = runCmd('oe-pkgdata-util lookup-pkg "glibc busybox"')21        self.assertEqual(result.output, 'libc6\nbusybox')22        result = runCmd('oe-pkgdata-util lookup-pkg zlib-dev')23        self.assertEqual(result.output, 'libz-dev')24        result = runCmd('oe-pkgdata-util lookup-pkg nonexistentpkg', ignore_status=True)25        self.assertEqual(result.status, 1)26        self.assertEqual(result.output, 'ERROR: The following packages could not be found: nonexistentpkg')27        # Reverse tests28        result = runCmd('oe-pkgdata-util lookup-pkg -r "libc6 busybox"')29        self.assertEqual(result.output, 'glibc\nbusybox')30        result = runCmd('oe-pkgdata-util lookup-pkg -r libz-dev')31        self.assertEqual(result.output, 'zlib-dev')32        result = runCmd('oe-pkgdata-util lookup-pkg -r nonexistentpkg', ignore_status=True)33        self.assertEqual(result.status, 1)34        self.assertEqual(result.output, 'ERROR: The following packages could not be found: nonexistentpkg')35    @testcase(1205)36    def test_read_value(self):37        result = runCmd('oe-pkgdata-util read-value PN libz1')38        self.assertEqual(result.output, 'zlib')39        result = runCmd('oe-pkgdata-util read-value PKGSIZE bash')40        pkgsize = int(result.output.strip())41        self.assertGreater(pkgsize, 1)42    @testcase(1198)43    def test_find_path(self):44        result = runCmd('oe-pkgdata-util find-path /lib/libc.so.6')45        self.assertEqual(result.output, 'glibc: /lib/libc.so.6')46        result = runCmd('oe-pkgdata-util find-path /bin/bash')47        self.assertEqual(result.output, 'bash: /bin/bash')48        result = runCmd('oe-pkgdata-util find-path /not/exist', ignore_status=True)49        self.assertEqual(result.status, 1)50        self.assertEqual(result.output, 'ERROR: Unable to find any package producing path /not/exist')51    @testcase(1204)52    def test_lookup_recipe(self):53        result = runCmd('oe-pkgdata-util lookup-recipe "libc6-staticdev busybox"')54        self.assertEqual(result.output, 'glibc\nbusybox')55        result = runCmd('oe-pkgdata-util lookup-recipe libz-dbg')56        self.assertEqual(result.output, 'zlib')57        result = runCmd('oe-pkgdata-util lookup-recipe nonexistentpkg', ignore_status=True)58        self.assertEqual(result.status, 1)59        self.assertEqual(result.output, 'ERROR: The following packages could not be found: nonexistentpkg')60    @testcase(1202)61    def test_list_pkgs(self):62        # No arguments63        result = runCmd('oe-pkgdata-util list-pkgs')64        pkglist = result.output.split()65        self.assertIn('glibc-utils', pkglist)66        self.assertIn('zlib-dev', pkglist)67        # No pkgspec, runtime68        result = runCmd('oe-pkgdata-util list-pkgs -r')69        pkglist = result.output.split()70        self.assertIn('libc6-utils', pkglist)71        self.assertIn('libz-dev', pkglist)72        # With recipe specified73        result = runCmd('oe-pkgdata-util list-pkgs -p zlib')74        pkglist = sorted(result.output.split())75        try:76            pkglist.remove('zlib-ptest') # in case ptest is disabled77        except ValueError:78            pass79        self.assertEqual(pkglist, ['zlib', 'zlib-dbg', 'zlib-dev', 'zlib-doc', 'zlib-staticdev'])80        # With recipe specified, runtime81        result = runCmd('oe-pkgdata-util list-pkgs -p zlib -r')82        pkglist = sorted(result.output.split())83        try:84            pkglist.remove('libz-ptest') # in case ptest is disabled85        except ValueError:86            pass87        self.assertEqual(pkglist, ['libz-dbg', 'libz-dev', 'libz-doc', 'libz-staticdev', 'libz1'])88        # With recipe specified and unpackaged89        result = runCmd('oe-pkgdata-util list-pkgs -p zlib -u')90        pkglist = sorted(result.output.split())91        self.assertIn('zlib-locale', pkglist)92        # With recipe specified and unpackaged, runtime93        result = runCmd('oe-pkgdata-util list-pkgs -p zlib -u -r')94        pkglist = sorted(result.output.split())95        self.assertIn('libz-locale', pkglist)96        # With recipe specified and pkgspec97        result = runCmd('oe-pkgdata-util list-pkgs -p zlib "*-d*"')98        pkglist = sorted(result.output.split())99        self.assertEqual(pkglist, ['zlib-dbg', 'zlib-dev', 'zlib-doc'])100        # With recipe specified and pkgspec, runtime101        result = runCmd('oe-pkgdata-util list-pkgs -p zlib -r "*-d*"')102        pkglist = sorted(result.output.split())103        self.assertEqual(pkglist, ['libz-dbg', 'libz-dev', 'libz-doc'])104    @testcase(1201)105    def test_list_pkg_files(self):106        def splitoutput(output):107            files = {}108            curpkg = None109            for line in output.splitlines():110                if line.startswith('\t'):111                    self.assertTrue(curpkg, 'Unexpected non-package line:\n%s' % line)112                    files[curpkg].append(line.strip())113                else:114                    self.assertTrue(line.rstrip().endswith(':'), 'Invalid package line in output:\n%s' % line)115                    curpkg = line.split(':')[0]116                    files[curpkg] = []117            return files118        base_libdir = get_bb_var('base_libdir')119        libdir = get_bb_var('libdir')120        includedir = get_bb_var('includedir')121        mandir = get_bb_var('mandir')122        # Test recipe-space package name123        result = runCmd('oe-pkgdata-util list-pkg-files zlib-dev zlib-doc')124        files = splitoutput(result.output)125        self.assertIn('zlib-dev', files.keys())126        self.assertIn('zlib-doc', files.keys())127        self.assertIn(os.path.join(includedir, 'zlib.h'), files['zlib-dev'])128        self.assertIn(os.path.join(mandir, 'man3/zlib.3'), files['zlib-doc'])129        # Test runtime package name130        result = runCmd('oe-pkgdata-util list-pkg-files -r libz1 libz-dev')131        files = splitoutput(result.output)132        self.assertIn('libz1', files.keys())133        self.assertIn('libz-dev', files.keys())134        self.assertGreater(len(files['libz1']), 1)135        libspec = os.path.join(base_libdir, 'libz.so.1.*')136        found = False137        for fileitem in files['libz1']:138            if fnmatch.fnmatchcase(fileitem, libspec):139                found = True140                break141        self.assertTrue(found, 'Could not find zlib library file %s in libz1 package file list: %s' % (libspec, files['libz1']))142        self.assertIn(os.path.join(includedir, 'zlib.h'), files['libz-dev'])143        # Test recipe144        result = runCmd('oe-pkgdata-util list-pkg-files -p zlib')145        files = splitoutput(result.output)146        self.assertIn('zlib-dbg', files.keys())147        self.assertIn('zlib-doc', files.keys())148        self.assertIn('zlib-dev', files.keys())149        self.assertIn('zlib-staticdev', files.keys())150        self.assertIn('zlib', files.keys())151        self.assertNotIn('zlib-locale', files.keys())152        # (ignore ptest, might not be there depending on config)153        self.assertIn(os.path.join(includedir, 'zlib.h'), files['zlib-dev'])154        self.assertIn(os.path.join(mandir, 'man3/zlib.3'), files['zlib-doc'])155        self.assertIn(os.path.join(libdir, 'libz.a'), files['zlib-staticdev'])156        # Test recipe, runtime157        result = runCmd('oe-pkgdata-util list-pkg-files -p zlib -r')158        files = splitoutput(result.output)159        self.assertIn('libz-dbg', files.keys())160        self.assertIn('libz-doc', files.keys())161        self.assertIn('libz-dev', files.keys())162        self.assertIn('libz-staticdev', files.keys())163        self.assertIn('libz1', files.keys())164        self.assertNotIn('libz-locale', files.keys())165        self.assertIn(os.path.join(includedir, 'zlib.h'), files['libz-dev'])166        self.assertIn(os.path.join(mandir, 'man3/zlib.3'), files['libz-doc'])167        self.assertIn(os.path.join(libdir, 'libz.a'), files['libz-staticdev'])168        # Test recipe, unpackaged169        result = runCmd('oe-pkgdata-util list-pkg-files -p zlib -u')170        files = splitoutput(result.output)171        self.assertIn('zlib-dbg', files.keys())172        self.assertIn('zlib-doc', files.keys())173        self.assertIn('zlib-dev', files.keys())174        self.assertIn('zlib-staticdev', files.keys())175        self.assertIn('zlib', files.keys())176        self.assertIn('zlib-locale', files.keys()) # this is the key one177        self.assertIn(os.path.join(includedir, 'zlib.h'), files['zlib-dev'])178        self.assertIn(os.path.join(mandir, 'man3/zlib.3'), files['zlib-doc'])179        self.assertIn(os.path.join(libdir, 'libz.a'), files['zlib-staticdev'])180        # Test recipe, runtime, unpackaged181        result = runCmd('oe-pkgdata-util list-pkg-files -p zlib -r -u')182        files = splitoutput(result.output)183        self.assertIn('libz-dbg', files.keys())184        self.assertIn('libz-doc', files.keys())185        self.assertIn('libz-dev', files.keys())186        self.assertIn('libz-staticdev', files.keys())187        self.assertIn('libz1', files.keys())188        self.assertIn('libz-locale', files.keys()) # this is the key one189        self.assertIn(os.path.join(includedir, 'zlib.h'), files['libz-dev'])190        self.assertIn(os.path.join(mandir, 'man3/zlib.3'), files['libz-doc'])191        self.assertIn(os.path.join(libdir, 'libz.a'), files['libz-staticdev'])192    @testcase(1200)193    def test_glob(self):194        tempdir = tempfile.mkdtemp(prefix='pkgdataqa')195        self.track_for_cleanup(tempdir)196        pkglistfile = os.path.join(tempdir, 'pkglist')197        with open(pkglistfile, 'w') as f:198            f.write('libc6\n')199            f.write('libz1\n')200            f.write('busybox\n')201        result = runCmd('oe-pkgdata-util glob %s "*-dev"' % pkglistfile)202        desiredresult = ['libc6-dev', 'libz-dev', 'busybox-dev']203        self.assertEqual(sorted(result.output.split()), sorted(desiredresult))204        # The following should not error (because when we use this during rootfs construction, sometimes the complementary package won't exist)205        result = runCmd('oe-pkgdata-util glob %s "*-nonexistent"' % pkglistfile)206        self.assertEqual(result.output, '')207        # Test exclude option208        result = runCmd('oe-pkgdata-util glob %s "*-dev *-dbg" -x "^libz"' % pkglistfile)209        resultlist = result.output.split()210        self.assertNotIn('libz-dev', resultlist)211        self.assertNotIn('libz-dbg', resultlist)212    @testcase(1206)213    def test_specify_pkgdatadir(self):214        result = runCmd('oe-pkgdata-util -p %s lookup-pkg glibc' % get_bb_var('PKGDATA_DIR'))...

1#!/usr/bin/python2#3# Test for smbcontrol command line argument handling.4#5import comfychair6class NoArgs(comfychair.TestCase):7    """Test no arguments produces usage message."""8    def runtest(self):9        out = self.runcmd("smbcontrol", expectedResult = 1)10        self.assert_re_match("Usage: smbcontrol", out[1])11        12class OneArg(comfychair.TestCase):13    """Test single argument produces usage message."""14    def runtest(self):15        out = self.runcmd("smbcontrol foo", expectedResult = 1)16        self.assert_re_match("Usage: smbcontrol", out[1])17        18class SmbdDest(comfychair.TestCase):19    """Test the broadcast destination 'smbd'."""20    def runtest(self):21        out = self.runcmd("smbcontrol smbd noop")22class NmbdDest(comfychair.TestCase):23    """Test the destination 'nmbd'."""24    def runtest(self):25        # We need a way to start/stop/whatever nmbd26        raise comfychair.NotRunError, "not implemented"27class PidDest(comfychair.TestCase):28    """Test a pid number destination'."""29    def runtest(self):30        out = self.runcmd("smbcontrol 1234 noop")31class SelfDest(comfychair.TestCase):32    """Test the destination 'self'."""33    def runtest(self):34        out = self.runcmd("smbcontrol self noop")35class WinbinddDest(comfychair.TestCase):36    """Test the destination 'winbindd'."""37    def runtest(self):38        # We need a way to start/stop/whatever winbindd39        raise comfychair.NotRunError, "not implemented"40class BadDest(comfychair.TestCase):41    """Test a bad destination."""42    def runtest(self):43        out = self.runcmd("smbcontrol foo noop", expectedResult = 1)44class BadCmd(comfychair.TestCase):45    """Test a bad command."""46    def runtest(self):47        out = self.runcmd("smbcontrol self spottyfoot", expectedResult = 1)48        self.assert_re_match("smbcontrol: unknown command", out[1]);49class NoArgCmdTest(comfychair.TestCase):50    """A test class that tests a command with no argument."""51    def runtest(self):52        self.require_root()53        out = self.runcmd("smbcontrol self %s" % self.cmd)54        out = self.runcmd("smbcontrol self %s spottyfoot" % self.cmd,55                          expectedResult = 1)56class ForceElection(NoArgCmdTest):57    """Test a force-election message."""58    def setup(self):59        self.cmd = "force-election"60class SamSync(NoArgCmdTest):61    """Test a samsync message."""62    def setup(self):63        self.cmd = "samsync"64class SamRepl(NoArgCmdTest):65    """Test a samrepl message."""66    def setup(self):67        self.cmd = "samrepl"68class DmallocChanged(NoArgCmdTest):69    """Test a dmalloc-changed message."""70    def setup(self):71        self.cmd = "dmalloc-log-changed"72class DmallocMark(NoArgCmdTest):73    """Test a dmalloc-mark message."""74    def setup(self):75        self.cmd = "dmalloc-mark"76class Shutdown(NoArgCmdTest):77    """Test a shutdown message."""78    def setup(self):79        self.cmd = "shutdown"80class Ping(NoArgCmdTest):81    """Test a ping message."""82    def setup(self):83        self.cmd = "ping"84class Debuglevel(NoArgCmdTest):85    """Test a debuglevel message."""86    def setup(self):87        self.cmd = "debuglevel"88class OneArgCmdTest(comfychair.TestCase):89    """A test class that tests a command with one argument."""90    def runtest(self):91        self.require_root()92        out = self.runcmd("smbcontrol self %s spottyfoot" % self.cmd)93        out = self.runcmd("smbcontrol self %s" % self.cmd, expectedResult = 1)94class DrvUpgrade(OneArgCmdTest):95    """Test driver upgrade message."""96    def setup(self):97        self.cmd = "drvupgrade"98        99class CloseShare(OneArgCmdTest):100    """Test close share message."""101    def setup(self):102        self.cmd = "close-share"103        104class Debug(OneArgCmdTest):105    """Test a debug message."""106    def setup(self):107        self.cmd = "debug"108class PrintNotify(comfychair.TestCase):109    """Test print notification commands."""110    def runtest(self):111        # No subcommand112        out = self.runcmd("smbcontrol self printnotify", expectedResult = 1)113        self.assert_re_match("Must specify subcommand", out[1]);114        # Invalid subcommand name115        out = self.runcmd("smbcontrol self printnotify spottyfoot",116                          expectedResult = 1)117        self.assert_re_match("Invalid subcommand", out[1]);118        # Queue commands119        for cmd in ["queuepause", "queueresume"]:120            out = self.runcmd("smbcontrol self printnotify %s" % cmd,121                              expectedResult = 1)122            self.assert_re_match("Usage:", out[1])123        124            out = self.runcmd("smbcontrol self printnotify %s spottyfoot"125                              % cmd)126        # Job commands127        for cmd in ["jobpause", "jobresume", "jobdelete"]:128            out = self.runcmd("smbcontrol self printnotify %s" % cmd,129                              expectedResult = 1)130            self.assert_re_match("Usage:", out[1])131            out = self.runcmd("smbcontrol self printnotify %s spottyfoot"132                              % cmd, expectedResult = 1)133            self.assert_re_match("Usage:", out[1])134            out = self.runcmd("smbcontrol self printnotify %s spottyfoot 123"135                              % cmd)136        # Printer properties137        out = self.runcmd("smbcontrol self printnotify printer",138                          expectedResult = 1)139        self.assert_re_match("Usage", out[1])140        out = self.runcmd("smbcontrol self printnotify printer spottyfoot",141                          expectedResult = 1)142        self.assert_re_match("Usage", out[1])143        for cmd in ["comment", "port", "driver"]:144            out = self.runcmd("smbcontrol self printnotify printer spottyfoot "145                              "%s" % cmd, expectedResult = 1)146            self.assert_re_match("Usage", out[1])147            out = self.runcmd("smbcontrol self printnotify printer spottyfoot "148                              "%s value" % cmd)149class Profile(comfychair.TestCase):150    """Test setting the profiling level."""151    def runtest(self):152        self.require_root()153        out = self.runcmd("smbcontrol self profile", expectedResult = 1)154        self.assert_re_match("Usage", out[1])155        156        out = self.runcmd("smbcontrol self profile spottyfoot",157                          expectedResult = 1)158        self.assert_re_match("Unknown", out[1])159        160        for cmd in ["off", "count", "on", "flush"]:161            out = self.runcmd("smbcontrol self profile %s" % cmd)162class ProfileLevel(comfychair.TestCase):163    """Test requesting the current profiling level."""164    def runtest(self):165        self.require_root()166        out = self.runcmd("smbcontrol self profilelevel spottyfoot",167                          expectedResult = 1)168        self.assert_re_match("Usage", out[1])169        170        out = self.runcmd("smbcontrol self profilelevel")171        172class TimeoutArg(comfychair.TestCase):173    """Test the --timeout argument."""174    def runtest(self):175        out = self.runcmd("smbcontrol --timeout 5 self noop")176        out = self.runcmd("smbcontrol --timeout spottyfoot self noop",177                          expectedResult = 1)178class ConfigFileArg(comfychair.TestCase):179    """Test the --configfile argument."""180    def runtest(self):181        out = self.runcmd("smbcontrol --configfile /dev/null self noop")182class BogusArg(comfychair.TestCase):183    """Test a bogus command line argument."""184    def runtest(self):185        out = self.runcmd("smbcontrol --bogus self noop", expectedResult = 1)186tests = [NoArgs, OneArg, SmbdDest, NmbdDest, WinbinddDest, PidDest,187         SelfDest, BadDest, BadCmd, Debug, ForceElection, SamSync,188         SamRepl, DmallocMark, DmallocChanged, Shutdown, DrvUpgrade,189         CloseShare, Ping, Debuglevel, PrintNotify, Profile, ProfileLevel,190         TimeoutArg, ConfigFileArg, BogusArg]191# Handle execution of this file as a main program192if __name__ == '__main__':...