How to use get_files method in Airtest

...39                              attribs[attrib_idx]['date_str'],40                              attribs[attrib_idx]['sha1'])41    # Failure should be expected when adding the same file multiple times with the same sha1.42    self.assertEqual(not shouldfail, result)43    # Return the result expected from get_files().44    return {45      'chromium_version': chromium_version,46      'sha1': attribs[attrib_idx]['sha1'],47      'name': name,48      'platform': platform,49      'last_modified': attribs[attrib_idx]['date_val'],50      'cef_version': version,51      'type': type,52      'size': attribs[attrib_idx]['size']53    }54  # Test with no file contents.55  def test_empty(self):56    builder = cef_json_builder()57    self._verify_write_read(builder)58    files = builder.get_files()59    self.assertEqual(len(files), 0)60  # Test add/get of a single file with the specified type.61  def _test_add_file(self, type):62    builder = cef_json_builder()63    expected = self._add_test_file(builder, type=type)64    self._verify_write_read(builder)65    files = builder.get_files()66    self.assertEqual(len(files), 1)67    self.assertEqual(expected, files[0])68  # Test add/get of a standard type file.69  def test_add_standard_file(self):70    self._test_add_file('standard')71  # Test add/get of a minimal type file.72  def test_add_minimal_file(self):73    self._test_add_file('minimal')74  # Test add/get of a client type file.75  def test_add_client_file(self):76    self._test_add_file('client')77  # Test add/get of a debug_symbols type file.78  def test_add_debug_symbols_file(self):79    self._test_add_file('debug_symbols')80  # Test add/get of a release_symbols type file.81  def test_add_release_symbols_file(self):82    self._test_add_file('release_symbols')83  # Test get_files() behavior with a single file.84  def test_get_files_single(self):85    builder = cef_json_builder()86    # Specify all values just in case the defaults change.87    expected = self._add_test_file(builder,88        platform='linux32', version='3.2704.1414.g185cd6c', type='standard')89    # No filter.90    files = builder.get_files()91    self.assertEqual(len(files), 1)92    self.assertEqual(expected, files[0])93    # Platform filter.94    files = builder.get_files(platform='linux32')95    self.assertEqual(len(files), 1)96    self.assertEqual(expected, files[0])97    files = builder.get_files(platform='linux64')98    self.assertEqual(len(files), 0)99    # Version filter exact.100    files = builder.get_files(version='3.2704.1414.g185cd6c')101    self.assertEqual(len(files), 1)102    self.assertEqual(expected, files[0])103    # Version filter partial.104    files = builder.get_files(version='3.2704')105    self.assertEqual(len(files), 1)106    self.assertEqual(expected, files[0])107    files = builder.get_files(version='3.2623')108    self.assertEqual(len(files), 0)109    # Type filter.110    files = builder.get_files(type='standard')111    self.assertEqual(len(files), 1)112    self.assertEqual(expected, files[0])113    files = builder.get_files(type='client')114    self.assertEqual(len(files), 0)115    # All filters.116    files = builder.get_files(platform='linux32', version='3.2704', type='standard')117    self.assertEqual(len(files), 1)118    self.assertEqual(expected, files[0])119    files = builder.get_files(platform='linux32', version='3.2704', type='minimal')120    self.assertEqual(len(files), 0)121    files = builder.get_files(platform='linux32', version='3.2623', type='standard')122    self.assertEqual(len(files), 0)123    files = builder.get_files(platform='linux64', version='3.2704', type='standard')124    self.assertEqual(len(files), 0)125  # Test add/get of multiple files.126  def test_add_multiple_files(self):127    builder = cef_json_builder()128    expected = []129    platforms = cef_json_builder.get_platforms()130    versions = ['3.2704.1414.g185cd6c', '3.2704.1400.gde36543']131    types = cef_json_builder.get_distrib_types()132    for platform in platforms:133      for version in versions:134        for type in types:135          expected.append(self._add_test_file(builder, platform=platform, type=type, version=version))136    self._verify_write_read(builder)137    # No filter.138    files = builder.get_files()139    self.assertEqual(len(files), len(platforms) * len(versions) * len(types))140    # Version filter.141    files = builder.get_files(version=version)142    self.assertEqual(len(files), len(platforms) * len(types))143    # Type filter.144    files = builder.get_files(type='client')145    self.assertEqual(len(files), len(platforms) * len(versions))146    # Platform filter.147    files = builder.get_files(platform='windows32')148    self.assertEqual(len(files), len(types) * len(versions))149    # All filters.150    idx = 0151    for platform in platforms:152      for version in versions:153        for type in types:154          files = builder.get_files(platform=platform, type=type, version=version)155          self.assertEqual(len(files), 1)156          self.assertEqual(expected[idx], files[0])157          idx = idx + 1158  # Test add/get/replace of multiple files.159  def test_replace_all_files(self):160    builder = cef_json_builder()161    version = '3.2704.1414.g185cd6c'162    platforms = ['linux32', 'linux64']163    types = ['standard', 'minimal']164    # Initial file versions.165    for platform in platforms:166      for type in types:167        self._add_test_file(builder, platform=platform, type=type, version=version)168    # No filter.169    files = builder.get_files()170    self.assertEqual(len(files), len(platforms) * len(types))171    expected = []172    # Replace all file versions (due to new sha1).173    for platform in platforms:174      for type in types:175        expected.append(self._add_test_file(builder,176            platform=platform, type=type, version=version, attrib_idx=1))177    # No filter.178    files = builder.get_files()179    self.assertEqual(len(files), len(platforms) * len(types))180    # All filters.181    idx = 0182    for platform in platforms:183      for type in types:184        files = builder.get_files(platform=platform, type=type, version=version)185        self.assertEqual(len(files), 1)186        self.assertEqual(expected[idx], files[0])187        idx = idx + 1188  # Test add/get/no replace of multiple files.189  def test_replace_no_files(self):190    builder = cef_json_builder()191    version = '3.2704.1414.g185cd6c'192    platforms = ['linux32', 'linux64']193    types = ['standard', 'minimal']194    # Initial file versions.195    for platform in platforms:196      for type in types:197        self._add_test_file(builder, platform=platform, type=type, version=version)198    # No filter.199    files = builder.get_files()200    self.assertEqual(len(files), len(platforms) * len(types))201    expected = []202    # Replace no file versions (due to same sha1).203    for platform in platforms:204      for type in types:205        expected.append(self._add_test_file(builder,206            platform=platform, type=type, version=version, shouldfail=True))207    # No filter.208    files = builder.get_files()209    self.assertEqual(len(files), len(platforms) * len(types))210    # All filters.211    idx = 0212    for platform in platforms:213      for type in types:214        files = builder.get_files(platform=platform, type=type, version=version)215        self.assertEqual(len(files), 1)216        self.assertEqual(expected[idx], files[0])217        idx = idx + 1218  # Test Chromium version.219  def test_chromium_version(self):220    builder = cef_json_builder()221    self.assertTrue(builder.is_valid_chromium_version('master'))222    self.assertTrue(builder.is_valid_chromium_version('49.0.2704.0'))223    self.assertTrue(builder.is_valid_chromium_version('49.0.2704.50'))224    self.assertTrue(builder.is_valid_chromium_version('49.0.2704.100'))225    self.assertFalse(builder.is_valid_chromium_version(None))226    self.assertFalse(builder.is_valid_chromium_version('09.0.2704.50'))227    self.assertFalse(builder.is_valid_chromium_version('00.0.0000.00'))228    self.assertFalse(builder.is_valid_chromium_version('foobar'))...

...10        f"{MINI_ESGF_MASTER_DIR}/test_data/badc/cmip5/data/cmip5/output1/MOHC/HadGEM2-ES/historical/mon/atmos/Amon"11        f"/r1i1p1/latest/tas/*.nc"12    )13    @pytest.fixture14    def get_files(self, load_test_data):15        test_paths = sorted(glob.glob(self.test_path))16        return test_paths17    # actual range in files is 185912-20051118    def test_no_subset(self, get_files):19        inputs = {}20        sac = SubsetAlignmentChecker(get_files, inputs)21        assert sac.is_aligned is True22        assert sac.aligned_files == get_files23    def test_area_subset(self, get_files):24        inputs = {"area": "0.,49.,10.,65"}25        sac = SubsetAlignmentChecker(get_files, inputs)26        assert sac.is_aligned is False27        assert sac.aligned_files == []28    def test_time_subset_no_match(self, get_files):29        inputs = {"time": "1886-01/1930-11"}30        sac = SubsetAlignmentChecker(get_files, inputs)31        assert sac.is_aligned is False32        assert sac.aligned_files == []33    def test_time_subset_one_match(self, get_files):34        inputs = {"time": "1886-01/1984-11"}35        sac = SubsetAlignmentChecker(get_files, inputs)36        assert sac.is_aligned is False37        assert sac.aligned_files == []38    def test_time_subset_match(self, get_files):39        inputs = {"time": "1859-12/2005-11"}40        # start defaults to 1st of month41        # end defaults to 30th of month42        # the start and end day of dataset is 16th so this results in a requested time range outside of the actual range43        # so this is aligned.44        sac = SubsetAlignmentChecker(get_files, inputs)45        assert sac.is_aligned is True46        assert sac.aligned_files == get_files47class TestYearMonthDay1200:48    """Tests with year month and day for dataset with a 12:00:00 time step"""49    test_path = (50        f"{MINI_ESGF_MASTER_DIR}/test_data/gws/nopw/j04/cp4cds1_vol1/data/c3s-cmip5/output1/ICHEC/"51        f"EC-EARTH/historical/day/atmos/day/r1i1p1/tas/v20131231/*.nc"52    )53    # Actual range in files is: 18500101-2009113054    @pytest.fixture55    def get_files(self, load_test_data):56        test_paths = sorted(glob.glob(self.test_path))57        return test_paths58    def test_no_subset(self, get_files):59        inputs = {}60        sac = SubsetAlignmentChecker(get_files, inputs)61        assert sac.is_aligned is True62        assert sac.aligned_files == get_files63    def test_area_subset(self, get_files):64        inputs = {"area": "0.,49.,10.,65"}65        sac = SubsetAlignmentChecker(get_files, inputs)66        assert sac.is_aligned is False67        assert sac.aligned_files == []68    def test_time_subset_no_match(self, get_files):69        inputs = {"time": "1886-01-01/1930-11-01"}70        sac = SubsetAlignmentChecker(get_files, inputs)71        assert sac.is_aligned is False72        assert sac.aligned_files == []73    def test_time_subset_one_match(self, get_files):74        inputs = {"time": "1900-01-01/1930-11-01"}75        sac = SubsetAlignmentChecker(get_files, inputs)76        assert sac.is_aligned is False77        assert sac.aligned_files == []78    def test_time_subset_matches_one_file(self, get_files):79        inputs = {"time": "1900-01-01T12:00:00/1909-12-31T12:00:00"}80        sac = SubsetAlignmentChecker(get_files, inputs)81        assert sac.is_aligned is True82        assert sac.aligned_files == [83            f"{MINI_ESGF_MASTER_DIR}/test_data/gws/nopw/j04/cp4cds1_vol1/data/c3s-cmip5"84            f"/output1/ICHEC/EC-EARTH/historical/day/atmos/day/r1i1p1/tas/v20131231"85            f"/tas_day_EC-EARTH_historical_r1i1p1_19000101-19091231.nc"86        ]87    def test_time_subset_matches_exact_range_excluding_hour(self, get_files):88        """Tests alignment of full dataset where:89        - Real range: 18500101-2009113090        - Start: 18500101 (exact start)91        - End:   20091130 (exact end)92        """93        inputs = {"time": "1850-01-01/2009-11-30"}94        sac = SubsetAlignmentChecker(get_files, inputs)95        assert sac.is_aligned is True96        assert sac.aligned_files == get_files97    def test_time_subset_matches_before_to_end(self, get_files):98        """Tests alignment of full dataset where:99        - Real range: 18500101-20091130100        - Start: 17000101 (before)101        - End:   20091130T12:00:00 (exact end)102        """103        inputs = {"time": "1700-01-01/2009-11-30T12:00:00"}104        sac = SubsetAlignmentChecker(get_files, inputs)105        assert sac.is_aligned is True106        assert sac.aligned_files == get_files107    def test_time_subset_matches_start_to_after(self, get_files):108        """Tests alignment of full dataset where:109        - Real range: 18500101-20091130110        - Start: 18500101T12:00:00 (exact start)111        - End:   29991230 (after)112        """113        inputs = {"time": "1850-01-01T12:00:00/2999-12-30"}114        sac = SubsetAlignmentChecker(get_files, inputs)115        assert sac.is_aligned is True116        assert sac.aligned_files == get_files117    def test_time_subset_matches_before_to_after(self, get_files):118        """Tests alignment of full dataset where:119        - Real range: 18500101-20091130120        - Start: 17000101 (before)121        - End:   29991230 (after)122        """123        inputs = {"time": "1700-01-01/2999-12-30"}124        sac = SubsetAlignmentChecker(get_files, inputs)125        assert sac.is_aligned is True126        assert sac.aligned_files == get_files127    def test_time_subset_matches_including_hour(self, get_files):128        """Tests alignment of full dataset where:129        - Real range: 1850-01-01T12:00:00 to 2009-11-30T12:00:00130        - Start: 1850-01-01T12:00:00 (exact start)131        - End:   2009-11-30T12:00:00 (exact end)132        """133        inputs = {"time": "1850-01-01T12:00:00/2009-11-30T12:00:00"}134        sac = SubsetAlignmentChecker(get_files, inputs)135        assert sac.is_aligned is True136        assert sac.aligned_files == get_files137    def test_time_subset_no_match_including_hour(self, get_files):138        """Tests not aligned when time not aligned with file:139        - Real range: 1850-01-01T12:00:00 to 2009-11-30T12:00:00140        - Start: 1850-01-01T14:00:00 (start)141        - End:   2009-11-30T12:00:00 (exact end)142        """143        inputs = {"time": "1850-01-01T14:00:00/2009-11-30T12:00:00"}144        sac = SubsetAlignmentChecker(get_files, inputs)145        assert sac.is_aligned is False146        assert sac.aligned_files == []147class TestYearMonthDay0000:148    """Tests with year month and day for dataset with a 00:00:00 time step"""149    test_path = (150        f"{MINI_ESGF_MASTER_DIR}/test_data/badc/cmip5/data/cmip5/output1/MOHC/HadGEM2-ES/historical/mon/atmos/Amon"151        f"/r1i1p1/latest/tas/*.nc"152    )153    @pytest.fixture154    def get_files(self, load_test_data):155        test_paths = sorted(glob.glob(self.test_path))156        return test_paths157    # actual range in files is 185912-200511158    def test_time_subset_matches_no_hour(self, get_files):159        """Tests alignment of full dataset where:160        - Real range: 1859-12-16T00:00:00 to 2005-11-16T00:00:00161        - Start: 1859-12-16 (start without hour)162        - End:   2005-11-16 (end without hour)163        """164        inputs = {"time": "1859-12-16/2005-11-16"}165        sac = SubsetAlignmentChecker(get_files, inputs)166        # aligns because default is timestamp of 00:00:00 if not provided167        assert sac.is_aligned is True168        assert sac.aligned_files == get_files...

1import os2from os import listdir3from os.path import isfile, join4def get_files(file_path):5    only_files = [f for f in listdir(file_path) if isfile(join(file_path, f))]6    only_files = [os.path.join(file_path, f) for f in only_files]7    return only_files8def get_folders(folder_path):9    only_folders = [f for f in listdir(folder_path) if not isfile(join(folder_path, f))]10    only_folders = [os.path.join(folder_path, f) for f in only_folders]11    return only_folders    12def remove_files(folder_path):13    all_crawled_folders = get_folders(folder_path)14    for folder in all_crawled_folders:15        all_files = get_files(folder)16        for file_item in all_files:17            if 'parsed_' in file_item:18                os.remove(file_item)19def read_file_lines(file_to_read):20    with open(file_to_read, 'r') as myfile:21        return myfile.readlines()22def write_file(file_name, lines_to_write):23    f = open(file_name, 'w')24    for line in lines_to_write:25        f.write(line)  26extracted_feature_files_1 = '/mnt/drive/work/features_1/'27extracted_feature_files_2 = '/mnt/drive/work/features_2/'28extracted_feature_files_3 = '/mnt/drive/work/features_3/'29extracted_feature_files_4 = '/mnt/drive/work/features_4/'30extracted_feature_files_5 = '/mnt/drive/work/features_5/'31extracted_feature_files_6 = '/mnt/drive/work/features_6/'32extracted_feature_files_7 = '/mnt/drive/work/features_7/'33extracted_feature_files_8 = '/mnt/drive/work/features_8/'34extracted_feature_files_9 = '/mnt/drive/work/features_9/'35extracted_feature_files_10 = '/mnt/drive/work/features_10/'36extracted_feature_files_11 = '/mnt/drive/work/features_11/'37extracted_feature_files_12 = '/mnt/drive/work/features_12/'38feature_data_files_entries = get_files(extracted_feature_files_1) +\39 get_files(extracted_feature_files_2) +\40 get_files(extracted_feature_files_3) +\41 get_files(extracted_feature_files_4) +\42 get_files(extracted_feature_files_5) +\43 get_files(extracted_feature_files_6) +\44 get_files(extracted_feature_files_7) +\45 get_files(extracted_feature_files_8) +\46 get_files(extracted_feature_files_9) +\47 get_files(extracted_feature_files_10) +\48 get_files(extracted_feature_files_11) +\49 get_files(extracted_feature_files_12)50# feature_folder = '/mnt/drive/work/f_temp/'51all_feature_file = '/mnt/drive/work/unique_features_data.csv'52# all_files = get_files(feature_folder)53rows = []54unique_only = []55non_unique = []56for single_file in feature_data_files_entries:57    if single_file.split('/')[5] not in unique_only:58        rows += read_file_lines(single_file)59        unique_only.append(single_file.split('/')[5])60    else:61        non_unique.append(single_file)62write_file(all_feature_file, rows)63print len(unique_only)...