How to use ListObjects method of gcs Package

Best Syzkaller code snippet using gcs.ListObjects

provider_test.go

Source:provider_test.go

1// Copyright 2021 IBM Corporation2//3// Licensed under the Apache License, Version 2.0 (the "License");4// you may not use this file except in compliance with the License.5// You may obtain a copy of the License at6//7//     http://www.apache.org/licenses/LICENSE-2.08//9// Unless required by applicable law or agreed to in writing, software10// distributed under the License is distributed on an "AS IS" BASIS,11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.12// See the License for the specific language governing permissions and13// limitations under the License.14package gcsprovider15import (16	"context"17	"path/filepath"18	"testing"19	"github.com/go-logr/logr"20	"github.com/golang/mock/gomock"21	"github.com/kserve/modelmesh-runtime-adapter/pullman"22	"github.com/stretchr/testify/assert"23	"sigs.k8s.io/controller-runtime/pkg/log/zap"24)25func newGCSProviderWithMocks(t *testing.T) (gcsProvider, *MockgcsDownloaderFactory, logr.Logger) {26	mockCtrl := gomock.NewController(t)27	mdf := NewMockgcsDownloaderFactory(mockCtrl)28	g := gcsProvider{29		gcsDownloaderFactory: mdf,30	}31	log := zap.New()32	return g, mdf, log33}34func newGCSRepositoryClientWithMock(t *testing.T) (*gcsRepositoryClient, *MockgcsDownloader) {35	mockCtrl := gomock.NewController(t)36	md := NewMockgcsDownloader(mockCtrl)37	log := zap.New()38	gcsrc := gcsRepositoryClient{39		gcsclient: md,40		log:       log,41	}42	return &gcsrc, md43}44func Test_NewRepositoryWithCredentials(t *testing.T) {45	g, mdf, log := newGCSProviderWithMocks(t)46	c := pullman.NewRepositoryConfig("gcs", nil)47	privateKey := "private key"48	clientEmail := "client@email.com"49	c.Set("private_key", privateKey)50	c.Set("client_email", clientEmail)51	// Test that credentials are passed to newDownloader if specified.52	mdf.EXPECT().newDownloader(gomock.Any(), gomock.Eq(map[string]string{53		"private_key": privateKey, "client_email": clientEmail, "type": "service_account"})).Times(1)54	_, err := g.NewRepository(c, log)55	assert.NoError(t, err)56	tokenUri := "http://foo.bar"57	c.Set("token_uri", tokenUri)58	// Test that optional token_uri field is passed to newDownloader if specified.59	mdf.EXPECT().newDownloader(gomock.Any(), gomock.Eq(map[string]string{60		"private_key": privateKey, "client_email": clientEmail, "type": "service_account", "token_uri": tokenUri})).Times(1)61	_, err = g.NewRepository(c, log)62	assert.NoError(t, err)63}64func Test_NewRepositoryNoCredentials(t *testing.T) {65	g, mdf, log := newGCSProviderWithMocks(t)66	c := pullman.NewRepositoryConfig("gcs", nil)67	mdf.EXPECT().newDownloader(gomock.Any(), gomock.Nil()).Times(1)68	_, err := g.NewRepository(c, log)69	assert.NoError(t, err)70}71func Test_Download_SimpleDirectory(t *testing.T) {72	gcsRc, mdf := newGCSRepositoryClientWithMock(t)73	bucket := "bucket"74	c := pullman.NewRepositoryConfig("gcs", nil)75	c.Set("bucket", bucket)76	downloadDir := filepath.Join("test", "output")77	inputPullCommand := pullman.PullCommand{78		RepositoryConfig: c,79		Directory:        downloadDir,80		Targets: []pullman.Target{81			{82				RemotePath: "path/to/modeldir",83			},84		},85	}86	mdf.EXPECT().listObjects(context.Background(), gomock.Eq(bucket), gomock.Eq("path/to/modeldir")).87		Return([]string{"path/to/modeldir/file.ext", "path/to/modeldir/subdir/another_file"}, nil).88		Times(1)89	expectedTargets := []pullman.Target{90		{91			RemotePath: "path/to/modeldir/file.ext",92			LocalPath:  filepath.Join(downloadDir, "file.ext"),93		},94		{95			RemotePath: "path/to/modeldir/subdir/another_file",96			LocalPath:  filepath.Join(downloadDir, "subdir", "another_file"),97		},98	}99	mdf.EXPECT().downloadBatch(gomock.Any(), gomock.Eq(bucket), gomock.Eq(expectedTargets)).100		Return(nil).101		Times(1)102	err := gcsRc.Pull(context.Background(), inputPullCommand)103	assert.NoError(t, err)104}105func Test_Download_MultipleTargets(t *testing.T) {106	gcsRc, mdf := newGCSRepositoryClientWithMock(t)107	bucket := "bucket"108	c := pullman.NewRepositoryConfig("gcs", nil)109	c.Set("bucket", bucket)110	downloadDir := filepath.Join("test", "output")111	inputPullCommand := pullman.PullCommand{112		RepositoryConfig: c,113		Directory:        downloadDir,114		Targets: []pullman.Target{115			{116				RemotePath: "dir",117			},118			{119				// test that single file can be renamed120				RemotePath: "some_file",121				LocalPath:  "some_name.json",122			},123			{124				// test that a directory can be "renamed"125				RemotePath: "another_dir",126				LocalPath:  "local_another_dir",127			},128			{129				// test that single file can be renamed into subdirectory130				RemotePath: "another_file",131				LocalPath:  "local_another_dir/some_other_name.ext",132			},133			{134				// test that single file can pulled into a target directory135				RemotePath: "yet_another_file",136				LocalPath:  "local_another_dir/",137			},138		},139	}140	mdf.EXPECT().listObjects(context.Background(), gomock.Eq(bucket), gomock.Eq("dir")).141		Return([]string{"dir/file1", "dir/file2"}, nil).142		Times(1)143	mdf.EXPECT().listObjects(context.Background(), gomock.Eq(bucket), gomock.Eq("some_file")).144		Return([]string{"some_file"}, nil).145		Times(1)146	mdf.EXPECT().listObjects(context.Background(), gomock.Eq(bucket), gomock.Eq("another_dir")).147		Return([]string{"another_dir/another_file", "another_dir/subdir1/subdir2/nested_file"}, nil).148		Times(1)149	mdf.EXPECT().listObjects(context.Background(), gomock.Eq(bucket), gomock.Eq("another_file")).150		Return([]string{"another_file"}, nil).151		Times(1)152	mdf.EXPECT().listObjects(context.Background(), gomock.Eq(bucket), gomock.Eq("yet_another_file")).153		Return([]string{"yet_another_file"}, nil).154		Times(1)155	expectedTargets := []pullman.Target{156		{157			RemotePath: "dir/file1",158			LocalPath:  filepath.Join(downloadDir, "file1"),159		},160		{161			RemotePath: "dir/file2",162			LocalPath:  filepath.Join(downloadDir, "file2"),163		},164		{165			RemotePath: "some_file",166			LocalPath:  filepath.Join(downloadDir, "some_name.json"),167		},168		{169			RemotePath: "another_dir/another_file",170			LocalPath:  filepath.Join(downloadDir, "local_another_dir", "another_file"),171		},172		{173			RemotePath: "another_dir/subdir1/subdir2/nested_file",174			LocalPath:  filepath.Join(downloadDir, "local_another_dir", "subdir1/subdir2/nested_file"),175		},176		{177			RemotePath: "another_file",178			LocalPath:  filepath.Join(downloadDir, "local_another_dir", "some_other_name.ext"),179		},180		{181			RemotePath: "yet_another_file",182			LocalPath:  filepath.Join(downloadDir, "local_another_dir", "yet_another_file"),183		},184	}185	mdf.EXPECT().downloadBatch(gomock.Any(), gomock.Eq("bucket"), gomock.Eq(expectedTargets)).186		Return(nil).187		Times(1)188	err := gcsRc.Pull(context.Background(), inputPullCommand)189	assert.NoError(t, err)190}191func Test_GetKey(t *testing.T) {192	provider := gcsProvider{}193	createTestConfig := func() *pullman.RepositoryConfig {194		config := pullman.NewRepositoryConfig("gcs", nil)195		config.Set(configPrivateKey, "secret key")196		config.Set(configClientEmail, "user@email.com")197		return config198	}199	// should return the same result given the same config200	t.Run("shouldMatchForSameConfig", func(t *testing.T) {201		config1 := createTestConfig()202		config2 := createTestConfig()203		assert.Equal(t, provider.GetKey(config1), provider.GetKey(config2))204	})205	// changing the token_uri should change the key206	t.Run("shouldChangeForTokenUri", func(t *testing.T) {207		config1 := createTestConfig()208		config2 := createTestConfig()209		config2.Set(configTokenUri, "https://oauth2.googleapis.com/token2")210		assert.NotEqual(t, provider.GetKey(config1), provider.GetKey(config2))211	})212	// changing the bucket should NOT change the key213	t.Run("shouldNotChangeForBucket", func(t *testing.T) {214		config1 := createTestConfig()215		config2 := createTestConfig()216		config2.Set(configBucket, "another_bucket")217		assert.Equal(t, provider.GetKey(config1), provider.GetKey(config2))218	})219}...

googlecloudstorage.go

Source:googlecloudstorage.go

...77}78func (o GoogleCloudStorageObject) GetPath() string {79	return o.Name80}81func (conn GoogleCloudStorageConnection) ListObjectsAsFolders(prefix string) ([]Object, error) {82	return conn.listObjects(prefix, "/")83}84func (conn GoogleCloudStorageConnection) ListObjectsAsAll(prefix string) ([]Object, error) {85	return conn.listObjects(prefix, ",")86}87func (conn GoogleCloudStorageConnection) listObjects(prefix string, delimiter string) ([]Object, error) {88	log.Debugf("GoogleCloudStorageConnection listObjects. prefix: %s, delimeter: %s", prefix, delimiter)89	objects := make([]Object, 0)90	query := &storage.Query{91		Prefix:    prefix,92		Delimiter: delimiter,93	}94	for {95		gcsObjects, err := storage.ListObjects(conn.Context, conn.BucketName, query)96		if err != nil {97			return objects, err98		}99		if delimiter == "/" { // folders100			for _, prefix := range gcsObjects.Prefixes {101				name := strings.TrimSuffix(prefix, delimiter)102				object := GoogleCloudStorageObject{103					Name: name,104				}105				objects = append(objects, object)106			}107		} else { // regular files108			for _, gcsObject := range gcsObjects.Results {109				object := GoogleCloudStorageObject{...

gcs_test.go

Source:gcs_test.go

...60	defer s.Unlock()61	println(r.Method, r.URL.String(), string(valueOf(r)))62	switch {63	case r.Method == http.MethodGet && strings.Contains(r.URL.String(), "/o?"):64		s.ListObjects(w, r)65	case r.Method == http.MethodGet:66		s.GetObject(w, r)67	default:68		w.WriteHeader(http.StatusNotImplemented)69	}70}71// ListObjects emulates GCS list objects72func (s *fakeGCS) ListObjects(w http.ResponseWriter, r *http.Request) {73	var matches []*Object74	prefix := r.URL.Query().Get("prefix")75	for _, o := range s.Objects {76		if strings.HasPrefix(o.Key, prefix) {77			matches = append(matches, &Object{78				Bucket:  "bucket",79				Name:    o.Key,80				Updated: time.Unix(0, o.ModifiedAt).UTC().Format(time.RFC3339Nano),81				Size:    uint64(len(o.Value)),82			})83		}84	}85	var resp Objects86	resp.Items = matches...

ListObjects

Using AI Code Generation

1import (2func main() {3    ctx := context.Background()4    service, err := storage.NewService(ctx)5    if err != nil {6        log.Fatalf("Unable to create storage service: %v", err)7    }8    objects, err := service.Objects.List("bucketname").Do()9    if err != nil {10        log.Fatalf("Unable to retrieve objects: %v", err)11    }12    if len(objects.Items) == 0 {13        fmt.Println("No objects found.")14    } else {15        fmt.Println("Objects:")16        for _, item := range objects.Items {17            fmt.Printf("%s (%s)18        }19    }20}21file1 (bucketname/file1)22file2 (bucketname/file2)23file3 (bucketname/file3)24file4 (bucketname/file4)25file5 (bucketname/file5)26file6 (bucketname/file6)27file7 (bucketname/file7)28file8 (bucketname/file8)29file9 (bucketname/file9)30file10 (bucketname/file10)31file11 (bucketname/file11)32file12 (bucketname/file12)33file13 (bucketname/file13)34file14 (bucketname/file14)35file15 (bucketname/file15)36file16 (bucketname/file16)37file17 (bucketname/file17)38file18 (bucketname/file18)39file19 (bucketname/file19)40file20 (bucketname/file20)41file21 (bucketname/file21)42file22 (bucketname/file22)43file23 (bucketname/file23)44file24 (bucketname/file24)45file25 (bucketname/file25)46file26 (bucketname/file26)47file27 (bucketname/file27)48file28 (bucketname/file28)49file29 (bucketname/file29)50file30 (bucketname/file30)51file31 (bucketname/file31)52file32 (bucketname/file32)53file33 (bucketname/file33)54file34 (bucketname/file34)55file35 (bucketname/file35)56file36 (bucketname/file36)57file37 (bucketname/file37)

ListObjects

Using AI Code Generation

1import (2func main() {3    g := gcs.NewGcs()4    g.ListObjects()5}6import (7func main() {8    g := gcs.NewGcs()9    g.GetObject()10}11import (12func main() {13    g := gcs.NewGcs()14    g.PutObject()15}16import (17func main() {18    g := gcs.NewGcs()19    g.DeleteObject()20}21import (22func main() {23    g := gcs.NewGcs()24    g.DeleteBucket()25}26import (

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.