TestMu Conference is Back! Future of Testing + AI LambdaTest G2 Announcement

How to use NewFetcher method of content Package

Best Testkube code snippet using content.NewFetcher

fetcher.go

Source:fetcher.go

...59	Header    CustomHeader60	Client    *http.Client             `json:"-"`61	Cache     map[string]CacheResponse `json:"-"`62}63func NewFetcher(host string) (f *Fetcher) {64	f = newFetcher(nil)65	f.Host = host66	return f67}68func NewFetcherHttps(host string) (f *Fetcher) {69	tr := &http.Transport{70		TLSClientConfig:    &tls.Config{RootCAs: nil, InsecureSkipVerify: true},71		DisableCompression: true,72	}73	f = newFetcher(tr)74	f.Host = host75	f.Https = true76	return77}78func newFetcher(tr http.RoundTripper) (f *Fetcher) {79	f = &Fetcher{}80	newTr := NewTransport(tr)81	newTr.AfterReq = func(resp *http.Response, req *http.Request) {82		f.mergeCookie(resp)...

fetcher_test.go

Source:fetcher_test.go

1package fetch2import (3	"bytes"4	"io/ioutil"5	"math/rand"6	"net/url"7	"strconv"8	"testing"9	"time"10	"github.com/spf13/viper"11	"github.com/stretchr/testify/assert"12)13func TestBaseFetcher_Proxy(t *testing.T) {14	viper.Set("PROXY", "http://127.0.0.1:3128")15	viper.Set("CHROME_TRACE", true)16	//viper.Set("PROXY", "")17	fetcher := newFetcher(Base)18	assert.NotNil(t, fetcher)19	fetcher = newFetcher(Chrome)20	assert.NotNil(t, fetcher)21}22func TestBaseFetcher_Fetch(t *testing.T) {23	viper.Set("PROXY", "")24	fetcher := newFetcher(Base)25	req := Request{26		URL:    tsURL + "/hello",27		Method: "GET",28	}29	html, err := fetcher.Fetch(req)30	assert.NoError(t, err, "Expected no error")31	data, err := ioutil.ReadAll(html)32	assert.NoError(t, err, "Expected no error")33	assert.Equal(t, helloContent, data)34	//Test 200 response35	req = Request{36		URL: tsURL,37	}38	content, err := fetcher.Fetch(req)39	assert.NoError(t, err)40	assert.NotNil(t, content, "Expected content not nil")41	//Test Form Data42	req = Request{43		Type:     "base",44		URL:      tsURL,45		FormData: "auth_key=880ea6a14ea49e853634fbdc5015a024&referer=http%3A%2F%2Fexample.com%2F&ips_username=user&ips_password=userpassword&rememberMe=1",46	}47	content, err = fetcher.Fetch(req)48	assert.NoError(t, err)49	assert.NotNil(t, content, "Expected content not nil")50	//Test Host()51	req = Request{52		URL: "http://google.com/somepage",53	}54	host, err := req.Host()55	assert.NoError(t, err)56	assert.Equal(t, "google.com", host, "Test BaseFetcherRequest Host()")57	req = Request{58		URL: "Invalid.%$^host",59	}60	_, err = req.Host()61	assert.Error(t, err)62	//fetch robots.txt data63	robots, _ := fetcher.Fetch(Request{64		URL:    tsURL + "/robots.txt",65		Method: "GET",66	})67	data, err = ioutil.ReadAll(robots)68	assert.NoError(t, err, "Expected no error")69	assert.Equal(t, robotsContent, string(data))70}71func TestChromeFetcher_Fetch(t *testing.T) {72	viper.Set("PROXY", "")73	fetcher := newFetcher(Chrome)74	req := Request{75		Type: "chrome",76		URL:  "http://testserver:12345",77	}78	resp, err := fetcher.Fetch(req)79	assert.Nil(t, err, "Expected no error")80	assert.NotNil(t, resp, "Expected resp not nil")81	//Test Form Data82	//TODO: Add real tests here83	req = Request{84		Type:     "chrome",85		URL:      "http://testserver:12345",86		FormData: "auth_key=880ea6a14ea49e853634fbdc5015a024&referer=http%3A%2F%2Fexample.com%2F&ips_username=user&ips_password=userpassword&rememberMe=1",87	}88	resp, err = fetcher.Fetch(req)89	assert.NoError(t, err)90	assert.NotNil(t, resp, "Expected content not nil")91	req = Request{92		Type: "chrome",93		URL:  "http://testserver:12345/status/200",94		//URL: ts.URL + "/index.html",95	}96	host, err := req.Host()97	assert.NoError(t, err)98	assert.Equal(t, "testserver:12345", host)99	req = Request{100		Type: "chrome",101		URL:  "Invalid.%$^host",102	}103	_, err = req.Host()104	assert.Error(t, err)105	//test runJSFromFile106	req = Request{107		Type: "chrome",108		URL:  "http://testserver:12345/status/200",109		//InfiniteScroll: true,110	}111	resp, err = fetcher.Fetch(req)112	assert.Nil(t, err, "Expected no error")113	assert.NotNil(t, resp, "Expected resp not nil")114}115func Test_parseFormData(t *testing.T) {116	formData := "auth_key=880ea6a14ea49e853634fbdc5015a024&referer=http%3A%2F%2Fexample.com%2F&ips_username=usr&ips_password=passw&rememberMe=0"117	values := parseFormData(formData)118	assert.Equal(t,119		url.Values{"auth_key": []string{"880ea6a14ea49e853634fbdc5015a024"},120			"referer": []string{"http%3A%2F%2Fexample.com%2F"}, "ips_username": []string{"usr"},121			"ips_password": []string{"passw"},122			"rememberMe":   []string{"0"}},123		values)124}125func TestInvalidFetcher(t *testing.T) {126	var fType Type127	fType = "unknownFetcher"128	defer func() {129		if r := recover(); r == nil {130			t.Errorf("The code did not panic")131		}132	}()133	fetcher := newFetcher(fType)134	assert.NotNil(t, fetcher)135}136func TestAuthFetcher(t *testing.T) {137	viper.Set("PROXY", "")138	fetcher := newFetcher(Chrome)139	randSrc := rand.NewSource(time.Now().UnixNano())140	nRand := rand.New(randSrc)141	randValue := strconv.Itoa(nRand.Intn(1000))142	username := "AnyUserNameAcceptable_" + randValue143	req := Request{144		Type:     "chrome",145		URL:      "http://testserver:12345/login",146		FormData: "username=" + username + "&password=123",147	}148	content, err := fetcher.Fetch(req)149	assert.NoError(t, err)150	pageContent, err := ioutil.ReadAll(content)151	assert.NoError(t, err)152	assert.Equal(t, true, bytes.Contains(pageContent, []byte(">"+username+"<")))153}154//readerToUtf8Encoding function is called in BaseFetcher.Fetch There is no need to call it for Chrome fetcher155func Test_BaseFetcher_Utf8Encoding(t *testing.T) {156	viper.Set("PROXY", "")157	fetcher := newFetcher(Base)158	req := Request{159		URL: tsURL + "/static/html/utf8.html",160		//URL: "https://www.tvojlekar.sk/lekari.php",161		Method: "GET",162	}163	html, err := fetcher.Fetch(req)164	assert.NoError(t, err, "Expected no error")165	data, err := ioutil.ReadAll(html)166	t.Log(string(data))167	req = Request{168		URL:    tsURL + "/static/html/win1250.html",169		Method: "GET",170	}171	html, err = fetcher.Fetch(req)172	assert.NoError(t, err, "Expected no error")173	data, err = ioutil.ReadAll(html)174	t.Log(string(data))175}...

fetch.go

Source:fetch.go

...12// SimpleFetcher download data from source and checks content type13type SimpleFetcher struct {14	contetTypes map[string]struct{}15}16// NewFetcher create new Fetcher17func NewFetcher() Fetcher {18	contetTypes := map[string]struct{}{19		"text/plain": struct{}{},20	}21	return SimpleFetcher{22		contetTypes: contetTypes,23	}24}25// Fetch retrieve data from url26func (f SimpleFetcher) Fetch(url string) (io.ReadCloser, error) {27	resp, err := http.Get(url)28	if err != nil {29		return nil, err30	}31	contentType, _, err := mime.ParseMediaType(resp.Header.Get("Content-Type"))...

NewFetcher

Using AI Code Generation

1import (2func main() {3	fetcher := content.NewFetcher()4	fmt.Println(fetcher)5}6import (7func main() {8	fetcher := content.NewFetcher()9	fmt.Println(fetcher)10}11import (12func main() {13	fetcher := content.NewFetcher()14	fmt.Println(fetcher)15}16import (17func main() {18	fetcher := content.NewFetcher()19	fmt.Println(fetcher)20}21import (22func main() {23	fetcher := content.NewFetcher()24	fmt.Println(fetcher)25}26import (27func main() {28	fetcher := content.NewFetcher()29	fmt.Println(fetcher)30}31import (32func main() {33	fetcher := content.NewFetcher()34	fmt.Println(fetcher)35}36import (37func main() {38	fetcher := content.NewFetcher()39	fmt.Println(fetcher)40}41import (42func main() {43	fetcher := content.NewFetcher()44	fmt.Println(fetcher)45}46import (47func main() {48	fetcher := content.NewFetcher()49	fmt.Println(fetcher)50}51import (

NewFetcher

Using AI Code Generation

1import "content"2func main() {3    f := content.NewFetcher()4    f.Fetch()5}6import "fetcher"7func main() {8    f := fetcher.NewFetcher()9    f.Fetch()10}11import "fetcher"12func main() {13    f := fetcher.NewFetcher()14    f.Fetch()15}

NewFetcher

Using AI Code Generation

1import (2func main() {3}4import (5func main() {6	fmt.Println(f)7}8import (9func main() {10	fmt.Println(f)11}12import (13func main() {14	fmt.Println(f)15	fmt.Println(f.Url)16}17import

NewFetcher

Using AI Code Generation

1import (2func main() {3    fmt.Println("Hello World!")4    c := content.NewFetcher()5    c.Fetch()6}7import (8func main() {9    fmt.Println("Hello World!")10    c := content.NewFetcher()11    c.Fetch()12}13import (14func main() {15    fmt.Println("Hello World!")16    c := content.NewFetcher()17    c.Fetch()18}

NewFetcher

Using AI Code Generation

1import (2func main() {3	fmt.Println(fetcher)4}5import (6func main() {7	fmt.Println(fetcher)8}9import (10func main() {11	fmt.Println(fetcher)12}13import (14func main() {15	fmt.Println(fetcher)16}17import (18func main() {19	fmt.Println(fetcher)20}21import (22func main() {23	fmt.Println(fetcher)24}25import (26func main() {27	fmt.Println(fetcher)28}29import (30func main() {

NewFetcher

Using AI Code Generation

1import (2type Content struct {3}4func (c *Content) NewFetcher() {5	fmt.Println("NewFetcher method called")6}7func main() {8	c := &Content{}9	fmt.Println(reflect.TypeOf(c))10}

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.