How to use hrefURL method of html Package

Best K6 code snippet using html.hrefURL

roundtrip.go

Source:roundtrip.go Github

copy

Full Screen

...374 } else {375 doc.Response.Body = d.Text()376 d.Find("base").Each(func(i int, s *goquery.Selection) {377 if val, ok := s.Attr("href"); ok {378 hrefURL, err := url.Parse(val)379 if err != nil {380 log.Debug("Error parsing url %s: %s", val, err.Error())381 return382 }383 if hrefURL.Host == targetURL.Host {384 hrefURL.Host = host.Host385 }386 s.SetAttr("href", hrefURL.String())387 }388 })389 d.Find("link").Each(func(i int, s *goquery.Selection) {390 if val, ok := s.Attr("href"); ok {391 hrefURL, err := url.Parse(val)392 if err != nil {393 log.Debug("Error parsing url %s: %s", val, err.Error())394 return395 }396 if hrefURL.Host == targetURL.Host {397 hrefURL.Host = host.Host398 }399 s.SetAttr("href", hrefURL.String())400 }401 })402 d.Find("form").Each(func(i int, s *goquery.Selection) {403 if val, ok := s.Attr("src"); ok {404 hrefURL, err := url.Parse(val)405 if err != nil {406 log.Debug("Error parsing url %s: %s", val, err.Error())407 return408 }409 if hrefURL.Host == targetURL.Host {410 hrefURL.Host = host.Host411 }412 s.SetAttr("src", hrefURL.String())413 }414 })415 d.Find("img").Each(func(i int, s *goquery.Selection) {416 if val, ok := s.Attr("src"); ok {417 hrefURL, err := url.Parse(val)418 if err != nil {419 log.Debug("Error parsing url %s: %s", val, err.Error())420 return421 }422 if hrefURL.Host == targetURL.Host {423 hrefURL.Host = host.Host424 }425 s.SetAttr("src", hrefURL.String())426 }427 })428 d.Find("script").Each(func(i int, s *goquery.Selection) {429 if val, ok := s.Attr("src"); ok {430 hrefURL, err := url.Parse(val)431 if err != nil {432 log.Debug("Error parsing url %s: %s", val, err.Error())433 return434 }435 if hrefURL.Host == targetURL.Host {436 hrefURL.Host = host.Host437 }438 s.SetAttr("src", hrefURL.String())439 }440 })441 d.Find("a").Each(func(i int, s *goquery.Selection) {442 if val, ok := s.Attr("href"); ok {443 hrefURL, err := url.Parse(val)444 if err != nil {445 log.Debug("Error parsing url %s: %s", val, err.Error())446 return447 }448 if hrefURL.Host == targetURL.Host {449 hrefURL.Host = host.Host450 }451 s.SetAttr("href", hrefURL.String())452 }453 })454 html, _ := d.Html()455 resp.Body = ioutil.NopCloser(strings.NewReader(html))456 }457 // rewrite location458 if val := resp.Header.Get("Location"); val == "" {459 } else if u, err := url.Parse(val); err != nil {460 log.Error("Error parsing url: %s", val)461 } else if targetURL.Host == u.Host {462 if u.Scheme != "https" {463 } else if t.ListenerTLS != "" {464 } else {465 u.Scheme = "http"...

Full Screen

Full Screen

wallpage_spider.go

Source:wallpage_spider.go Github

copy

Full Screen

1package main2import (3 "bytes"4 "fmt"5 "github.com/antchfx/htmlquery"6 _ "github.com/go-sql-driver/mysql"7 "github.com/jinzhu/gorm"8 "golang.org/x/net/html"9 "hi_golang/tools/lop"10 "io/ioutil"11 "net/http"12 "strings"13)14const (15 wallpagerHtmlTag = `//a[@href]`16 pageSize = 2317 currentPageIndex = 118)19/*20https://wallhaven.cc/toplist21wallhaven-caught-result-DB-Table22*/23type Wallpage struct {24 gorm.Model25 /* seconds */26 HrefUrl string `json:"href_url"`27 OfPageIndex uint32 `json:"of_page_index"`28 IsGet bool `json:"is_get"`29 ImageID string `json:"image_id"`30 FromWebsite string `json:"from_website"`31}32func (w Wallpage) TableName() string {33 return `wallpage`34}35var (36 db *gorm.DB37 err error38 wallpagesChan chan []Wallpage39)40func init() {41 db, err = gorm.Open("mysql", "root:Taohui520MARIADB!@#@(35.186.152.49)/wwwthsunkist?charset=utf8&parseTime=True&loc=Local")42 panic(err)43}44func main() {45 lop.T("running")46 lop.D("running")47 lop.I("running")48 lop.W("running")49 lop.E("running")50 go func() {51 if request, err := http.NewRequest("GET",52 fmt.Sprintf("https://wallhaven.cc/toplist?page=%d",53 currentPageIndex), nil); err != nil {54 lop.E(err)55 } else {56 httpCli := http.Client{}57 response, err := httpCli.Do(request)58 if err != nil {59 lop.E(err)60 return61 }62 responseBodyBytes, err := ioutil.ReadAll(response.Body)63 responseBodyBytesReader := bytes.NewReader(responseBodyBytes)64 doc, err := htmlquery.Parse(responseBodyBytesReader)65 if err != nil {66 lop.E(err)67 } else {68 parseResponse(doc, wallpagesChan)69 }70 }71 }()72 select {73 case wallpages := <-wallpagesChan:74 {75 downloadWallpages(wallpages)76 }77 }78}79func downloadWallpages(wallpages []Wallpage) {80 /* 如果无法下载了, 那么就需要通知上游, 停止拉取 */81}82/* 获取抓取目录的列表/ */83func parseResponse(doc *html.Node, wallpageChan chan<- []Wallpage) {84 if list := htmlquery.Find(doc, wallpagerHtmlTag); len(list) == 0 {85 return86 } else {87 var wallpages = make([]Wallpage, pageSize)88 for _, v := range list {89 hrefUrl := htmlquery.SelectAttr(v, "href")90 if len(v.Attr) < 1 || v.Attr[0].Val != "preview" {91 continue92 }93 var hrefAttrs = strings.SplitAfter(hrefUrl, "/")94 imageID := hrefAttrs[len(hrefAttrs)-1]95 var wallpage = Wallpage{96 ImageID: imageID,97 HrefUrl: hrefUrl,98 OfPageIndex: currentPageIndex,99 FromWebsite: strings.ReplaceAll(hrefAttrs[2], `/`, ``),100 }101 wallpages = append(wallpages, wallpage)102 lop.T(wallpage)103 }104 wallpageChan <- wallpages105 return106 }107}...

Full Screen

Full Screen

links_counter.go

Source:links_counter.go Github

copy

Full Screen

...10 f = func(n *html.Node) {11 if n.Type == html.ElementNode && n.Data == "a" { // only get links from a tags12 for _, attr := range n.Attr {13 if attr.Key == "href" && attr.Val != "" {14 hrefURL, err := url.Parse(attr.Val)15 if err != nil {16 log.Println("malformed href value: ", attr.Val, err)17 break // nested links are forbidden => assuming there is only one href per node, we can break from the loop18 }19 if (hrefURL.Hostname() == page.Hostname()) ||20 (hrefURL.Hostname() == "" && hrefURL.Path != "") { // if host isn't set but path is set, then the link is most likely internal21 internal++22 continue23 }24 external++25 }26 }27 }28 for c := n.FirstChild; c != nil; c = c.NextSibling {29 f(c)30 }31 }32 f(document)33 return34}...

Full Screen

Full Screen

hrefURL

Using AI Code Generation

copy

Full Screen

1import (2func main() {3 if err != nil {4 panic(err)5 }6 defer res.Body.Close()7 if res.StatusCode != 200 {8 panic(res.Status)9 }10 doc, err := goquery.NewDocumentFromReader(res.Body)11 if err != nil {12 panic(err)13 }14 doc.Find("a").Each(func(i int, s *goquery.Selection) {15 href, _ := s.Attr("href")16 fmt.Println(href)17 })18}19import (20func main() {21 if err != nil {22 panic(err)23 }24 defer res.Body.Close()25 if res.StatusCode != 200 {26 panic(res.Status)27 }28 doc, err := goquery.NewDocumentFromReader(res.Body)29 if err != nil {30 panic(err)31 }32 doc.Find("a").Each(func(i int, s *goquery.Selection) {33 href, _ := s.Attr("href")34 fmt.Println(href)35 })36}37import (38func main() {39 if err != nil {40 panic(err)41 }42 defer res.Body.Close()43 if res.StatusCode != 200 {44 panic(res.Status)45 }

Full Screen

Full Screen

hrefURL

Using AI Code Generation

copy

Full Screen

1import (2func main() {3 if err != nil {4 fmt.Println(err)5 }6 defer resp.Body.Close()7 doc, err := html.Parse(resp.Body)8 if err != nil {9 fmt.Println(err)10 }11 for _, link := range visit(nil, doc) {12 fmt.Println(link)13 }14}15func visit(links []string, n *html.Node) []string {16 if n.Type == html.ElementNode && n.Data == "a" {17 for _, a := range n.Attr {18 if a.Key == "href" {19 links = append(links, a.Val)20 }21 }22 }23 for c := n.FirstChild; c != nil; c = c.NextSibling {24 links = visit(links, c)25 }26}

Full Screen

Full Screen

hrefURL

Using AI Code Generation

copy

Full Screen

1import "fmt"2import "github.com/gocolly/colly"3func main() {4 c := colly.NewCollector()5 c.OnHTML("a[href]", func(e *colly.HTMLElement) {6 link := e.Attr("href")7 fmt.Printf("Link found: %q -> %s8 })9}10import "fmt"11import "github.com/gocolly/colly"12func main() {13 c := colly.NewCollector()14 c.OnHTML("a[href]", func(e *colly.HTMLElement) {15 link := e.Request.AbsoluteURL(e.Attr("href"))16 fmt.Printf("Link found: %q -> %s17 })18}19import "fmt"20import "github.com/gocolly/colly"21func main() {22 c := colly.NewCollector()23 c.OnHTML("a[href]", func(e *colly.HTMLElement) {24 e.Request.Visit(e.Attr("href"))25 })26}27import "fmt"28import "github.com/gocolly/colly"29func main() {30 c := colly.NewCollector()31 c.OnHTML("a[href]", func(e *colly.HTMLElement) {32 e.Request.Visit(e.Attr("href"))33 })34 c.OnRequest(func(r *colly.Request) {35 fmt.Println("Visiting", r.URL)36 })

Full Screen

Full Screen

hrefURL

Using AI Code Generation

copy

Full Screen

1import (2func main() {3 if err != nil {4 fmt.Println(err)5 }6 defer resp.Body.Close()7 doc, err := html.Parse(resp.Body)8 if err != nil {9 fmt.Println(err)10 }11 links := visit(nil, doc)12 for _, link := range links {13 fmt.Println(link)14 }15}16func visit(links []string, n *html.Node) []string {17 if n.Type == html.ElementNode && n.Data == "a" {18 for _, a := range n.Attr {19 if a.Key == "href" {20 links = append(links, a.Val)21 }22 }23 }24 for c := n.FirstChild; c != nil; c = c.NextSibling {25 links = visit(links, c)26 }27}

Full Screen

Full Screen

hrefURL

Using AI Code Generation

copy

Full Screen

1import (2func main() {3 if err != nil {4 log.Fatal(err)5 }6 defer resp.Body.Close()7 body, err := ioutil.ReadAll(resp.Body)8 if err != nil {9 log.Fatal(err)10 }11 fmt.Println(golhtml.HrefURL(string(body)))12}13import (14func main() {15 if err != nil {16 log.Fatal(err)17 }18 defer resp.Body.Close()19 body, err := ioutil.ReadAll(resp.Body)20 if err != nil {21 log.Fatal(err)22 }23 fmt.Println(golhtml.HrefURL(string(body)))24}25import (26func main() {27 if err != nil {28 log.Fatal(err)29 }30 defer resp.Body.Close()31 body, err := ioutil.ReadAll(resp.Body)32 if err != nil {33 log.Fatal(err)34 }35 fmt.Println(golhtml.HrefURL(string(body)))36}37import (38func main() {39 if err != nil {40 log.Fatal(err)41 }42 defer resp.Body.Close()43 body, err := ioutil.ReadAll(resp.Body)44 if err != nil {45 log.Fatal(err)46 }47 fmt.Println(golhtml.HrefURL

Full Screen

Full Screen

hrefURL

Using AI Code Generation

copy

Full Screen

1import (2func main() {3 if len(os.Args) != 2 {4 fmt.Println("Please provide the URL")5 os.Exit(1)6 }7 resp, err := http.Get(url)8 if err != nil {9 fmt.Println("Error in getting the response from the url: ", err)10 os.Exit(1)11 }12 body, err := ioutil.ReadAll(resp.Body)13 if err != nil {14 fmt.Println("Error in reading the response body: ", err)15 os.Exit(1)16 }17 resp.Body.Close()18 links := html.Parse(os.Stdin)19 for _, link := range links {20 fmt.Println(link)21 }22}23import (24func main() {25 if len(os.Args) != 2 {26 fmt.Println("Please provide the URL")27 os.Exit(1)28 }29 resp, err := http.Get(url)30 if err != nil {31 fmt.Println("Error in getting the response from the url: ", err)32 os.Exit(1)33 }34 body, err := ioutil.ReadAll(resp.Body)35 if err != nil {36 fmt.Println("Error in reading the response body: ", err)37 os.Exit(1)38 }39 resp.Body.Close()40 links := html.Parse(os.Stdin)41 for _, link := range links {42 fmt.Println(link)43 }44}45import (

Full Screen

Full Screen

hrefURL

Using AI Code Generation

copy

Full Screen

1import (2func main() {3 http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {4 fmt.Fprintf(w, "Hello, %q", html.EscapeString(r.URL.Path))5 })6 http.ListenAndServe(":8080", nil)7}

Full Screen

Full Screen

hrefURL

Using AI Code Generation

copy

Full Screen

1import (2func main() {3 if err != nil {4 log.Fatal(err)5 }6 doc, err := html.Parse(resp.Body)7 resp.Body.Close()8 if err != nil {9 log.Fatal(err)10 }11 for _, link := range visit(nil, doc) {12 fmt.Println(link)13 }14}15func visit(links []string, n *html.Node) []string {16 if n.Type == html.ElementNode && n.Data == "a" {17 for _, a := range n.Attr {18 if a.Key == "href" {19 links = append(links, a.Val)20 }21 }22 }23 for c := n.FirstChild; c != nil; c = c.NextSibling {24 links = visit(links, c)25 }26}

Full Screen

Full Screen

hrefURL

Using AI Code Generation

copy

Full Screen

1html := html.New()2fmt.Println(url)3html := html.New()4fmt.Println(url)5html := html.New()6fmt.Println(url)7html := html.New()8fmt.Println(url)9html := html.New()10fmt.Println(url)11html := html.New()12fmt.Println(url)13html := html.New()14fmt.Println(url)15html := html.New()16fmt.Println(url)17html := html.New()18fmt.Println(url)

Full Screen

Full Screen

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.

LambdaTest Learning Hubs:

YouTube

You could also refer to video tutorials over LambdaTest YouTube channel to get step by step demonstration from industry experts.

Run K6 automation tests on LambdaTest cloud grid

Perform automation testing on 3000+ real desktop and mobile devices online.

Most used method in

Try LambdaTest Now !!

Get 100 minutes of automation test minutes FREE!!

Next-Gen App & Browser Testing Cloud

Was this article helpful?

Helpful

NotHelpful