How to use tokenize method of gop Package

1package gop_test2import (3	"bytes"4	"encoding/base64"5	"fmt"6	"io/ioutil"7	"os"8	"os/exec"9	"reflect"10	"testing"11	"text/template"12	"time"13	"unsafe"14	"github.com/ysmood/got"15	"github.com/ysmood/got/lib/diff"16	"github.com/ysmood/got/lib/gop"17)18func TestStyle(t *testing.T) {19	g := got.T(t)20	s := gop.Style{Set: "<s>", Unset: "</s>"}21	g.Eq(gop.S("test", s), "<s>test</s>")22	g.Eq(gop.S("", s), "<s></s>")23	g.Eq(gop.S("", gop.None), "")24}25func TestTokenize(t *testing.T) {26	g := got.T(t)27	ref := "test"28	timeStamp, _ := time.Parse(time.RFC3339Nano, "2021-08-28T08:36:36.807908+08:00")29	fn := func(string) int { return 10 }30	ch1 := make(chan int)31	ch2 := make(chan string, 3)32	ch3 := make(chan struct{})33	v := []interface{}{34		nil,35		[]int{},36		[]interface{}{true, false, uintptr(0x17), float32(100.121111133)},37		true, 10, int8(2), int32(100),38		float64(100.121111133),39		complex64(1 + 2i), complex128(1 + 2i),40		[3]int{1, 2},41		ch1,42		ch2,43		ch3,44		fn,45		map[interface{}]interface{}{46			`"test"`: 10,47			"a":      1,48		},49		unsafe.Pointer(&ref),50		struct {51			Int int52			str string53			M   map[int]int54		}{10, "ok", map[int]int{1: 0x20}},55		[]byte("aa\xe2"),56		[]byte("bytes\n\tbytes"),57		[]byte("long long long long string"),58		byte('a'),59		byte(1),60		'天',61		"long long long long string",62		"\ntest",63		"\t\n`",64		&ref,65		(*struct{ Int int })(nil),66		&struct{ Int int }{},67		&map[int]int{1: 2, 3: 4},68		&[]int{1, 2},69		&[2]int{1, 2},70		&[]byte{1, 2},71		timeStamp,72		time.Hour,73		`{"a": 1}`,74		[]byte(`{"a": 1}`),75	}76	check := func(out string, tpl ...string) {77		g.Helper()78		expected := bytes.NewBuffer(nil)79		t := template.New("")80		g.E(t.Parse(g.Read(g.Open(false, tpl...)).String()))81		g.E(t.Execute(expected, map[string]interface{}{82			"ch1": fmt.Sprintf("0x%x", reflect.ValueOf(ch1).Pointer()),83			"ch2": fmt.Sprintf("0x%x", reflect.ValueOf(ch2).Pointer()),84			"ch3": fmt.Sprintf("0x%x", reflect.ValueOf(ch3).Pointer()),85			"fn":  fmt.Sprintf("0x%x", reflect.ValueOf(fn).Pointer()),86			"ptr": fmt.Sprintf("%v", &ref),87		}))88		if out != expected.String() {89			g.Fail()90			g.Log(diff.Diff(out, expected.String()))91		}92	}93	out := gop.StripANSI(gop.F(v))94	{95		code := fmt.Sprintf(g.Read(g.Open(false, "fixtures", "compile_check.go.tmpl")).String(), out)96		f := g.Open(true, "tmp", g.RandStr(8), "main.go")97		g.Cleanup(func() { _ = os.Remove(f.Name()) })98		g.Write(code)(f)99		b, err := exec.Command("go", "run", f.Name()).CombinedOutput()100		if err != nil {101			g.Error(string(b))102		}103	}104	check(out, "fixtures", "expected.tmpl")105	out = gop.VisualizeANSI(gop.F(v))106	check(out, "fixtures", "expected_with_color.tmpl")107}108func TestRef(t *testing.T) {109	g := got.T(t)110	a := [2][]int{{1}}111	a[1] = a[0]112	g.Eq(gop.Plain(a), `[2][]int{113    []int/* len=1 cap=1 */{114        1,115    },116    []int/* len=1 cap=1 */{117        1,118    },119}`)120}121type A struct {122	Int int123	B   *B124}125type B struct {126	s string127	a *A128}129func TestCircularRef(t *testing.T) {130	g := got.T(t)131	a := A{Int: 10}132	b := B{"test", &a}133	a.B = &b134	g.Eq(gop.StripANSI(gop.F(a)), `gop_test.A{135    Int: 10,136    B: &gop_test.B{137        s: "test",138        a: &gop_test.A{139            Int: 10,140            B: gop.Circular("B").(*gop_test.B),141        },142    },143}`)144}145func TestCircularNilRef(t *testing.T) {146	arr := []A{{}, {}}147	got.T(t).Eq(gop.StripANSI(gop.F(arr)), `[]gop_test.A/* len=2 cap=2 */{148    gop_test.A{149        Int: 0,150        B: (*gop_test.B)(nil),151    },152    gop_test.A{153        Int: 0,154        B: (*gop_test.B)(nil),155    },156}`)157}158func TestCircularMap(t *testing.T) {159	g := got.T(t)160	a := map[int]interface{}{}161	a[0] = a162	ts := gop.Tokenize(a)163	g.Eq(gop.Format(ts, gop.ThemeNone), `map[int]interface {}{164    0: gop.Circular().(map[int]interface {}),165}`)166}167func TestCircularSlice(t *testing.T) {168	g := got.New(t)169	a := [][]interface{}{{nil}, {nil}}170	a[0][0] = a[1]171	a[1][0] = a[0][0]172	ts := gop.Tokenize(a)173	g.Eq(gop.Format(ts, gop.ThemeNone), `[][]interface {}/* len=2 cap=2 */{174    []interface {}/* len=1 cap=1 */{175        []interface {}/* len=1 cap=1 */{176            gop.Circular(0, 0).([]interface {}),177        },178    },179    []interface {}/* len=1 cap=1 */{180        gop.Circular(1).([]interface {}),181    },182}`)183}184func TestPlain(t *testing.T) {185	g := got.T(t)186	g.Eq(gop.Plain(10), "10")187}188func TestP(t *testing.T) {189	gop.Stdout = ioutil.Discard190	_ = gop.P("test")191	gop.Stdout = os.Stdout192}193func TestConvertors(t *testing.T) {194	g := got.T(t)195	g.Nil(gop.Circular(""))196	s := g.RandStr(8)197	g.Eq(gop.Ptr(s).(*string), &s)198	bs := base64.StdEncoding.EncodeToString([]byte(s))199	g.Eq(gop.Base64(bs), []byte(s))200	now := time.Now()201	g.Eq(gop.Time(now.Format(time.RFC3339Nano), 1234), now)202	g.Eq(gop.Duration("10m"), 10*time.Minute)203	g.Eq(gop.JSONStr(nil, "[1, 2]"), "[1, 2]")204	g.Eq(gop.JSONBytes(nil, "[1, 2]"), []byte("[1, 2]"))205}206func TestGetPrivateFieldErr(t *testing.T) {207	g := got.T(t)208	g.Panic(func() {209		gop.GetPrivateField(reflect.ValueOf(1), 0)210	})211	g.Panic(func() {212		gop.GetPrivateFieldByName(reflect.ValueOf(1), "test")213	})214}215func TestTypeName(t *testing.T) {216	g := got.T(t)217	type f float64218	type i int219	type c complex128220	type b byte221	g.Eq(gop.Plain(f(1)), "gop_test.f(1.0)")222	g.Eq(gop.Plain(i(1)), "gop_test.i(1)")223	g.Eq(gop.Plain(c(1)), "gop_test.c(1+0i)")224	g.Eq(gop.Plain(b('a')), "gop_test.b(97)")225}226func TestSliceCapNotEqual(t *testing.T) {227	g := got.T(t)228	x := gop.Plain(make([]int, 3, 10))229	y := gop.Plain(make([]int, 3))230	g.Desc("we should show the diff of cap").Neq(x, y)231}232func TestFixNestedStyle(t *testing.T) {233	g := got.T(t)234	s := gop.S(" 0 "+gop.S(" 1 "+235		gop.S(" 2 "+236			gop.S(" 3 ", gop.Cyan)+237			" 4 ", gop.Blue)+238		" 5 ", gop.Red)+" 6 ", gop.BgRed)239	fmt.Println(gop.VisualizeANSI(s))240	out := gop.VisualizeANSI(gop.FixNestedStyle(s))241	g.Eq(out, `<41> 0 <31> 1 <39><34> 2 <39><36> 3 <39><34> 4 <39><31> 5 <39> 6 <49>`)242	gop.FixNestedStyle("test")243}244func TestStripANSI(t *testing.T) {245	g := got.T(t)246	g.Eq(gop.StripANSI(gop.S("test", gop.Red)), "test")247}248func TestTheme(t *testing.T) {249	g := got.T(t)250	g.Eq(gop.ThemeDefault(gop.Error), []gop.Style{gop.Underline, gop.Red})251}...

1package diff2import (3	"context"4	"time"5	"github.com/ysmood/got/lib/gop"6)7// Theme for diff8type Theme func(t Type) []gop.Style9// ThemeDefault colors for Sprint10var ThemeDefault = func(t Type) []gop.Style {11	switch t {12	case AddSymbol:13		return []gop.Style{gop.Green}14	case DelSymbol:15		return []gop.Style{gop.Red}16	case AddWords:17		return []gop.Style{gop.Green}18	case DelWords:19		return []gop.Style{gop.Red}20	case ChunkStart:21		return []gop.Style{gop.Black, gop.BgMagenta}22	}23	return []gop.Style{gop.None}24}25// ThemeNone colors for Sprint26var ThemeNone = func(t Type) []gop.Style {27	return []gop.Style{gop.None}28}29// Diff x and y into a human readable string.30func Diff(x, y string) string {31	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)32	defer cancel()33	return Format(Tokenize(ctx, x, y), ThemeDefault)34}35// Tokenize x and y into diff tokens with diff words and narrow chunks.36func Tokenize(ctx context.Context, x, y string) []*Token {37	ts := TokenizeText(ctx, x, y)38	lines := ParseTokenLines(ts)39	lines = Narrow(1, lines)40	Words(ctx, lines)41	return SpreadTokenLines(lines)42}43// Format tokens into a human readable string44func Format(ts []*Token, theme Theme) string {45	out := ""46	for _, t := range ts {47		s := t.Literal48		out += gop.Stylize(s, theme(t.Type))49	}50	return out51}52// Narrow the context around each diff section to n lines.53func Narrow(n int, lines []*TokenLine) []*TokenLine {54	if n < 0 {55		n = 056	}57	keep := map[int]bool{}58	for i, l := range lines {59		switch l.Type {60		case AddSymbol, DelSymbol:61			for j := max(i-n, 0); j <= i+n && j < len(lines); j++ {62				keep[j] = true63			}64		}65	}66	out := []*TokenLine{}67	for i, l := range lines {68		if !keep[i] {69			continue70		}71		if _, has := keep[i-1]; !has {72			ts := []*Token{{ChunkStart, "@@ diff chunk @@"}, {Newline, "\n"}}73			out = append(out, &TokenLine{ChunkStart, ts})74		}75		out = append(out, l)76		if _, has := keep[i+1]; !has {77			ts := []*Token{{ChunkEnd, ""}, {Newline, "\n"}}78			out = append(out, &TokenLine{ChunkEnd, ts})79		}80	}81	return out82}83// Words diff84func Words(ctx context.Context, lines []*TokenLine) {85	delLines := []*TokenLine{}86	addLines := []*TokenLine{}87	df := func() {88		if len(delLines) == 0 || len(delLines) != len(addLines) {89			return90		}91		for i := 0; i < len(delLines); i++ {92			d := delLines[i]93			a := addLines[i]94			dts, ats := TokenizeLine(ctx, d.Tokens[2].Literal, a.Tokens[2].Literal)95			d.Tokens = append(d.Tokens[0:2], append(dts, d.Tokens[3:]...)...)96			a.Tokens = append(a.Tokens[0:2], append(ats, a.Tokens[3:]...)...)97		}98		delLines = []*TokenLine{}99		addLines = []*TokenLine{}100	}101	for _, l := range lines {102		switch l.Type {103		case DelSymbol:104			delLines = append(delLines, l)105		case AddSymbol:106			addLines = append(addLines, l)107		default:108			df()109		}110	}111	df()112}113func max(x, y int) int {114	if x < y {115		return y116	}117	return x118}...

1import (2func main() {3	for i := 1; i < len(os.Args); i++ {4	}5	fmt.Println(s)6}7import (8func main() {9	fmt.Println(strings.Join(os.Args[1:], " "))10	fmt.Println(os.Args[1:])11}12import (13func main() {14	for index, arg := range os.Args[1:] {15		fmt.Println(index, arg)16	}17}18import (19func main() {20	for _, arg := range os.Args[1:] {21		fmt.Println(arg)22	}23}24import (25func main() {26	for range os.Args[1:] {27		fmt.Println("hi")28	}29}30import (31func main() {32	for range os.Args[1:] {33		fmt.Println("hi")34	}35}36import (37func main() {38	for range os.Args[1:] {39		fmt.Println("hi")40	}41}42import (43func main() {44	for range os.Args[1:] {45		fmt.Println("hi")46	}47}48import (49func main() {50	for range os.Args[1:] {51		fmt.Println("hi")52	}53}54import

1import (2func main() {3    g := gop.New()4    tokens := g.Tokenize("this is a test")5    for _, token := range tokens {6        fmt.Println(token)7    }8}

1import (2func main() {3    a = gop.NewGop()4    a.Tokenize("Hello world")5    fmt.Println(a)6}7import (8func main() {9    a = gop.NewGop()10    a.Tokenize("Hello world")11    fmt.Println(a)12}13import (14func main() {15    a = gop.NewGop()16    a.Tokenize("Hello world")17    fmt.Println(a)18}19import (20func main() {21    a = gop.NewGop()22    a.Tokenize("Hello world")23    fmt.Println(a)24}25import (26func main() {27    a = gop.NewGop()28    a.Tokenize("Hello world")29    fmt.Println(a)30}31import (32func main() {33    a = gop.NewGop()34    a.Tokenize("Hello world")35    fmt.Println(a)36}37import (38func main() {39    a = gop.NewGop()40    a.Tokenize("Hello world")41    fmt.Println(a

1import (2func main() {3	g := gop.New()4	g.Load("test.gop")5	fmt.Println(g.Tokenize())6}7import (8func main() {9	g := gop.New()10	g.Load("test.gop")11	fmt.Println(g.Parse())12}13import (14func main() {15	g := gop.New()16	g.Load("test.gop")17	fmt.Println(g.Compile())18}19import (20func main() {21	g := gop.New()22	g.Load("test.gop")23	fmt.Println(g.Execute())24}25import (26func main() {27	g := gop.New()28	g.Load("test.gop")29	fmt.Println(g.Tokenize())30	fmt.Println(g.Parse())31	fmt.Println(g.Compile())32	fmt.Println(g.Execute())33}34print(c)

1import (2func main() {3    g.Init("1.txt")4    for g.HasNext() {5        fmt.Println(g.Next())6    }7}