How to use tokenizeBytes method of gop Package

...157	case reflect.Interface:158		ts = append(ts, tokenize(sn, p, v.Elem())...)159	case reflect.Slice, reflect.Array:160		if data, ok := v.Interface().([]byte); ok {161			ts = append(ts, tokenizeBytes(data)...)162			if len(data) > 1 {163				ts = append(ts, &Token{Len, fmt.Sprintf("/* len=%d */", len(data))})164			}165			break166		} else {167			ts = append(ts, &Token{TypeName, v.Type().String()})168		}169		if v.Kind() == reflect.Slice {170			ts = append(ts, &Token{Len, fmt.Sprintf("/* len=%d cap=%d */", v.Len(), v.Cap())})171		}172		ts = append(ts, &Token{SliceOpen, "{"})173		for i := 0; i < v.Len(); i++ {174			p := append(p, i)175			el := v.Index(i)176			ts = append(ts, &Token{SliceItem, ""})177			ts = append(ts, tokenize(sn, p, el)...)178			ts = append(ts, &Token{Comma, ","})179		}180		ts = append(ts, &Token{SliceClose, "}"})181	case reflect.Map:182		ts = append(ts, &Token{TypeName, v.Type().String()})183		keys := v.MapKeys()184		sort.Slice(keys, func(i, j int) bool {185			return Compare(keys[i], keys[j]) < 0186		})187		if len(keys) > 1 {188			ts = append(ts, &Token{Len, fmt.Sprintf("/* len=%d */", len(keys))})189		}190		ts = append(ts, &Token{MapOpen, "{"})191		for _, k := range keys {192			p := append(p, k)193			ts = append(ts, &Token{MapKey, ""})194			ts = append(ts, tokenize(sn, p, k)...)195			ts = append(ts, &Token{Colon, ":"})196			ts = append(ts, tokenize(sn, p, v.MapIndex(k))...)197			ts = append(ts, &Token{Comma, ","})198		}199		ts = append(ts, &Token{MapClose, "}"})200	case reflect.Struct:201		t := v.Type()202		ts = append(ts, &Token{TypeName, t.String()})203		if v.NumField() > 1 {204			ts = append(ts, &Token{Len, fmt.Sprintf("/* len=%d */", v.NumField())})205		}206		ts = append(ts, &Token{StructOpen, "{"})207		for i := 0; i < v.NumField(); i++ {208			name := t.Field(i).Name209			ts = append(ts, &Token{StructKey, ""})210			ts = append(ts, &Token{StructField, name})211			f := v.Field(i)212			if !f.CanInterface() {213				f = GetPrivateField(v, i)214			}215			ts = append(ts, &Token{Colon, ":"})216			ts = append(ts, tokenize(sn, append(p, name), f)...)217			ts = append(ts, &Token{Comma, ","})218		}219		ts = append(ts, &Token{StructClose, "}"})220	case reflect.Bool:221		t.Type = Bool222		if v.Bool() {223			t.Literal = "true"224		} else {225			t.Literal = "false"226		}227		ts = append(ts, t)228	case reflect.Int:229		t.Type = Number230		t.Literal = strconv.FormatInt(v.Int(), 10)231		ts = append(ts, t)232	case reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64,233		reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64,234		reflect.Float32, reflect.Float64,235		reflect.Uintptr:236		ts = append(ts, &Token{TypeName, v.Type().Name()})237		ts = append(ts, &Token{ParenOpen, "("})238		t.Type = Number239		t.Literal = fmt.Sprintf("%v", v.Interface())240		ts = append(ts, t)241		ts = append(ts, &Token{ParenClose, ")"})242	case reflect.Complex64:243		ts = append(ts, &Token{TypeName, v.Type().Name()})244		ts = append(ts, &Token{ParenOpen, "("})245		t.Type = Number246		t.Literal = fmt.Sprintf("%v", v.Interface())247		t.Literal = t.Literal[1 : len(t.Literal)-1]248		ts = append(ts, t)249		ts = append(ts, &Token{ParenClose, ")"})250	case reflect.Complex128:251		t.Type = Number252		t.Literal = fmt.Sprintf("%v", v.Interface())253		t.Literal = t.Literal[1 : len(t.Literal)-1]254		ts = append(ts, t)255	case reflect.String:256		t.Type = String257		t.Literal = fmt.Sprintf("%#v", v.Interface())258		ts = append(ts, t)259		if regNewline.MatchString(v.Interface().(string)) {260			ts = append(ts, &Token{Len, fmt.Sprintf("/* len=%d */", v.Len())})261		}262	case reflect.Chan:263		t.Type = Chan264		if v.Cap() == 0 {265			t.Literal = fmt.Sprintf("make(chan %s)", v.Type().Elem().Name())266		} else {267			t.Literal = fmt.Sprintf("make(chan %s, %d)", v.Type().Elem().Name(), v.Cap())268		}269		ts = append(ts, t)270	case reflect.Func:271		t.Type = Func272		t.Literal = fmt.Sprintf("(%s)(nil)", v.Type().String())273		ts = append(ts, t)274	case reflect.Ptr:275		ts = append(ts, tokenizePtr(sn, p, v)...)276	case reflect.UnsafePointer:277		t.Type = UnsafePointer278		t.Literal = fmt.Sprintf("unsafe.Pointer(uintptr(%v))", v.Interface())279		ts = append(ts, t)280	}281	return ts282}283func tokenizeRune(t *Token, r rune) *Token {284	t.Type = Rune285	t.Literal = fmt.Sprintf("'%s'", string(r))286	return t287}288func tokenizeByte(t *Token, b byte) *Token {289	t.Type = Byte290	if unicode.IsGraphic(rune(b)) {291		t.Literal = fmt.Sprintf("byte('%s')", string(b))292	} else {293		t.Literal = fmt.Sprintf("byte(0x%x)", b)294	}295	return t296}297func tokenizeTime(t time.Time) []*Token {298	ts := []*Token{}299	ts = append(ts, &Token{TypeName, "Time"})300	ts = append(ts, &Token{ParenOpen, "("})301	ts = append(ts, &Token{String, `"` + t.Format(time.RFC3339Nano) + `"`})302	ts = append(ts, &Token{ParenClose, ")"})303	return ts304}305func tokenizeDuration(d time.Duration) []*Token {306	ts := []*Token{}307	ts = append(ts, &Token{TypeName, "Time.Duration"})308	ts = append(ts, &Token{ParenOpen, "("})309	ts = append(ts, &Token{String, `"` + d.String() + `"`})310	ts = append(ts, &Token{ParenClose, ")"})311	return ts312}313func tokenizeBytes(data []byte) []*Token {314	ts := []*Token{}315	if utf8.Valid(data) {316		ts = append(ts, &Token{TypeName, "[]byte"})317		ts = append(ts, &Token{ParenOpen, "("})318		ts = append(ts, &Token{String, fmt.Sprintf("%#v", string(data))})319		ts = append(ts, &Token{ParenClose, ")"})320		return ts321	}322	ts = append(ts, &Token{ParenOpen, "Base64("})323	ts = append(ts, &Token{String, fmt.Sprintf("%#v", base64.StdEncoding.EncodeToString(data))})324	ts = append(ts, &Token{ParenClose, ")"})325	return ts326}327func tokenizePtr(sn seen, p path, v reflect.Value) []*Token {...

1import (2func main() {3    vm := otto.New()4    vm.Set("myfunc", func(call otto.FunctionCall) otto.Value {5        fmt.Println("myfunc called")6        return otto.UndefinedValue()7    })8}9import (10func main() {11    vm := otto.New()12    vm.Set("myfunc", func(call otto.FunctionCall) otto.Value {13        fmt.Println("myfunc called")14        return otto.UndefinedValue()15    })16}17import (18func main() {19    vm := otto.New()20    vm.Set("myfunc", func(call otto.FunctionCall) otto.Value {21        fmt.Println("myfunc called")22        return otto.UndefinedValue()23    })24}25import (26func main() {27    vm := otto.New()28    vm.Set("myfunc", func(call otto.FunctionCall) otto.Value {29        fmt.Println("myfunc called")30        return otto.UndefinedValue()31    })32}33import (34func main() {35    vm := otto.New()36    vm.Set("myfunc", func(call otto.FunctionCall) otto.Value {37        fmt.Println("myfunc called")38        return otto.UndefinedValue()39    })

1import (2func main() {3	g := gop.New()4	tokens := g.TokenizeBytes([]byte("This is a test."))5	fmt.Println(tokens)6}7import (8func main() {9	g := gop.New()10	tokens := g.Tokenize("This is a test.")11	fmt.Println(tokens)12}13import (14func main() {15	g := gop.New()16	tokens := g.Tokenize("This is a test.")17	fmt.Println(tokens)18}19import (20func main() {21	g := gop.New()22	tokens := g.Tokenize("This is a test.")23	fmt.Println(tokens)24}25import (26func main() {27	g := gop.New()28	tokens := g.Tokenize("This is a test.")29	fmt.Println(tokens)30}31import (32func main() {33	g := gop.New()34	tokens := g.Tokenize("This is a test.")35	fmt.Println(tokens)36}37import (38func main() {

1import (2func main() {3	tokens := tokenize.TokenizeBytes([]byte("The quick brown fox jumped over the lazy dog."))4	for _, tok := range tokens {5		fmt.Println(tok)6	}7}

1import (2func main() {3    g := gop.New()4    b := []byte("This is a test")5    g.TokenizeBytes(b)6    fmt.Println(g.GetTokens())7}8import (9func main() {10    g := gop.New()11    g.TokenizeString(s)12    fmt.Println(g.GetTokens())13}14import (15func main() {16    g := gop.New()17    g.TokenizeFile("sample.txt")18    fmt.Println(g.GetTokens())19}20import (21func main() {22    g := gop.New()23    g.TokenizeFile("sample.txt")24    fmt.Println(g.GetTokens())25}26import (27func main() {28    g := gop.New()29    g.TokenizeFile("sample.txt")30    fmt.Println(g.GetTokens())31}32import (

1import (2func main() {3    gop := NewGop()4    gop.Add("abc", "def", "ghi")5    gop.Add("abc", "def", "jkl")6    gop.Add("abc", "def", "mno")7    gop.Add("abc", "def", "pqr")8    gop.Add("abc", "stu", "vwx")9    gop.Add("abc", "stu", "y")10    gop.Add("abc", "stu", "z")11    gop.Add("abc", "stu", "a")12    gop.Add("abc", "stu", "b")13    gop.Add("abc", "stu", "c")14    gop.Add("abc", "stu", "d")15    gop.Add("abc", "stu", "e")16    gop.Add("abc", "stu", "f")17    gop.Add("abc", "stu", "g")18    gop.Add("abc", "stu", "h")19    gop.Add("abc", "stu", "i")20    gop.Add("abc", "stu", "j")21    gop.Add("abc", "stu", "k")22    gop.Add("abc", "stu", "l")23    gop.Add("abc", "stu", "m")24    gop.Add("abc", "stu", "n")25    gop.Add("abc", "stu", "o")26    gop.Add("abc", "stu", "p")27    gop.Add("abc", "stu", "q")28    gop.Add("abc", "stu", "r")29    gop.Add("abc", "stu", "s")30    gop.Add("abc", "stu", "t")31    gop.Add("abc", "stu", "u")32    gop.Add("abc", "stu", "v")33    gop.Add("abc", "stu", "w")34    gop.Add("abc", "stu", "x")35    gop.Add("abc", "stu", "y")36    gop.Add("abc", "stu", "z")37    gop.Add("abc", "stu", "a")38    gop.Add("abc", "stu", "b")

1import (2func main() {3 file, err := os.Open("input.txt")4 if err != nil {5  panic(err)6 }7 defer file.Close()8 scanner := bufio.NewScanner(file)9 for scanner.Scan() {10  line := scanner.Text()11  tokens := gop.TokenizeBytes([]byte(line))12  json, err := json.Marshal(tokens)13  if err != nil {14   panic(err)15  }16  fmt.Println(string(json))17 }18}19["int","a","=","10",";"]20["int","b","=","20",";"]21["int","c","=","a","*","b",";"]22["printf","(","%","d","\\n",",","c",")",";"]23["int","a","=","10",";"]24["int","b","=","20",";"]25["int","c","=","a","*","b",";"]26["printf","(","%","d","\\n",",","c",")",";"]27var tokens = JSON.parse('[["int","a","=","10",";"],["int","b","=","20",";"],["int","c","=","a","*","b",";"],["printf","(","%","d","\\n",",","c",")",";"]]');28console.log(tokens);29[ [ 'int', 'a', '=', '10', ';' ],30  [ 'int', 'b', '=', '20', ';' ],