How to use Tokenize method of diff Package

...7	"testing"8	"github.com/google/go-cmp/cmp"9	"github.com/sam8helloworld/json-go/token"10)11func TestSuccessStringTokenize(t *testing.T) {12	f, err := os.Open("./testdata/string_only.json")13	if err != nil {14		fmt.Println("error")15	}16	defer f.Close()17	// 一気に全部読み取り18	b, err := ioutil.ReadAll(f)19	if err != nil {20		fmt.Println("error")21	}22	sut := NewLexer(string(b))23	got, err := sut.Execute()24	if err != nil {25		t.Fatalf("failed to execute lexer %#v", err)26	}27	want := &[]token.Token{28		token.LeftBraceToken{},29		token.NewStringToken("string"),30		token.ColonToken{},31		token.NewStringToken("hogehoge"),32		token.RightBraceToken{},33	}34	if diff := cmp.Diff(got, want, cmp.AllowUnexported(token.StringToken{})); diff != "" {35		t.Fatalf("got differs: (-got +want)\n%s", diff)36	}37}38func TestSuccessStringTokenizeEscape(t *testing.T) {39	f, err := os.Open("./testdata/escape_string_only.json")40	if err != nil {41		fmt.Println("error")42	}43	defer f.Close()44	// 一気に全部読み取り45	b, err := ioutil.ReadAll(f)46	if err != nil {47		fmt.Println("error")48	}49	sut := NewLexer(string(b))50	got, err := sut.Execute()51	if err != nil {52		t.Fatalf("failed to execute lexer %#v", err)53	}54	want := &[]token.Token{55		token.LeftBraceToken{},56		token.NewStringToken("escape_double_quote"),57		token.ColonToken{},58		token.NewStringToken(`"ダブルクォーテーション"`),59		token.CommaToken{},60		token.NewStringToken("escape_backslash"),61		token.ColonToken{},62		token.NewStringToken(`\バックスラッシュ`),63		token.CommaToken{},64		token.NewStringToken("escape_slash"),65		token.ColonToken{},66		token.NewStringToken(`\/スラッシュ`),67		token.CommaToken{},68		token.NewStringToken("escape_utf_16_text"),69		token.ColonToken{},70		token.NewStringToken("あいうabc"),71		token.CommaToken{},72		token.NewStringToken("escape_utf_16_emoji"),73		token.ColonToken{},74		token.NewStringToken("😄😇👺"),75		token.CommaToken{},76		token.NewStringToken("escape_special_chars"),77		token.ColonToken{},78		token.NewStringToken(` \b \f \n \r \t \/ " `),79		token.RightBraceToken{},80	}81	if diff := cmp.Diff(got, want, cmp.AllowUnexported(token.StringToken{})); diff != "" {82		t.Fatalf("got differs: (-got +want)\n%s", diff)83	}84}85func TestFailedStringTokenize(t *testing.T) {86	f, err := os.Open("./testdata/string_only_fragile.json")87	if err != nil {88		fmt.Println("error")89	}90	defer f.Close()91	// 一気に全部読み取り92	b, err := ioutil.ReadAll(f)93	if err != nil {94		fmt.Println("error")95	}96	sut := NewLexer(string(b))97	got, err := sut.Execute()98	if got != nil {99		t.Errorf("want error %v, but got result %v", ErrStringTokenize, got)100	}101	if !errors.Is(err, ErrStringTokenize) {102		t.Fatalf("want ErrStringTokenize, but got %v", err)103	}104}105func TestSuccessBoolTokenize(t *testing.T) {106	f, err := os.Open("./testdata/bool_only.json")107	if err != nil {108		fmt.Println("error")109	}110	defer f.Close()111	// 一気に全部読み取り112	b, err := ioutil.ReadAll(f)113	if err != nil {114		fmt.Println("error")115	}116	sut := NewLexer(string(b))117	got, err := sut.Execute()118	if err != nil {119		t.Fatalf("failed to execute lexer %#v", err)120	}121	want := &[]token.Token{122		token.LeftBraceToken{},123		token.NewStringToken("boolTrue"),124		token.ColonToken{},125		token.TrueToken{},126		token.CommaToken{},127		token.NewStringToken("boolFalse"),128		token.ColonToken{},129		token.FalseToken{},130		token.RightBraceToken{},131	}132	if diff := cmp.Diff(got, want, cmp.AllowUnexported(token.StringToken{})); diff != "" {133		t.Fatalf("got differs: (-got +want)\n%s", diff)134	}135}136func TestFailedBoolTokenize(t *testing.T) {137	f, err := os.Open("./testdata/bool_only_fragile.json")138	if err != nil {139		fmt.Println("error")140	}141	defer f.Close()142	// 一気に全部読み取り143	b, err := ioutil.ReadAll(f)144	if err != nil {145		fmt.Println("error")146	}147	sut := NewLexer(string(b))148	got, err := sut.Execute()149	if got != nil {150		t.Errorf("want error %v, but got result %v", ErrBoolTokenize, got)151	}152	if !errors.Is(err, ErrBoolTokenize) {153		t.Fatalf("want ErrBoolTokenize, but got %v", err)154	}155}156func TestSuccessNullTokenize(t *testing.T) {157	f, err := os.Open("./testdata/null_only.json")158	if err != nil {159		fmt.Println("error")160	}161	defer f.Close()162	// 一気に全部読み取り163	b, err := ioutil.ReadAll(f)164	if err != nil {165		fmt.Println("error")166	}167	sut := NewLexer(string(b))168	got, err := sut.Execute()169	if err != nil {170		t.Fatalf("failed to execute lexer %#v", err)171	}172	want := &[]token.Token{173		token.LeftBraceToken{},174		token.NewStringToken("null"),175		token.ColonToken{},176		token.NullToken{},177		token.RightBraceToken{},178	}179	if diff := cmp.Diff(got, want, cmp.AllowUnexported(token.StringToken{})); diff != "" {180		t.Fatalf("got differs: (-got +want)\n%s", diff)181	}182}183func TestFailedNullTokenize(t *testing.T) {184	f, err := os.Open("./testdata/null_only_fragile.json")185	if err != nil {186		fmt.Println("error")187	}188	defer f.Close()189	// 一気に全部読み取り190	b, err := ioutil.ReadAll(f)191	if err != nil {192		fmt.Println("error")193	}194	sut := NewLexer(string(b))195	got, err := sut.Execute()196	if got != nil {197		t.Errorf("want error %v, but got result %v", ErrNullTokenize, got)198	}199	if !errors.Is(err, ErrNullTokenize) {200		t.Fatalf("want ErrNullTokenize, but got %v", err)201	}202}203func TestSuccessNumberTokenize(t *testing.T) {204	f, err := os.Open("./testdata/number_only.json")205	if err != nil {206		fmt.Println("error")207	}208	defer f.Close()209	// 一気に全部読み取り210	b, err := ioutil.ReadAll(f)211	if err != nil {212		fmt.Println("error")213	}214	sut := NewLexer(string(b))215	got, err := sut.Execute()216	if err != nil {217		t.Fatalf("failed to execute lexer %#v", err)...

...20`)21}22func TestFormat(t *testing.T) {23	g := setup(t)24	ts := diff.TokenizeText(25		g.Context(),26		strings.ReplaceAll("a b c d f g h h j q z", " ", "\n"),27		strings.ReplaceAll("a b c d e f g i j k r x y z", " ", "\n"),28	)29	df := diff.Format(ts, diff.ThemeNone)30	g.Eq(df, ""+31		"01 01   a\n"+32		"02 02   b\n"+33		"03 03   c\n"+34		"04 04   d\n"+35		"   05 + e\n"+36		"05 06   f\n"+37		"06 07   g\n"+38		"07    - h\n"+39		"08    - h\n"+40		"   08 + i\n"+41		"09 09   j\n"+42		"10    - q\n"+43		"   10 + k\n"+44		"   11 + r\n"+45		"   12 + x\n"+46		"   13 + y\n"+47		"11 14   z\n"+48		"")49}50func TestDisconnectedChunks(t *testing.T) {51	g := setup(t)52	ts := diff.TokenizeText(53		g.Context(),54		strings.ReplaceAll("a b c d f g h i j k l m n", " ", "\n"),55		strings.ReplaceAll("x b c d f g h i x k l m n", " ", "\n"),56	)57	lines := diff.ParseTokenLines(ts)58	lines = diff.Narrow(1, lines)59	ts = diff.SpreadTokenLines(lines)60	df := diff.Format(ts, diff.ThemeNone)61	g.Eq(df, ""+62		"@@ diff chunk @@\n"+63		"01    - a\n"+64		"   01 + x\n"+65		"02 02   b\n"+66		"\n"+67		"@@ diff chunk @@\n"+68		"08 08   i\n"+69		"09    - j\n"+70		"   09 + x\n"+71		"10 10   k\n"+72		"\n"+73		"")74}75func TestChunks0(t *testing.T) {76	g := setup(t)77	ts := diff.TokenizeText(78		g.Context(),79		strings.ReplaceAll("a b c", " ", "\n"),80		strings.ReplaceAll("a x c", " ", "\n"),81	)82	lines := diff.ParseTokenLines(ts)83	lines = diff.Narrow(-1, lines)84	ts = diff.SpreadTokenLines(lines)85	df := diff.Format(ts, diff.ThemeNone)86	g.Eq(df, ""+87		"@@ diff chunk @@\n"+88		"2   - b\n"+89		"  2 + x\n"+90		"\n"+91		"")92}93func TestNoDifference(t *testing.T) {94	g := setup(t)95	ts := diff.TokenizeText(g.Context(), "a", "b")96	df := diff.Format(ts, diff.ThemeNone)97	g.Eq(df, ""+98		"1   - a\n"+99		"  1 + b\n"+100		"")101}102func TestTwoLines(t *testing.T) {103	g := setup(t)104	format := func(ts []*diff.Token) string {105		out := ""106		for _, t := range ts {107			txt := strings.TrimSpace(strings.ReplaceAll(t.Literal, "", " "))108			switch t.Type {109			case diff.DelWords:110				out += "-" + txt111			case diff.AddWords:112				out += "+" + txt113			default:114				out += "=" + txt115			}116		}117		return out118	}119	check := func(x, y, ex, ey string) {120		t.Helper()121		tx, ty := diff.TokenizeLine(g.Context(),122			strings.ReplaceAll(x, " ", ""),123			strings.ReplaceAll(y, " ", ""))124		dx, dy := format(tx), format(ty)125		if dx != ex || dy != ey {126			t.Error("\n", dx, "\n", dy, "\n!=\n", ex, "\n", ey)127		}128	}129	check(130		" a b c d f g h i j k l m n",131		" x x b c d f g h i x k l m n",132		"-a=b c d f g h i-j=k l m n",133		"+x x=b c d f g h i+x=k l m n",134	)135	check(136		" 4 9 0 4 5 0 8 8 5 3",137		" 4 9 0 5 4 3 7 5 2",138		"=4 9 0 4-5 0 8 8 5=3",139		"=4 9 0+5=4 3+7 5 2",140	)141	check(142		" 4 9 0 4 5 0 8",143		" 4 9 0 5 4 3 7",144		"=4 9 0-4=5-0 8",145		"=4 9 0 5+4 3 7",146	)147}148func TestColor(t *testing.T) {149	g := setup(t)150	out := diff.Diff("abc", "axc")151	g.Eq(gop.VisualizeANSI(out), `<45><30>@@ diff chunk @@<39><49>152<31>1   -<39> a<31>b<39>c153<32>  1 +<39> a<32>x<39>c154`)155}156func TestCustomSplit(t *testing.T) {157	g := setup(t)158	ctx := context.WithValue(g.Context(), diff.SplitKey, split)159	g.Eq(diff.TokenizeLine(ctx, "abc", "abc"))160}...

...29// Diff x and y into a human readable string.30func Diff(x, y string) string {31	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)32	defer cancel()33	return Format(Tokenize(ctx, x, y), ThemeDefault)34}35// Tokenize x and y into diff tokens with diff words and narrow chunks.36func Tokenize(ctx context.Context, x, y string) []*Token {37	ts := TokenizeText(ctx, x, y)38	lines := ParseTokenLines(ts)39	lines = Narrow(1, lines)40	Words(ctx, lines)41	return SpreadTokenLines(lines)42}43// Format tokens into a human readable string44func Format(ts []*Token, theme Theme) string {45	out := ""46	for _, t := range ts {47		s := t.Literal48		out += gop.Stylize(s, theme(t.Type))49	}50	return out51}52// Narrow the context around each diff section to n lines.53func Narrow(n int, lines []*TokenLine) []*TokenLine {54	if n < 0 {55		n = 056	}57	keep := map[int]bool{}58	for i, l := range lines {59		switch l.Type {60		case AddSymbol, DelSymbol:61			for j := max(i-n, 0); j <= i+n && j < len(lines); j++ {62				keep[j] = true63			}64		}65	}66	out := []*TokenLine{}67	for i, l := range lines {68		if !keep[i] {69			continue70		}71		if _, has := keep[i-1]; !has {72			ts := []*Token{{ChunkStart, "@@ diff chunk @@"}, {Newline, "\n"}}73			out = append(out, &TokenLine{ChunkStart, ts})74		}75		out = append(out, l)76		if _, has := keep[i+1]; !has {77			ts := []*Token{{ChunkEnd, ""}, {Newline, "\n"}}78			out = append(out, &TokenLine{ChunkEnd, ts})79		}80	}81	return out82}83// Words diff84func Words(ctx context.Context, lines []*TokenLine) {85	delLines := []*TokenLine{}86	addLines := []*TokenLine{}87	df := func() {88		if len(delLines) == 0 || len(delLines) != len(addLines) {89			return90		}91		for i := 0; i < len(delLines); i++ {92			d := delLines[i]93			a := addLines[i]94			dts, ats := TokenizeLine(ctx, d.Tokens[2].Literal, a.Tokens[2].Literal)95			d.Tokens = append(d.Tokens[0:2], append(dts, d.Tokens[3:]...)...)96			a.Tokens = append(a.Tokens[0:2], append(ats, a.Tokens[3:]...)...)97		}98		delLines = []*TokenLine{}99		addLines = []*TokenLine{}100	}101	for _, l := range lines {102		switch l.Type {103		case DelSymbol:104			delLines = append(delLines, l)105		case AddSymbol:106			addLines = append(addLines, l)107		default:108			df()...

1import (2func main() {3	dmp := diffmatchpatch.New()4	diffs := dmp.DiffMain("Hello World", "Hello Gophers", false)5	fmt.Println(diffs)6}7[{1 Hello } {- World} {1  Gophers}]8import (9func main() {10	dmp := diffmatchpatch.New()11	diffs := dmp.DiffMain("Hello World", "Hello Gophers", false)12	fmt.Println(dmp.DiffPrettyText(diffs))13}14import (15func main() {16	dmp := diffmatchpatch.New()17	diffs := dmp.DiffMain("Hello World", "Hello Gophers", false)18	fmt.Println(dmp.DiffPrettyHtml(diffs))19}20import (21func main() {22	dmp := diffmatchpatch.New()23	diffs := dmp.DiffMain("Hello World", "Hello Gophers", false)24	fmt.Println(dmp.DiffText1(diffs))25}26import (27func main() {28	dmp := diffmatchpatch.New()29	diffs := dmp.DiffMain("Hello World", "Hello Gophers", false)30	fmt.Println(dmp.DiffText2(diffs))31}32import (33func main() {34	dmp := diffmatchpatch.New()35	diffs := dmp.DiffMain("Hello World

1import (2func main() {3	dmp := diffmatchpatch.New()4	diffs := dmp.DiffMain(a, b, false)5	fmt.Println(dmp.DiffPrettyText(diffs))6}

1import (2func main() {3    diff := diffmatchpatch.New()4    d := diff.DiffMain("Hello World", "Hello Golang", false)5    fmt.Println(diff.DiffPrettyText(d))6    fmt.Println(diff.DiffPrettyHtml(d))7    fmt.Println(diff.DiffToDelta(d))8    fmt.Println(diff.DiffFromDelta("Hello World", diff.DiffToDelta(d)))9    fmt.Println(diff.DiffToText(d))10    fmt.Println(diff.DiffFromText("Hello World", diff.DiffToText(d)))11    fmt.Println(diff.DiffLinesToChars("Hello World", "Hello Golang"))12    fmt.Println(diff.DiffCharsToLines(diff.DiffLinesToChars("Hello World", "Hello Golang"), "Hello World", "Hello Golang"))13    fmt.Println(diff.DiffCommonPrefix("Hello World", "Hello Golang"))14    fmt.Println(diff.DiffCommonSuffix("Hello World", "Hello Golang"))15    fmt.Println(diff.DiffCommonOverlap("Hello World", "Hello Golang"))16    fmt.Println(diff.DiffHalfMatch("Hello World", "Hello Golang"))17    fmt.Println(diff.DiffLinesToChars("Hello World", "Hello Golang"))18    fmt.Println(diff.DiffCharsToLines(diff.DiffLinesToChars("Hello World", "Hello Golang"), "Hello World", "Hello Golang"))19    fmt.Println(diff.DiffWordsToChars("Hello World", "Hello Golang"))20    fmt.Println(diff.DiffCharsToWords(diff.DiffWordsToChars("Hello World", "Hello Golang"), "Hello World", "Hello Golang"))21    fmt.Println(diff.DiffCleanupMerge(d))22    fmt.Println(diff.DiffCleanupSemantic(d))23    fmt.Println(diff.DiffCleanupSemanticLossless(d))24    fmt.Println(diff.DiffCleanupEfficiency(d))25    fmt.Println(diff.DiffXIndex(d, 2))26    fmt.Println(diff.DiffLevenshtein(d))27    fmt.Println(diff.DiffBisect("Hello World", "Hello Golang", 1))28    fmt.Println(diff.DiffBisectSplit("Hello World", "Hello Golang", 1))29    fmt.Println(diff.DiffMain("Hello World", "Hello Golang", true))30    fmt.Println(diff.DiffMainRunes([]rune("Hello World"), []rune("Hello Golang"), true))31    fmt.Println(diff.DiffMainRunes([]rune("Hello World"), []rune("Hello G

1import (2func main() {3    dmp := diffmatchpatch.New()4    d := dmp.New()5    dmp.DiffCleanupSemantic(d)6    dmp.DiffCleanupEfficiency(d)7    fmt.Println(dmp.DiffPrettyText(d))8}9Example 2: DMP.DiffPrettyText()10import (11func main() {12    dmp := diffmatchpatch.New()13    d := dmp.New()14    dmp.DiffCleanupSemantic(d)15    dmp.DiffCleanupEfficiency(d)16    fmt.Println(dmp.DiffPrettyText(d))17}18Example 3: DMP.DiffToDelta()19import (20func main() {21    dmp := diffmatchpatch.New()22    d := dmp.New()23    dmp.DiffCleanupSemantic(d)24    dmp.DiffCleanupEfficiency(d)25    fmt.Println(dmp.DiffToDelta(d))26}27Example 4: DMP.DiffFromDelta()28import (29func main() {30    dmp := diffmatchpatch.New()31    d := dmp.New()32    dmp.DiffCleanupSemantic(d)33    dmp.DiffCleanupEfficiency(d)34    fmt.Println(dmp.DiffFromDelta("Hello World", "=4"))35}36Example 5: DMP.DiffXIndex()37import (38func main() {39    dmp := diffmatchpatch.New()40    d := dmp.New()41    dmp.DiffCleanupSemantic(d)

1import (2func main() {3	dmp := diffmatchpatch.New()4	diff := dmp.DiffMain("Hello World!", "Hello Go World!", false)5	fmt.Println(diff)6}7[{1 Hello } {0 World!} {-  } {1 Go } {0 World!}]8import (9func main() {10	dmp := diffmatchpatch.New()11	diff := dmp.DiffMain("Hello World!", "Hello Go World!", false)12	fmt.Println(diff)13}14[{1 Hello } {0 World!} {-  } {1 Go } {0 World!}]15import (16func main() {17	dmp := diffmatchpatch.New()18	diff := dmp.DiffMain("Hello World!", "Hello Go World!", false)19	fmt.Println(diff)20}21[{1 Hello } {0 World!} {-  } {1 Go } {0 World!}]22import (23func main() {24	dmp := diffmatchpatch.New()25	diff := dmp.DiffMain("Hello World!", "Hello Go World!", false)26	fmt.Println(diff)27}28[{1 Hello } {0 World!} {-  } {1 Go } {0 World!}]29import (30func main() {31	dmp := diffmatchpatch.New()32	diff := dmp.DiffMain("Hello World!", "Hello Go World!", false)33	fmt.Println(diff)34}35[{1 Hello } {0 World!} {-  } {1 Go } {0 World!}]

1import (2func main() {3	dmp := diffmatchpatch.New()4	diffs := dmp.DiffMain(a, b, false)5	fmt.Println(diffs)6}7[{1 Hello world.} {- } {1 !}]8[{1 Hello world.} {- } {1 !}]9import (10func main() {11	dmp := diffmatchpatch.New()12	diffs := dmp.DiffMain(a, b, false)13	for _, diff := range diffs {14		if diff.Type == 1 {15			fmt.Println(diff.Text)16		}17	}18}19import (20func main() {21	dmp := diffmatchpatch.New()22	diffs := dmp.DiffMain(a, b, false)23	for _, diff := range diffs {24		if diff.Type == -1 {25			fmt.Println(diff.Text)26		}27	}28}29import (30func main() {31	dmp := diffmatchpatch.New()

1import (2func main() {3dmp := diffmatchpatch.New()4diff := dmp.DiffMain(a, b, false)5diffs := dmp.DiffCleanupSemantic(diff)6fmt.Println(dmp.DiffPrettyText(diffs))7}8import (9func main() {10dmp := diffmatchpatch.New()11diff := dmp.DiffMain(a, b, false)12diffs := dmp.DiffCleanupSemantic(diff)13fmt.Println(dmp.DiffPrettyText(diffs))14}15import (16func main() {17dmp := diffmatchpatch.New()18diff := dmp.DiffMain(a, b, false)19diffs := dmp.DiffCleanupSemantic(diff)20fmt.Println(dmp.DiffPrettyText(diffs))21}22import (23func main() {24dmp := diffmatchpatch.New()25diff := dmp.DiffMain(a, b, false)26diffs := dmp.DiffCleanupSemantic(diff)27fmt.Println(dmp.DiffPrettyText(diffs))28}29import (30func main() {31dmp := diffmatchpatch.New()32diff := dmp.DiffMain(a, b, false)33diffs := dmp.DiffCleanupSemantic(diff)34fmt.Println(dmp.DiffPrettyText(diffs))35}

1import (2type diff struct {3}4func (d *diff) Tokenize() (string, string) {5    return strings.Split(d.a, ""), strings.Split(d.b, "")6}7func (d *diff) EditDistance() int {8    a, b := d.Tokenize()9    m, n := len(a), len(b)10    dp := make([][]int, m+1)11    for i := range dp {12        dp[i] = make([]int, n+1)13    }14    for i := 0; i <= m; i++ {15    }16    for j := 0; j <= n; j++ {17    }18    for i := 1; i <= m; i++ {19        for j := 1; j <= n; j++ {20            dp[i][j] = int(math.Min(float64(dp[i-1][j]+1), math.Min(float64(dp[i][j-1]+1), float64(dp[i-1][j-1]+int(math.Abs(float64(a[i-1][0]-b[j-1][0])))))))21        }22    }23}24func main() {25    fmt.Println("Enter the number of test cases")26    fmt.Scanln(&n)27    for i := 0; i < n; i++ {28        fmt.Println("Enter the first string")29        fmt.Scanln(&str1)30        fmt.Println("Enter the second string")31        fmt.Scanln(&str2)32        d := diff{str1, str2}33        fmt.Println(d.EditDistance())34    }35}