How to use SpreadTokenLines method of diff Package

1package diff_test
2
3import (
4	"context"
5	"strings"
6	"testing"
7
8	"github.com/ysmood/got"
9	"github.com/ysmood/got/lib/diff"
10	"github.com/ysmood/got/lib/gop"
11)
12
13var setup = got.Setup(func(g got.G) {
14	g.ErrorHandler = got.NewDefaultAssertionError(nil, nil)
15})
16
17func split(s string) []string { return strings.Split(s, "") }
18
19func TestDiff(t *testing.T) {
20	g := setup(t)
21
22	out := gop.StripANSI(diff.Diff("abc", "axc"))
23
24	g.Eq(out, `@@ diff chunk @@
251   - abc
26  1 + axc
27
28`)
29}
30
31func TestFormat(t *testing.T) {
32	g := setup(t)
33	ts := diff.TokenizeText(
34		g.Context(),
35		strings.ReplaceAll("a b c d f g h h j q z", " ", "\n"),
36		strings.ReplaceAll("a b c d e f g i j k r x y z", " ", "\n"),
37	)
38
39	df := diff.Format(ts, diff.ThemeNone)
40
41	g.Eq(df, ""+
42		"01 01   a\n"+
43		"02 02   b\n"+
44		"03 03   c\n"+
45		"04 04   d\n"+
46		"   05 + e\n"+
47		"05 06   f\n"+
48		"06 07   g\n"+
49		"07    - h\n"+
50		"08    - h\n"+
51		"   08 + i\n"+
52		"09 09   j\n"+
53		"10    - q\n"+
54		"   10 + k\n"+
55		"   11 + r\n"+
56		"   12 + x\n"+
57		"   13 + y\n"+
58		"11 14   z\n"+
59		"")
60}
61
62func TestDisconnectedChunks(t *testing.T) {
63	g := setup(t)
64	ts := diff.TokenizeText(
65		g.Context(),
66		strings.ReplaceAll("a b c d f g h i j k l m n", " ", "\n"),
67		strings.ReplaceAll("x b c d f g h i x k l m n", " ", "\n"),
68	)
69
70	lines := diff.ParseTokenLines(ts)
71	lines = diff.Narrow(1, lines)
72	ts = diff.SpreadTokenLines(lines)
73
74	df := diff.Format(ts, diff.ThemeNone)
75
76	g.Eq(df, ""+
77		"@@ diff chunk @@\n"+
78		"01    - a\n"+
79		"   01 + x\n"+
80		"02 02   b\n"+
81		"\n"+
82		"@@ diff chunk @@\n"+
83		"08 08   i\n"+
84		"09    - j\n"+
85		"   09 + x\n"+
86		"10 10   k\n"+
87		"\n"+
88		"")
89}
90
91func TestChunks0(t *testing.T) {
92	g := setup(t)
93	ts := diff.TokenizeText(
94		g.Context(),
95		strings.ReplaceAll("a b c", " ", "\n"),
96		strings.ReplaceAll("a x c", " ", "\n"),
97	)
98
99	lines := diff.ParseTokenLines(ts)
100	lines = diff.Narrow(-1, lines)
101	ts = diff.SpreadTokenLines(lines)
102
103	df := diff.Format(ts, diff.ThemeNone)
104
105	g.Eq(df, ""+
106		"@@ diff chunk @@\n"+
107		"2   - b\n"+
108		"  2 + x\n"+
109		"\n"+
110		"")
111}
112
113func TestNoDifference(t *testing.T) {
114	g := setup(t)
115	ts := diff.TokenizeText(g.Context(), "a", "b")
116
117	df := diff.Format(ts, diff.ThemeNone)
118
119	g.Eq(df, ""+
120		"1   - a\n"+
121		"  1 + b\n"+
122		"")
123}
124
125func TestTwoLines(t *testing.T) {
126	g := setup(t)
127
128	format := func(ts []*diff.Token) string {
129		out := ""
130		for _, t := range ts {
131			txt := strings.TrimSpace(strings.ReplaceAll(t.Literal, "", " "))
132			switch t.Type {
133			case diff.DelWords:
134				out += "-" + txt
135			case diff.AddWords:
136				out += "+" + txt
137			default:
138				out += "=" + txt
139			}
140		}
141		return out
142	}
143
144	check := func(x, y, ex, ey string) {
145		t.Helper()
146
147		tx, ty := diff.TokenizeLine(g.Context(),
148			strings.ReplaceAll(x, " ", ""),
149			strings.ReplaceAll(y, " ", ""))
150		dx, dy := format(tx), format(ty)
151
152		if dx != ex || dy != ey {
153			t.Error("\n", dx, "\n", dy, "\n!=\n", ex, "\n", ey)
154		}
155	}
156
157	check(
158		" a b c d f g h i j k l m n",
159		" x x b c d f g h i x k l m n",
160		"-a=b c d f g h i-j=k l m n",
161		"+x x=b c d f g h i+x=k l m n",
162	)
163
164	check(
165		" 4 9 0 4 5 0 8 8 5 3",
166		" 4 9 0 5 4 3 7 5 2",
167		"=4 9 0 4-5 0 8 8 5=3",
168		"=4 9 0+5=4 3+7 5 2",
169	)
170
171	check(
172		" 4 9 0 4 5 0 8",
173		" 4 9 0 5 4 3 7",
174		"=4 9 0-4=5-0 8",
175		"=4 9 0 5+4 3 7",
176	)
177}
178
179func TestColor(t *testing.T) {
180	g := setup(t)
181
182	out := diff.Diff("abc", "axc")
183
184	g.Eq(gop.VisualizeANSI(out), `<45><30>@@ diff chunk @@<39><49>
185<31>1   -<39> a<31>b<39>c
186<32>  1 +<39> a<32>x<39>c
187
188`)
189}
190
191func TestCustomSplit(t *testing.T) {
192	g := setup(t)
193
194	ctx := context.WithValue(g.Context(), diff.SplitKey, split)
195
196	g.Eq(diff.TokenizeLine(ctx, "abc", "abc"))
197}
198

Source: format_test.go

package diff_test

import (
	"context"
	"strings"
	"testing"

	"github.com/ysmood/got"
	"github.com/ysmood/got/lib/diff"
	"github.com/ysmood/got/lib/gop"
)

var setup = got.Setup(func(g got.G) {
	g.ErrorHandler = got.NewDefaultAssertionError(nil, nil)
})

func split(s string) []string { return strings.Split(s, "") }

func TestDiff(t *testing.T) {
	g := setup(t)

	out := gop.StripANSI(diff.Diff("abc", "axc"))

	g.Eq(out, `@@ diff chunk @@
1   - abc
  1 + axc

`)
}

func TestFormat(t *testing.T) {
	g := setup(t)
	ts := diff.TokenizeText(
		g.Context(),
		strings.ReplaceAll("a b c d f g h h j q z", " ", "\n"),
		strings.ReplaceAll("a b c d e f g i j k r x y z", " ", "\n"),
	)

	df := diff.Format(ts, diff.ThemeNone)

	g.Eq(df, ""+
		"01 01   a\n"+
		"02 02   b\n"+
		"03 03   c\n"+
		"04 04   d\n"+
		"   05 + e\n"+
		"05 06   f\n"+
		"06 07   g\n"+
		"07    - h\n"+
		"08    - h\n"+
		"   08 + i\n"+
		"09 09   j\n"+
		"10    - q\n"+
		"   10 + k\n"+
		"   11 + r\n"+
		"   12 + x\n"+
		"   13 + y\n"+
		"11 14   z\n"+
		"")
}

func TestDisconnectedChunks(t *testing.T) {
	g := setup(t)
	ts := diff.TokenizeText(
		g.Context(),
		strings.ReplaceAll("a b c d f g h i j k l m n", " ", "\n"),
		strings.ReplaceAll("x b c d f g h i x k l m n", " ", "\n"),
	)

	lines := diff.ParseTokenLines(ts)
	lines = diff.Narrow(1, lines)
	ts = diff.SpreadTokenLines(lines)

	df := diff.Format(ts, diff.ThemeNone)

	g.Eq(df, ""+
		"@@ diff chunk @@\n"+
		"01    - a\n"+
		"   01 + x\n"+
		"02 02   b\n"+
		"\n"+
		"@@ diff chunk @@\n"+
		"08 08   i\n"+
		"09    - j\n"+
		"   09 + x\n"+
		"10 10   k\n"+
		"\n"+
		"")
}

func TestChunks0(t *testing.T) {
	g := setup(t)
	ts := diff.TokenizeText(
		g.Context(),
		strings.ReplaceAll("a b c", " ", "\n"),
		strings.ReplaceAll("a x c", " ", "\n"),
	)

	lines := diff.ParseTokenLines(ts)
	lines = diff.Narrow(-1, lines)
	ts = diff.SpreadTokenLines(lines)

	df := diff.Format(ts, diff.ThemeNone)

	g.Eq(df, ""+
		"@@ diff chunk @@\n"+
		"2   - b\n"+
		"  2 + x\n"+
		"\n"+
		"")
}

func TestNoDifference(t *testing.T) {
	g := setup(t)
	ts := diff.TokenizeText(g.Context(), "a", "b")

	df := diff.Format(ts, diff.ThemeNone)

	g.Eq(df, ""+
		"1   - a\n"+
		"  1 + b\n"+
		"")
}

func TestTwoLines(t *testing.T) {
	g := setup(t)

	format := func(ts []*diff.Token) string {
		out := ""
		for _, t := range ts {
			txt := strings.TrimSpace(strings.ReplaceAll(t.Literal, "", " "))
			switch t.Type {
			case diff.DelWords:
				out += "-" + txt
			case diff.AddWords:
				out += "+" + txt
			default:
				out += "=" + txt
			}
		}
		return out
	}

	check := func(x, y, ex, ey string) {
		t.Helper()

		tx, ty := diff.TokenizeLine(g.Context(),
			strings.ReplaceAll(x, " ", ""),
			strings.ReplaceAll(y, " ", ""))
		dx, dy := format(tx), format(ty)

		if dx != ex || dy != ey {
			t.Error("\n", dx, "\n", dy, "\n!=\n", ex, "\n", ey)
		}
	}

	check(
		" a b c d f g h i j k l m n",
		" x x b c d f g h i x k l m n",
		"-a=b c d f g h i-j=k l m n",
		"+x x=b c d f g h i+x=k l m n",
	)

	check(
		" 4 9 0 4 5 0 8 8 5 3",
		" 4 9 0 5 4 3 7 5 2",
		"=4 9 0 4-5 0 8 8 5=3",
		"=4 9 0+5=4 3+7 5 2",
	)

	check(
		" 4 9 0 4 5 0 8",
		" 4 9 0 5 4 3 7",
		"=4 9 0-4=5-0 8",
		"=4 9 0 5+4 3 7",
	)
}

func TestColor(t *testing.T) {
	g := setup(t)

	out := diff.Diff("abc", "axc")

	g.Eq(gop.VisualizeANSI(out), `<45><30>@@ diff chunk @@<39><49>
<31>1   -<39> a<31>b<39>c
<32>  1 +<39> a<32>x<39>c

`)
}

func TestCustomSplit(t *testing.T) {
	g := setup(t)

	ctx := context.WithValue(g.Context(), diff.SplitKey, split)

	g.Eq(diff.TokenizeLine(ctx, "abc", "abc"))
}

1package diff
2
3// TokenLine of tokens
4type TokenLine struct {
5	Type   Type
6	Tokens []*Token
7}
8
9// ParseTokenLines of tokens
10func ParseTokenLines(ts []*Token) []*TokenLine {
11	list := []*TokenLine{}
12	var l *TokenLine
13	for _, t := range ts {
14		switch t.Type {
15		case SameSymbol, AddSymbol, DelSymbol:
16			l = &TokenLine{}
17			list = append(list, l)
18			l.Type = t.Type
19		}
20		l.Tokens = append(l.Tokens, t)
21	}
22	return list
23}
24
25// SpreadTokenLines to tokens
26func SpreadTokenLines(lines []*TokenLine) []*Token {
27	out := []*Token{}
28	for _, l := range lines {
29		out = append(out, l.Tokens...)
30	}
31	return out
32}
33

Source: ast.go

package diff

// TokenLine of tokens
type TokenLine struct {
	Type   Type
	Tokens []*Token
}

// ParseTokenLines of tokens
func ParseTokenLines(ts []*Token) []*TokenLine {
	list := []*TokenLine{}
	var l *TokenLine
	for _, t := range ts {
		switch t.Type {
		case SameSymbol, AddSymbol, DelSymbol:
			l = &TokenLine{}
			list = append(list, l)
			l.Type = t.Type
		}
		l.Tokens = append(l.Tokens, t)
	}
	return list
}

// SpreadTokenLines to tokens
func SpreadTokenLines(lines []*TokenLine) []*Token {
	out := []*Token{}
	for _, l := range lines {
		out = append(out, l.Tokens...)
	}
	return out
}

1package diff
2
3import (
4	"context"
5	"time"
6
7	"github.com/ysmood/got/lib/gop"
8)
9
10// Theme for diff
11type Theme func(t Type) []gop.Style
12
13// ThemeDefault colors for Sprint
14var ThemeDefault = func(t Type) []gop.Style {
15	switch t {
16	case AddSymbol:
17		return []gop.Style{gop.Green}
18	case DelSymbol:
19		return []gop.Style{gop.Red}
20	case AddWords:
21		return []gop.Style{gop.Green}
22	case DelWords:
23		return []gop.Style{gop.Red}
24	case ChunkStart:
25		return []gop.Style{gop.Black, gop.BgMagenta}
26	}
27	return []gop.Style{gop.None}
28}
29
30// ThemeNone colors for Sprint
31var ThemeNone = func(t Type) []gop.Style {
32	return []gop.Style{gop.None}
33}
34
35// Diff x and y into a human readable string.
36func Diff(x, y string) string {
37	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
38	defer cancel()
39	return Format(Tokenize(ctx, x, y), ThemeDefault)
40}
41
42// Tokenize x and y into diff tokens with diff words and narrow chunks.
43func Tokenize(ctx context.Context, x, y string) []*Token {
44	ts := TokenizeText(ctx, x, y)
45	lines := ParseTokenLines(ts)
46	lines = Narrow(1, lines)
47	Words(ctx, lines)
48	return SpreadTokenLines(lines)
49}
50
51// Format tokens into a human readable string
52func Format(ts []*Token, theme Theme) string {
53	out := ""
54
55	for _, t := range ts {
56		s := t.Literal
57		out += gop.Stylize(s, theme(t.Type))
58	}
59
60	return out
61}
62
63// Narrow the context around each diff section to n lines.
64func Narrow(n int, lines []*TokenLine) []*TokenLine {
65	if n < 0 {
66		n = 0
67	}
68
69	keep := map[int]bool{}
70	for i, l := range lines {
71		switch l.Type {
72		case AddSymbol, DelSymbol:
73			for j := max(i-n, 0); j <= i+n && j < len(lines); j++ {
74				keep[j] = true
75			}
76		}
77	}
78
79	out := []*TokenLine{}
80	for i, l := range lines {
81		if !keep[i] {
82			continue
83		}
84
85		if _, has := keep[i-1]; !has {
86			ts := []*Token{{ChunkStart, "@@ diff chunk @@"}, {Newline, "\n"}}
87			out = append(out, &TokenLine{ChunkStart, ts})
88		}
89
90		out = append(out, l)
91
92		if _, has := keep[i+1]; !has {
93			ts := []*Token{{ChunkEnd, ""}, {Newline, "\n"}}
94			out = append(out, &TokenLine{ChunkEnd, ts})
95		}
96	}
97
98	return out
99}
100
101// Words diff
102func Words(ctx context.Context, lines []*TokenLine) {
103	delLines := []*TokenLine{}
104	addLines := []*TokenLine{}
105
106	df := func() {
107		if len(delLines) == 0 || len(delLines) != len(addLines) {
108			return
109		}
110
111		for i := 0; i < len(delLines); i++ {
112			d := delLines[i]
113			a := addLines[i]
114
115			dts, ats := TokenizeLine(ctx, d.Tokens[2].Literal, a.Tokens[2].Literal)
116			d.Tokens = append(d.Tokens[0:2], append(dts, d.Tokens[3:]...)...)
117			a.Tokens = append(a.Tokens[0:2], append(ats, a.Tokens[3:]...)...)
118		}
119
120		delLines = []*TokenLine{}
121		addLines = []*TokenLine{}
122	}
123
124	for _, l := range lines {
125		switch l.Type {
126		case DelSymbol:
127			delLines = append(delLines, l)
128		case AddSymbol:
129			addLines = append(addLines, l)
130		default:
131			df()
132		}
133	}
134
135	df()
136}
137
138func max(x, y int) int {
139	if x < y {
140		return y
141	}
142	return x
143}
144

Source: format.go

package diff

import (
	"context"
	"time"

	"github.com/ysmood/got/lib/gop"
)

// Theme for diff
type Theme func(t Type) []gop.Style

// ThemeDefault colors for Sprint
var ThemeDefault = func(t Type) []gop.Style {
	switch t {
	case AddSymbol:
		return []gop.Style{gop.Green}
	case DelSymbol:
		return []gop.Style{gop.Red}
	case AddWords:
		return []gop.Style{gop.Green}
	case DelWords:
		return []gop.Style{gop.Red}
	case ChunkStart:
		return []gop.Style{gop.Black, gop.BgMagenta}
	}
	return []gop.Style{gop.None}
}

// ThemeNone colors for Sprint
var ThemeNone = func(t Type) []gop.Style {
	return []gop.Style{gop.None}
}

// Diff x and y into a human readable string.
func Diff(x, y string) string {
	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
	defer cancel()
	return Format(Tokenize(ctx, x, y), ThemeDefault)
}

// Tokenize x and y into diff tokens with diff words and narrow chunks.
func Tokenize(ctx context.Context, x, y string) []*Token {
	ts := TokenizeText(ctx, x, y)
	lines := ParseTokenLines(ts)
	lines = Narrow(1, lines)
	Words(ctx, lines)
	return SpreadTokenLines(lines)
}

// Format tokens into a human readable string
func Format(ts []*Token, theme Theme) string {
	out := ""

	for _, t := range ts {
		s := t.Literal
		out += gop.Stylize(s, theme(t.Type))
	}

	return out
}

// Narrow the context around each diff section to n lines.
func Narrow(n int, lines []*TokenLine) []*TokenLine {
	if n < 0 {
		n = 0
	}

	keep := map[int]bool{}
	for i, l := range lines {
		switch l.Type {
		case AddSymbol, DelSymbol:
			for j := max(i-n, 0); j <= i+n && j < len(lines); j++ {
				keep[j] = true
			}
		}
	}

	out := []*TokenLine{}
	for i, l := range lines {
		if !keep[i] {
			continue
		}

		if _, has := keep[i-1]; !has {
			ts := []*Token{{ChunkStart, "@@ diff chunk @@"}, {Newline, "\n"}}
			out = append(out, &TokenLine{ChunkStart, ts})
		}

		out = append(out, l)

		if _, has := keep[i+1]; !has {
			ts := []*Token{{ChunkEnd, ""}, {Newline, "\n"}}
			out = append(out, &TokenLine{ChunkEnd, ts})
		}
	}

	return out
}

// Words diff
func Words(ctx context.Context, lines []*TokenLine) {
	delLines := []*TokenLine{}
	addLines := []*TokenLine{}

	df := func() {
		if len(delLines) == 0 || len(delLines) != len(addLines) {
			return
		}

		for i := 0; i < len(delLines); i++ {
			d := delLines[i]
			a := addLines[i]

			dts, ats := TokenizeLine(ctx, d.Tokens[2].Literal, a.Tokens[2].Literal)
			d.Tokens = append(d.Tokens[0:2], append(dts, d.Tokens[3:]...)...)
			a.Tokens = append(a.Tokens[0:2], append(ats, a.Tokens[3:]...)...)
		}

		delLines = []*TokenLine{}
		addLines = []*TokenLine{}
	}

	for _, l := range lines {
		switch l.Type {
		case DelSymbol:
			delLines = append(delLines, l)
		case AddSymbol:
			addLines = append(addLines, l)
		default:
			df()
		}
	}

	df()
}

func max(x, y int) int {
	if x < y {
		return y
	}
	return x
}