STRLCPY/termdash

Implements a buffer limit for the Text widget. (#301)

See issue #293 where memory and performance can degrade with a high number of lines written to the Text widget. 

This is a very simplistic implementation to limit the possible length the text buffer can grow to with the `maxContent` option. 

Default value of -1 means there's no limit and therefore behaviour should remain standard.

It has been working in our test app and allows the use of the Text widget to monitor logs (ie tail) and therefore doesn't bloat over time, but happy to adjust as required.

Jakub Sobon committed with GitHub 3 years ago

4238ac6f

1 parent 3cfeb8ad

Total 6 files Show one by one

■ ■ ■ ■ ■ ■

CHANGELOG.md

		skipped 6 lines
7	7
8	8		## [Unreleased]
9	9
	10	+	### Added
	11	+
	12	+	- The `Text` widget has a new option `MaxTextCells` which can be used to limit
	13	+	the maximum number of cells the widget keeps in memory.
	14	+
10	15		### Changed
11	16
12	17		- Bump github.com/mattn/go-runewidth from 0.0.10 to 0.0.12.
		skipped 469 lines

■ ■ ■ ■ ■ ■ ■

private/runewidth/runewidth.go

		skipped 17 lines
18	18
19	19		import runewidth "github.com/mattn/go-runewidth"
20	20
	21	+	// Option is used to provide options.
	22	+	type Option interface {
	23	+	// set sets the provided option.
	24	+	set(*options)
	25	+	}
	26	+
	27	+	// options stores the provided options.
	28	+	type options struct {
	29	+	runeWidths map[rune]int
	30	+	}
	31	+
	32	+	// newOptions create a new instance of options.
	33	+	func newOptions() *options {
	34	+	return &options{
	35	+	runeWidths: map[rune]int{},
	36	+	}
	37	+	}
	38	+
	39	+	// option implements Option.
	40	+	type option func(*options)
	41	+
	42	+	// set implements Option.set.
	43	+	func (o option) set(opts *options) {
	44	+	o(opts)
	45	+	}
	46	+
	47	+	// CountAsWidth overrides the default behavior, counting the specified runes as
	48	+	// the specified width. Can be provided multiple times to specify an override
	49	+	// for multiple runes.
	50	+	func CountAsWidth(r rune, width int) Option {
	51	+	return option(func(opts *options) {
	52	+	opts.runeWidths[r] = width
	53	+	})
	54	+	}
	55	+
21	56		// RuneWidth returns the number of cells needed to draw r.
22	57		// Background in http://www.unicode.org/reports/tr11/.
23	58		//
		skipped 5 lines
29	64		// This should be safe, since even in locales where these runes have ambiguous
30	65		// width, we still place all the character content around them so they should
31	66		// have be half-width.
32		-	func RuneWidth(r rune) int {
	67	+	func RuneWidth(r rune, opts ...Option) int {
	68	+	o := newOptions()
	69	+	for _, opt := range opts {
	70	+	opt.set(o)
	71	+	}
	72	+
	73	+	if w, ok := o.runeWidths[r]; ok {
	74	+	return w
	75	+	}
	76	+
33	77		if inTable(r, exceptions) {
34	78		return 1
35	79		}
		skipped 2 lines
38	82
39	83		// StringWidth is like RuneWidth, but returns the number of cells occupied by
40	84		// all the runes in the string.
41		-	func StringWidth(s string) int {
	85	+	func StringWidth(s string, opts ...Option) int {
42	86		var width int
43	87		for _, r := range []rune(s) {
44		-	width += RuneWidth(r)
	88	+	width += RuneWidth(r, opts...)
45	89		}
46	90		return width
47	91		}
		skipped 52 lines

■ ■ ■ ■ ■ ■

private/runewidth/runewidth_test.go

		skipped 23 lines
24	24		tests := []struct {
25	25		desc string
26	26		runes []rune
	27	+	opts []Option
27	28		eastAsian bool
28	29		want int
29	30		}{
		skipped 4 lines
34	35		},
35	36		{
36	37		desc: "non-printable characters from mattn/runewidth/runewidth_test",
37		-	runes: []rune{'\x00', '\x01', '\u0300', '\u2028', '\u2029'},
	38	+	runes: []rune{'\x00', '\x01', '\u0300', '\u2028', '\u2029', '\n'},
38	39		want: 0,
	40	+	},
	41	+	{
	42	+	desc: "override rune width with an option",
	43	+	runes: []rune{'\n'},
	44	+	opts: []Option{
	45	+	CountAsWidth('\n', 3),
	46	+	},
	47	+	want: 3,
39	48		},
40	49		{
41	50		desc: "half-width runes from mattn/runewidth/runewidth_test",
		skipped 65 lines
107	116		}()
108	117
109	118		for _, r := range tc.runes {
110		-	if got := RuneWidth(r); got != tc.want {
	119	+	if got := RuneWidth(r, tc.opts...); got != tc.want {
111	120		t.Errorf("RuneWidth(%c, %#x) => %v, want %v", r, r, got, tc.want)
112	121		}
113	122		}
		skipped 5 lines
119	128		tests := []struct {
120	129		desc string
121	130		str string
	131	+	opts []Option
122	132		eastAsian bool
123	133		want int
124	134		}{
		skipped 1 lines
126	136		desc: "ascii characters",
127	137		str: "hello",
128	138		want: 5,
	139	+	},
	140	+	{
	141	+	desc: "override rune widths with an option",
	142	+	str: "hello",
	143	+	opts: []Option{
	144	+	CountAsWidth('h', 5),
	145	+	CountAsWidth('e', 5),
	146	+	},
	147	+	want: 13,
129	148		},
130	149		{
131	150		desc: "string from mattn/runewidth/runewidth_test",
		skipped 26 lines
158	177		runewidth.DefaultCondition.EastAsianWidth = false
159	178		}()
160	179
161		-	if got := StringWidth(tc.str); got != tc.want {
	180	+	if got := StringWidth(tc.str, tc.opts...); got != tc.want {
162	181		t.Errorf("StringWidth(%q) => %v, want %v", tc.str, got, tc.want)
163	182		}
164	183		})
		skipped 3 lines

■ ■ ■ ■ ■ ■

widgets/text/options.go

		skipped 35 lines
36	36		scrollDown rune
37	37		wrapMode wrap.Mode
38	38		rollContent bool
	39	+	maxTextCells int
39	40		disableScrolling bool
40	41		mouseUpButton mouse.Button
41	42		mouseDownButton mouse.Button
		skipped 14 lines
56	57		keyDown: DefaultScrollKeyDown,
57	58		keyPgUp: DefaultScrollKeyPageUp,
58	59		keyPgDown: DefaultScrollKeyPageDown,
	60	+	maxTextCells: DefaultMaxTextCells,
59	61		}
60	62		for _, o := range opts {
61	63		o.set(opt)
		skipped 14 lines
76	78		}
77	79		if o.mouseUpButton == o.mouseDownButton {
78	80		return fmt.Errorf("invalid ScrollMouseButtons(up:%v, down:%v), the buttons must be unique", o.mouseUpButton, o.mouseDownButton)
	81	+	}
	82	+	if o.maxTextCells < 0 {
	83	+	return fmt.Errorf("invalid MaxTextCells(%d), must be zero or a positive integer", o.maxTextCells)
79	84		}
80	85		return nil
81	86		}
		skipped 93 lines
175	180		})
176	181		}
177	182
	183	+	// The default value for the MaxTextCells option.
	184	+	// Use zero as no limit, for logs you may wish to try 10,000 or higher.
	185	+	const (
	186	+	DefaultMaxTextCells = 0
	187	+	)
	188	+
	189	+	// MaxTextCells limits the text content to this number of terminal cells.
	190	+	// This is useful when sending large amounts of text to the Text widget, e.g.
	191	+	// when tailing logs as it will limit the memory usage.
	192	+	// When the newly added content goes over this number of cells, the Text widget
	193	+	// behaves as a circular buffer and drops earlier content to accommodate the
	194	+	// new one.
	195	+	// Note the count is in cells, not runes, some wide runes can take multiple
	196	+	// terminal cells.
	197	+	func MaxTextCells(max int) Option {
	198	+	return option(func(opts *options) {
	199	+	opts.maxTextCells = max
	200	+	})
	201	+	}
	202	+

■ ■ ■ ■ ■ ■ ■

widgets/text/text.go

		skipped 17 lines
18	18		import (
19	19		"fmt"
20	20		"image"
	21	+	"strings"
21	22		"sync"
22	23
23	24		"github.com/mum4k/termdash/private/canvas"
24	25		"github.com/mum4k/termdash/private/canvas/buffer"
	26	+	"github.com/mum4k/termdash/private/runewidth"
25	27		"github.com/mum4k/termdash/private/wrap"
26	28		"github.com/mum4k/termdash/terminal/terminalapi"
27	29		"github.com/mum4k/termdash/widgetapi"
		skipped 62 lines
90	92		t.contentChanged = true
91	93		}
92	94
	95	+	// contentCells calculates the number of cells the content takes to display on
	96	+	// terminal.
	97	+	func (t *Text) contentCells() int {
	98	+	cells := 0
	99	+	for _, c := range t.content {
	100	+	cells += runewidth.RuneWidth(c.Rune, runewidth.CountAsWidth('\n', 1))
	101	+	}
	102	+	return cells
	103	+	}
	104	+
93	105		// Write writes text for the widget to display. Multiple calls append
94	106		// additional text. The text contain cannot control characters
95	107		// (unicode.IsControl) or space character (unicode.IsSpace) other than:
		skipped 12 lines
108	120		if opts.replace {
109	121		t.reset()
110	122		}
111		-	for _, r := range text {
	123	+
	124	+	truncated := truncateToCells(text, t.opts.maxTextCells)
	125	+	textCells := runewidth.StringWidth(truncated, runewidth.CountAsWidth('\n', 1))
	126	+	contentCells := t.contentCells()
	127	+	// If MaxTextCells has been set, limit the content if needed.
	128	+	if t.opts.maxTextCells > 0 && contentCells+textCells > t.opts.maxTextCells {
	129	+	diff := contentCells + textCells - t.opts.maxTextCells
	130	+	t.content = t.content[diff:]
	131	+	}
	132	+
	133	+	for _, r := range truncated {
112	134		t.content = append(t.content, buffer.NewCell(r, opts.cellOpts))
113	135		}
114	136		t.contentChanged = true
		skipped 170 lines
285	307		}
286	308		}
287	309
	310	+	// truncateToCells truncates the beginning of text, so that it can be displayed
	311	+	// in at most maxCells. Setting maxCells to zero disables truncating.
	312	+	func truncateToCells(text string, maxCells int) string {
	313	+	textCells := runewidth.StringWidth(text, runewidth.CountAsWidth('\n', 1))
	314	+	if maxCells == 0 \|\| textCells <= maxCells {
	315	+	return text
	316	+	}
	317	+
	318	+	haveCells := 0
	319	+	textRunes := []rune(text)
	320	+	i := len(textRunes) - 1
	321	+	for ; i >= 0; i-- {
	322	+	haveCells += runewidth.RuneWidth(textRunes[i], runewidth.CountAsWidth('\n', 1))
	323	+	if haveCells > maxCells {
	324	+	break
	325	+	}
	326	+	}
	327	+
	328	+	var b strings.Builder
	329	+	for j := i + 1; j < len(textRunes); j++ {
	330	+	b.WriteRune(textRunes[j])
	331	+	}
	332	+	return b.String()
	333	+	}
	334	+

■ ■ ■ ■ ■ ■

widgets/text/text_test.go

		skipped 808 lines
809	809		return ft
810	810		},
811	811		},
	812	+	{
	813	+	desc: "tests maxTextCells length being applied - multiline",
	814	+	canvas: image.Rect(0, 0, 10, 3),
	815	+	opts: []Option{
	816	+	MaxTextCells(10),
	817	+	RollContent(),
	818	+	},
	819	+	writes: func(widget *Text) error {
	820	+	return widget.Write("line0\nline1\nline2\nline3\nline4")
	821	+	},
	822	+	want: func(size image.Point) *faketerm.Terminal {
	823	+	ft := faketerm.MustNew(size)
	824	+	c := testcanvas.MustNew(ft.Area())
	825	+	// \n still counts as a chacter in the string length
	826	+	testdraw.MustText(c, "ine3", image.Point{0, 0})
	827	+	testdraw.MustText(c, "line4", image.Point{0, 1})
	828	+	testcanvas.MustApply(c, ft)
	829	+	return ft
	830	+	},
	831	+	},
	832	+	{
	833	+	desc: "tests maxTextCells - multiple writes - first one fits",
	834	+	canvas: image.Rect(0, 0, 10, 3),
	835	+	opts: []Option{
	836	+	MaxTextCells(10),
	837	+	RollContent(),
	838	+	},
	839	+	writes: func(widget *Text) error {
	840	+	if err := widget.Write("line0\nline"); err != nil {
	841	+	return err
	842	+	}
	843	+	return widget.Write("1\nline2\nline3\nline4")
	844	+	},
	845	+	want: func(size image.Point) *faketerm.Terminal {
	846	+	ft := faketerm.MustNew(size)
	847	+	c := testcanvas.MustNew(ft.Area())
	848	+	// \n still counts as a chacter in the string length
	849	+	testdraw.MustText(c, "ine3", image.Point{0, 0})
	850	+	testdraw.MustText(c, "line4", image.Point{0, 1})
	851	+	testcanvas.MustApply(c, ft)
	852	+	return ft
	853	+	},
	854	+	},
	855	+	{
	856	+	desc: "tests maxTextCells - multiple writes - first one does not fit",
	857	+	canvas: image.Rect(0, 0, 10, 3),
	858	+	opts: []Option{
	859	+	MaxTextCells(10),
	860	+	RollContent(),
	861	+	},
	862	+	writes: func(widget *Text) error {
	863	+	if err := widget.Write("line0\nline123"); err != nil {
	864	+	return err
	865	+	}
	866	+	return widget.Write("1\nline2\nline3\nline4")
	867	+	},
	868	+	want: func(size image.Point) *faketerm.Terminal {
	869	+	ft := faketerm.MustNew(size)
	870	+	c := testcanvas.MustNew(ft.Area())
	871	+	testdraw.MustText(c, "ine3", image.Point{0, 0})
	872	+	testdraw.MustText(c, "line4", image.Point{0, 1})
	873	+	testcanvas.MustApply(c, ft)
	874	+	return ft
	875	+	},
	876	+	},
	877	+	{
	878	+	desc: "tests maxTextCells - accounts for pre-existing full-width runes on the content",
	879	+	canvas: image.Rect(0, 0, 10, 3),
	880	+	opts: []Option{
	881	+	MaxTextCells(3),
	882	+	RollContent(),
	883	+	},
	884	+	writes: func(widget *Text) error {
	885	+	if err := widget.Write("界"); err != nil {
	886	+	return err
	887	+	}
	888	+	return widget.Write("ab")
	889	+	},
	890	+	want: func(size image.Point) *faketerm.Terminal {
	891	+	ft := faketerm.MustNew(size)
	892	+	c := testcanvas.MustNew(ft.Area())
	893	+	testdraw.MustText(c, "ab", image.Point{0, 0})
	894	+	testcanvas.MustApply(c, ft)
	895	+	return ft
	896	+	},
	897	+	},
	898	+	{
	899	+	desc: "tests maxTextCells exact length of 5",
	900	+	canvas: image.Rect(0, 0, 10, 1),
	901	+	opts: []Option{
	902	+	RollContent(),
	903	+	MaxTextCells(5),
	904	+	},
	905	+	writes: func(widget *Text) error {
	906	+	return widget.Write("12345")
	907	+	},
	908	+	want: func(size image.Point) *faketerm.Terminal {
	909	+	ft := faketerm.MustNew(size)
	910	+	c := testcanvas.MustNew(ft.Area())
	911	+	// Line return (\n) counts as one character
	912	+	testdraw.MustText(
	913	+	c,
	914	+	"12345",
	915	+	image.Point{0, 0},
	916	+	)
	917	+	testcanvas.MustApply(c, ft)
	918	+	return ft
	919	+	},
	920	+	},
	921	+	{
	922	+	desc: "tests maxTextCells partial bufffer replacement",
	923	+	canvas: image.Rect(0, 0, 10, 1),
	924	+	opts: []Option{
	925	+	RollContent(),
	926	+	MaxTextCells(10),
	927	+	},
	928	+	writes: func(widget *Text) error {
	929	+	return widget.Write("hello wor你12345678")
	930	+	},
	931	+	want: func(size image.Point) *faketerm.Terminal {
	932	+	ft := faketerm.MustNew(size)
	933	+	c := testcanvas.MustNew(ft.Area())
	934	+	testdraw.MustText(
	935	+	c,
	936	+	"你12345678",
	937	+	image.Point{0, 0},
	938	+	)
	939	+	testcanvas.MustApply(c, ft)
	940	+	return ft
	941	+	},
	942	+	},
	943	+	{
	944	+	desc: "tests maxTextCells length not being limited",
	945	+	canvas: image.Rect(0, 0, 72, 1),
	946	+	opts: []Option{
	947	+	RollContent(),
	948	+	},
	949	+	writes: func(widget *Text) error {
	950	+	return widget.Write("1234567890abcdefghijklmnopqrstuvwxyz")
	951	+	},
	952	+	want: func(size image.Point) *faketerm.Terminal {
	953	+	ft := faketerm.MustNew(size)
	954	+	c := testcanvas.MustNew(ft.Area())
	955	+	testdraw.MustText(
	956	+	c,
	957	+	"1234567890abcdefghijklmnopqrstuvwxyz",
	958	+	image.Point{0, 0},
	959	+	)
	960	+	testcanvas.MustApply(c, ft)
	961	+	return ft
	962	+	},
	963	+	},
	964	+	{
	965	+	desc: "tests maxTextCells length being applied - single line",
	966	+	canvas: image.Rect(0, 0, 10, 3),
	967	+	opts: []Option{
	968	+	MaxTextCells(5),
	969	+	RollContent(),
	970	+	},
	971	+	writes: func(widget *Text) error {
	972	+	return widget.Write("1234567890abcdefghijklmnopqrstuvwxyz")
	973	+	},
	974	+	want: func(size image.Point) *faketerm.Terminal {
	975	+	ft := faketerm.MustNew(size)
	976	+	c := testcanvas.MustNew(ft.Area())
	977	+	testdraw.MustText(c, "vwxyz", image.Point{0, 0})
	978	+	testcanvas.MustApply(c, ft)
	979	+	return ft
	980	+	},
	981	+	},
812	982		}
813	983
814	984		for _, tc := range tests {
		skipped 87 lines
902	1072		}
903	1073		}
904	1074
	1075	+	func TestTruncateToCells(t *testing.T) {
	1076	+	tests := []struct {
	1077	+	desc string
	1078	+	text string
	1079	+	maxCells int
	1080	+	want string
	1081	+	}{
	1082	+	{
	1083	+	desc: "returns empty on empty text",
	1084	+	text: "",
	1085	+	maxCells: 0,
	1086	+	want: "",
	1087	+	},
	1088	+	{
	1089	+	desc: "no need to truncate, length matches max",
	1090	+	text: "a",
	1091	+	maxCells: 1,
	1092	+	want: "a",
	1093	+	},
	1094	+	{
	1095	+	desc: "no need to truncate, shorter than max",
	1096	+	text: "a",
	1097	+	maxCells: 2,
	1098	+	want: "a",
	1099	+	},
	1100	+	{
	1101	+	desc: "no need to truncate, maxCells set to zero",
	1102	+	text: "a",
	1103	+	maxCells: 0,
	1104	+	want: "a",
	1105	+	},
	1106	+	{
	1107	+	desc: "truncates single rune to enforce max cells",
	1108	+	text: "abc",
	1109	+	maxCells: 2,
	1110	+	want: "bc",
	1111	+	},
	1112	+	{
	1113	+	desc: "truncates multiple runes to enforce max cells",
	1114	+	text: "abcde",
	1115	+	maxCells: 3,
	1116	+	want: "cde",
	1117	+	},
	1118	+	{
	1119	+	desc: "accounts for cells taken by newline characters",
	1120	+	text: "a\ncde",
	1121	+	maxCells: 3,
	1122	+	want: "cde",
	1123	+	},
	1124	+	{
	1125	+	desc: "truncates full-width rune on its edge",
	1126	+	text: "世界",
	1127	+	maxCells: 2,
	1128	+	want: "界",
	1129	+	},
	1130	+	{
	1131	+	desc: "truncates full-width rune because only half of it fits",
	1132	+	text: "世界",
	1133	+	maxCells: 3,
	1134	+	want: "界",
	1135	+	},
	1136	+	{
	1137	+	desc: "full-width runes - truncating not needed",
	1138	+	text: "世界",
	1139	+	maxCells: 4,
	1140	+	want: "世界",
	1141	+	},
	1142	+	}
	1143	+
	1144	+	for _, tc := range tests {
	1145	+	t.Run(tc.desc, func(t *testing.T) {
	1146	+	got := truncateToCells(tc.text, tc.maxCells)
	1147	+	if diff := pretty.Compare(tc.want, got); diff != "" {
	1148	+	t.Errorf("truncateToCells => unexpected diff (-want, +got):\n%s", diff)
	1149	+	}
	1150	+	})
	1151	+	}
	1152	+	}
	1153	+