STRLCPY/dolt

■ ■ ■ ■ ■ ■

go/libraries/doltcore/doltdb/doltdb.go

		skipped 1148 lines
1149	1149		return datas.ChunkStoreFromDatabase(ddb.db).Rebase(ctx)
1150	1150		}
1151	1151
1152		-	// GC performs garbage collection on this ddb. Values passed in \|uncommitedVals\| will be temporarily saved during gc.
1153		-	func (ddb *DoltDB) GC(ctx context.Context, uncommitedVals ...hash.Hash) error {
	1152	+	// GC performs garbage collection on this ddb. Values passed in \|uncommittedVals\| will be temporarily saved during gc.
	1153	+	func (ddb *DoltDB) GC(ctx context.Context, uncommittedVals ...hash.Hash) error {
1154	1154		collector, ok := ddb.db.Database.(datas.GarbageCollector)
1155	1155		if !ok {
1156	1156		return fmt.Errorf("this database does not support garbage collection")
		skipped 8 lines
1165	1165		if err != nil {
1166	1166		return err
1167	1167		}
1168		-	newGen := hash.NewHashSet(uncommitedVals...)
	1168	+	newGen := hash.NewHashSet(uncommittedVals...)
1169	1169		oldGen := make(hash.HashSet)
1170	1170		err = datasets.IterAll(ctx, func(keyStr string, h hash.Hash) error {
1171	1171		var isOldGen bool
		skipped 153 lines

■ ■ ■ ■ ■ ■

go/libraries/doltcore/env/environment.go

		skipped 1128 lines
1129	1129		return absPath, nil
1130	1130		}
1131	1131
1132		-	// GetGCKeepers returns the hashes of all the objects in the environment provided that should be perserved during GC.
	1132	+	// GetGCKeepers returns the hashes of all the objects in the environment provided that should be preserved during GC.
1133	1133		// TODO: this should be unnecessary since we now store the working set in a noms dataset, remove it
1134	1134		func GetGCKeepers(ctx context.Context, env *DoltEnv) ([]hash.Hash, error) {
1135	1135		workingRoot, err := env.WorkingRoot(ctx)
		skipped 158 lines

■ ■ ■ ■ ■ ■

go/libraries/doltcore/sqle/dprocedures/dolt_gc.go

1	+	// Copyright 2022 Dolthub, Inc.
2	+	//
3	+	// Licensed under the Apache License, Version 2.0 (the "License");
4	+	// you may not use this file except in compliance with the License.
5	+	// You may obtain a copy of the License at
6	+	//
7	+	// http://www.apache.org/licenses/LICENSE-2.0
8	+	//
9	+	// Unless required by applicable law or agreed to in writing, software
10	+	// distributed under the License is distributed on an "AS IS" BASIS,
11	+	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12	+	// See the License for the specific language governing permissions and
13	+	// limitations under the License.
14	+
15	+	package dprocedures
16	+
17	+	import (
18	+	"fmt"
19	+
20	+	"github.com/dolthub/dolt/go/libraries/doltcore/branch_control"
21	+	"github.com/dolthub/dolt/go/libraries/doltcore/sqle/dsess"
22	+	"github.com/dolthub/go-mysql-server/sql"
23	+	)
24	+
25	+	const (
26	+	cmdFailure = 0
27	+	cmdSuccess = 1
28	+	)
29	+
30	+	// doltGC is the stored procedure version of the functions `gc` and `dolt_gc`.
31	+	func doltGC(ctx *sql.Context, args ...string) (sql.RowIter, error) {
32	+	res, err := doDoltGC(ctx, args)
33	+	if err != nil {
34	+	return nil, err
35	+	}
36	+	return rowToIter(int64(res)), nil
37	+	}
38	+
39	+	func doDoltGC(ctx *sql.Context, args []string) (int, error) {
40	+	dbName := ctx.GetCurrentDatabase()
41	+
42	+	if len(dbName) == 0 {
43	+	return cmdFailure, fmt.Errorf("Empty database name.")
44	+	}
45	+	if err := branch_control.CheckAccess(ctx, branch_control.Permissions_Write); err != nil {
46	+	return cmdFailure, err
47	+	}
48	+
49	+	dSess := dsess.DSessFromSess(ctx.Session)
50	+	ddb, ok := dSess.GetDoltDB(ctx, dbName)
51	+	if !ok {
52	+	return cmdFailure, fmt.Errorf("Could not load database %s", dbName)
53	+	}
54	+
55	+	err := ddb.ShallowGC(ctx)
56	+	if err != nil {
57	+	return cmdFailure, err
58	+	}
59	+
60	+	return cmdSuccess, nil
61	+	}
62	+

■ ■ ■ ■ ■ ■

go/libraries/doltcore/sqle/dprocedures/init.go

		skipped 25 lines
26	26		{Name: "dolt_commit", Schema: stringSchema("hash"), Function: doltCommit},
27	27		{Name: "dolt_conflicts_resolve", Schema: int64Schema("status"), Function: doltConflictsResolve},
28	28		{Name: "dolt_fetch", Schema: int64Schema("success"), Function: doltFetch},
	29	+	{Name: "dolt_gc", Schema: int64Schema("success"), Function: doltGC},
29	30		{Name: "dolt_merge", Schema: int64Schema("fast_forward", "conflicts"), Function: doltMerge},
30	31		{Name: "dolt_pull", Schema: int64Schema("fast_forward", "conflicts"), Function: doltPull},
31	32		{Name: "dolt_push", Schema: int64Schema("success"), Function: doltPush},
		skipped 11 lines
43	44		{Name: "dclone", Schema: int64Schema("status"), Function: doltClone},
44	45		{Name: "dcommit", Schema: stringSchema("hash"), Function: doltCommit},
45	46		{Name: "dfetch", Schema: int64Schema("success"), Function: doltFetch},
	47	+	{Name: "dgc", Schema: int64Schema("status"), Function: doltGC},
46	48		{Name: "dmerge", Schema: int64Schema("fast_forward", "conflicts"), Function: doltMerge},
47	49		{Name: "dpull", Schema: int64Schema("fast_forward", "conflicts"), Function: doltPull},
48	50		{Name: "dpush", Schema: int64Schema("success"), Function: doltPush},
		skipped 42 lines

■ ■ ■ ■ ■ ■

go/libraries/doltcore/sqle/enginetest/dolt_queries.go

		skipped 3891 lines
3892	3892		},
3893	3893		}
3894	3894
	3895	+	func gcSetup() []string {
	3896	+	queries := []string{
	3897	+	"create table t (pk int primary key);",
	3898	+	"call dolt_commit('-Am', 'create table');",
	3899	+	}
	3900	+	for i := 0; i < 250; i++ {
	3901	+	queries = append(
	3902	+	queries,
	3903	+	fmt.Sprintf("INSERT INTO t VALUES (%d);", i),
	3904	+	fmt.Sprintf("CALL DOLT_COMMIT('-am', 'added pk %d')", i),
	3905	+	)
	3906	+	}
	3907	+	return queries
	3908	+	}
	3909	+
	3910	+	var DoltGC = []queries.ScriptTest{
	3911	+	{
	3912	+	Name: "base case: shallow gc",
	3913	+	SetUpScript: gcSetup(),
	3914	+	Assertions: []queries.ScriptTestAssertion{
	3915	+	{
	3916	+	Query: "CALL DOLT_GC();",
	3917	+	Expected: []sql.Row{{0}},
	3918	+	},
	3919	+	},
	3920	+	},
	3921	+	}
	3922	+
3895	3923		var DiffSystemTableScriptTests = []queries.ScriptTest{
3896	3924		{
3897	3925		Name: "base case: added rows",
		skipped 4451 lines

■ ■ ■ ■ ■ ■

integration-tests/bats/garbage_collection.bats

		skipped 268 lines
269	269		[[ "${lines[1]}" =~ "9,99" ]] \|\| false
270	270		}
271	271
	272	+	create_many_commits() {
	273	+	dolt sql <<SQL
	274	+	CREATE TABLE test (pk int PRIMARY KEY);
	275	+	CALL DOLT_COMMIT('-Am', 'Create test table');
	276	+	SQL
	277	+
	278	+	# Create a lot of commits to create some conjoin garbage
	279	+	NUM_COMMITS=250
	280	+
	281	+	for i in $(eval echo "{1..$NUM_COMMITS}")
	282	+	do
	283	+	dolt sql <<SQL
	284	+	INSERT INTO test VALUES ($i);
	285	+	CALL DOLT_COMMIT('-am', 'Add new val $i');
	286	+	SQL
	287	+	done
	288	+
	289	+	run dolt sql -q "select count(*) from test"
	290	+	[ "$status" -eq 0 ]
	291	+	[[ "$output" =~ "$NUM_COMMITS" ]] \|\| false
	292	+	}
	293	+
	294	+	@test "garbage_collection: shallow gc" {
	295	+	create_many_commits
	296	+
	297	+	# leave data in the working set
	298	+	dolt sql -q "INSERT INTO test VALUES ($(($NUM_COMMITS+1))),($(($NUM_COMMITS+2))),($(($NUM_COMMITS+3)));"
	299	+
	300	+	BEFORE=$(du -c .dolt/noms/ \| grep total \| sed 's/[^0-9]*//g')
	301	+	run dolt gc --shallow
	302	+	[ "$status" -eq 0 ]
	303	+
	304	+	run dolt sql -q "select count(*) from test"
	305	+	[ "$status" -eq 0 ]
	306	+	[[ "$output" =~ "$(($NUM_COMMITS+3))" ]] \|\| false
	307	+
	308	+	AFTER=$(du -c .dolt/noms/ \| grep total \| sed 's/[^0-9]*//g')
	309	+
	310	+	# assert space was reclaimed
	311	+	echo "$BEFORE"
	312	+	echo "$AFTER"
	313	+	[ "$BEFORE" -gt "$AFTER" ]
	314	+	}
	315	+
	316	+	@test "garbage_collection: online shallow gc" {
	317	+	create_many_commits
	318	+
	319	+	# leave data in the working set
	320	+	dolt sql -q "INSERT INTO test VALUES ($(($NUM_COMMITS+1))),($(($NUM_COMMITS+2))),($(($NUM_COMMITS+3)));"
	321	+
	322	+	BEFORE=$(du -c .dolt/noms/ \| grep total \| sed 's/[^0-9]*//g')
	323	+	run dolt sql -q "call dolt_gc();"
	324	+	[ "$status" -eq 0 ]
	325	+
	326	+	run dolt sql -q "select count(*) from test"
	327	+	[ "$status" -eq 0 ]
	328	+	[[ "$output" =~ "$(($NUM_COMMITS+3))" ]] \|\| false
	329	+
	330	+	AFTER=$(du -c .dolt/noms/ \| grep total \| sed 's/[^0-9]*//g')
	331	+
	332	+	# assert space was reclaimed
	333	+	echo "$BEFORE"
	334	+	echo "$AFTER"
	335	+	[ "$BEFORE" -gt "$AFTER" ]
	336	+	}

Add some shallow gc tests