STRLCPY/dolt

■ ■ ■ ■ ■ ■

docker/docker-entrypoint.sh

		skipped 18 lines
19	19		mysql_log ERROR "$@" >&2
20	20		exit 1
21	21		}
	22	+	docker_process_sql() {
	23	+	dolt sql
	24	+	}
22	25
23	26		CONTAINER_DATA_DIR="/var/lib/dolt"
	27	+	INIT_COMPLETED="$CONTAINER_DATA_DIR/.init_completed"
24	28		DOLT_CONFIG_DIR="/etc/dolt/doltcfg.d"
25	29		SERVER_CONFIG_DIR="/etc/dolt/servercfg.d"
26	30		DOLT_ROOT_PATH="/.dolt"
27		-
28		-	# create all dirs in path
29		-	_create_dir() {
30		-	local path="$1"
31		-	mkdir -p "$path"
32		-	}
33	31
34	32		check_for_dolt() {
35	33		local dolt_bin=$(which dolt)
		skipped 44 lines
80	78		# ie: docker_process_init_files /always-initdb.d/*
81	79		# process initializer files, based on file extensions
82	80		docker_process_init_files() {
83		-	echo
	81	+	mysql_note "Running init scripts"
84	82		local f
85	83		for f; do
86	84		case "$f" in
		skipped 19 lines
106	104		done
107	105		}
108	106
109		-	start_server() {
110		-	# start the server in fixed data directory at /var/lib/dolt
111		-	cd $CONTAINER_DATA_DIR
112		-	"$@"
113		-	}
114		-
115	107		# if there is config file provided through /etc/dolt/doltcfg.d,
116	108		# we overwrite $HOME/.dolt/config_global.json file with this file.
117	109		set_dolt_config_if_defined() {
		skipped 30 lines
148	140		if [ ! -z $CONFIG_PROVIDED ]; then
149	141		set -- "$@" --config=$CONFIG_PROVIDED
150	142		fi
151		-	start_server
152	143
153		-	# run any file provided in /docker-entrypoint-initdb.d directory after the server starts
154		-	docker_process_init_files /docker-entrypoint-initdb.d/*
	144	+	if [[ ! -f $INIT_COMPLETED ]]; then
	145	+	# run any file provided in /docker-entrypoint-initdb.d directory after the server starts
	146	+	docker_process_init_files /docker-entrypoint-initdb.d/*
	147	+	touch $INIT_COMPLETED
	148	+	fi
	149	+	fi
155	150
156		-	mysql_note "Dolt Server $dolt_version is started."
157		-	fi
158	151		exec "$@"
159	152		}
160	153
		skipped 2 lines

■ ■ ■ ■ ■ ■

docker/serverDockerfile

		skipped 19 lines
20	20		ENTRYPOINT ["docker-entrypoint.sh"]
21	21
22	22		EXPOSE 3306 33060
	23	+	WORKDIR /var/lib/dolt
23	24		CMD [ "dolt", "sql-server", "--host=0.0.0.0" , "--port=3306" ]
24	25

■ ■ ■ ■ ■ ■

go/go.mod

		skipped 14 lines
15	15		github.com/dolthub/fslock v0.0.3
16	16		github.com/dolthub/ishell v0.0.0-20220112232610-14e753f0f371
17	17		github.com/dolthub/sqllogictest/go v0.0.0-20201107003712-816f3ae12d81
18		-	github.com/dolthub/vitess v0.0.0-20221116234926-60a2fd96afae
	18	+	github.com/dolthub/vitess v0.0.0-20221121184553-8d519d0bbb91
19	19		github.com/dustin/go-humanize v1.0.0
20	20		github.com/fatih/color v1.13.0
21	21		github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568
		skipped 36 lines
58	58		github.com/cenkalti/backoff/v4 v4.1.3
59	59		github.com/cespare/xxhash v1.1.0
60	60		github.com/creasty/defaults v1.6.0
61		-	github.com/dolthub/go-mysql-server v0.14.1-0.20221118195531-883fae8ef2a6
	61	+	github.com/dolthub/go-mysql-server v0.14.1-0.20221122003003-d6e384c64854
62	62		github.com/google/flatbuffers v2.0.6+incompatible
63	63		github.com/kch42/buzhash v0.0.0-20160816060738-9bdec3dec7c6
64	64		github.com/mitchellh/go-ps v1.0.0
		skipped 80 lines

■ ■ ■ ■ ■ ■

go/go.sum

		skipped 179 lines
180	180		github.com/dolthub/flatbuffers v1.13.0-dh.1/go.mod h1:CorYGaDmXjHz1Z7i50PYXG1Ricn31GcA2wNOTFIQAKE=
181	181		github.com/dolthub/fslock v0.0.3 h1:iLMpUIvJKMKm92+N1fmHVdxJP5NdyDK5bK7z7Ba2s2U=
182	182		github.com/dolthub/fslock v0.0.3/go.mod h1:QWql+P17oAAMLnL4HGB5tiovtDuAjdDTPbuqx7bYfa0=
183		-	github.com/dolthub/go-mysql-server v0.14.1-0.20221118195531-883fae8ef2a6 h1:9jIfVSFh7LINxyFgdhGL1/NVfNZK8etDFMSig/5cY8w=
184		-	github.com/dolthub/go-mysql-server v0.14.1-0.20221118195531-883fae8ef2a6/go.mod h1:z8i7fusnVa0hic93c/58X5tyF4lRj0e4yk3BhO7M0JY=
	183	+	github.com/dolthub/go-mysql-server v0.14.1-0.20221122003003-d6e384c64854 h1:djtBgdu8fj3MI7d+O9KWQy56TBocZ7KZGiNOOQqsO9c=
	184	+	github.com/dolthub/go-mysql-server v0.14.1-0.20221122003003-d6e384c64854/go.mod h1:drVceZC7lt+ZRzd0LnS7o3CURk7xGDDfjbsDfTKR7O0=
185	185		github.com/dolthub/ishell v0.0.0-20220112232610-14e753f0f371 h1:oyPHJlzumKta1vnOQqUnfdz+pk3EmnHS3Nd0cCT0I2g=
186	186		github.com/dolthub/ishell v0.0.0-20220112232610-14e753f0f371/go.mod h1:dhGBqcCEfK5kuFmeO5+WOx3hqc1k3M29c1oS/R7N4ms=
187	187		github.com/dolthub/jsonpath v0.0.0-20210609232853-d49537a30474 h1:xTrR+l5l+1Lfq0NvhiEsctylXinUMFhhsqaEcl414p8=
188	188		github.com/dolthub/jsonpath v0.0.0-20210609232853-d49537a30474/go.mod h1:kMz7uXOXq4qRriCEyZ/LUeTqraLJCjf0WVZcUi6TxUY=
189	189		github.com/dolthub/sqllogictest/go v0.0.0-20201107003712-816f3ae12d81 h1:7/v8q9XGFa6q5Ap4Z/OhNkAMBaK5YeuEzwJt+NZdhiE=
190	190		github.com/dolthub/sqllogictest/go v0.0.0-20201107003712-816f3ae12d81/go.mod h1:siLfyv2c92W1eN/R4QqG/+RjjX5W2+gCTRjZxBjI3TY=
191		-	github.com/dolthub/vitess v0.0.0-20221116234926-60a2fd96afae h1:3S1jX5/x+3S6Mhg9j2gpC4MxwRGw7RMEdJIo1c03g00=
192		-	github.com/dolthub/vitess v0.0.0-20221116234926-60a2fd96afae/go.mod h1:oVFIBdqMFEkt4Xz2fzFJBNtzKhDEjwdCF0dzde39iKs=
	191	+	github.com/dolthub/vitess v0.0.0-20221121184553-8d519d0bbb91 h1:xZRbvMhwTMkddrp4JVpA/APGI0kDAwvR35S5a46EbbU=
	192	+	github.com/dolthub/vitess v0.0.0-20221121184553-8d519d0bbb91/go.mod h1:oVFIBdqMFEkt4Xz2fzFJBNtzKhDEjwdCF0dzde39iKs=
193	193		github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
194	194		github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
195	195		github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
		skipped 1051 lines

■ ■ ■ ■ ■ ■

go/libraries/doltcore/doltdb/system_table.go

		skipped 19 lines
20	20		"sort"
21	21		"strings"
22	22
	23	+	"github.com/dolthub/dolt/go/libraries/doltcore/schema/typeinfo"
	24	+
23	25		"github.com/dolthub/dolt/go/libraries/doltcore/schema"
24	26		"github.com/dolthub/dolt/go/libraries/utils/funcitr"
25	27		"github.com/dolthub/dolt/go/libraries/utils/set"
		skipped 6 lines
32	34		)
33	35
34	36		var ErrSystemTableCannotBeModified = errors.New("system tables cannot be dropped or altered")
	37	+
	38	+	var OldDocsSchema = schema.MustSchemaFromCols(schema.NewColCollection(
	39	+	schema.NewColumn(DocPkColumnName, schema.DocNameTag, types.StringKind, true, schema.NotNullConstraint{}),
	40	+	schema.NewColumn(DocTextColumnName, schema.DocTextTag, types.StringKind, false),
	41	+	))
	42	+
	43	+	var DocsSchema schema.Schema
	44	+
	45	+	func init() {
	46	+	docTextCol, err := schema.NewColumnWithTypeInfo(DocTextColumnName, schema.DocTextTag, typeinfo.LongTextType, false, "", false, "")
	47	+	if err != nil {
	48	+	panic(err)
	49	+	}
	50	+	doltDocsColumns := schema.NewColCollection(
	51	+	schema.NewColumn(DocPkColumnName, schema.DocNameTag, types.StringKind, true, schema.NotNullConstraint{}),
	52	+	docTextCol,
	53	+	)
	54	+	DocsSchema = schema.MustSchemaFromCols(doltDocsColumns)
	55	+	}
35	56
36	57		// HasDoltPrefix returns a boolean whether or not the provided string is prefixed with the DoltNamespace. Users should
37	58		// not be able to create tables in this reserved namespace.
		skipped 124 lines
162	183		ReadmeDoc = "README.md"
163	184		)
164	185
165		-	var doltDocsColumns = schema.NewColCollection(
166		-	schema.NewColumn(DocPkColumnName, schema.DocNameTag, types.StringKind, true, schema.NotNullConstraint{}),
167		-	schema.NewColumn(DocTextColumnName, schema.DocTextTag, types.StringKind, false),
168		-	)
169		-	var DocsSchema = schema.MustSchemaFromCols(doltDocsColumns)
170		-
171	186		var DocsMaybeCreateTableStmt = `
172	187		CREATE TABLE IF NOT EXISTS dolt_docs (
173	188		doc_name varchar(16383) NOT NULL,
174		-	doc_text varchar(16383),
	189	+	doc_text longtext,
175	190		PRIMARY KEY (doc_name)
176	191		);`
177	192
		skipped 110 lines

■ ■ ■ ■ ■ ■

go/libraries/doltcore/migrate/tuples.go

		skipped 242 lines
243	243		case val.StringAddrEnc:
244	244		// note: previously, TEXT fields were serialized as types.String
245	245		rd := strings.NewReader(string(value))
246		-	t, err := tree.NewImmutableTreeFromReader(ctx, rd, ns, tree.DefaultFixedChunkLength)
	246	+	bb := ns.BlobBuilder()
	247	+	bb.Init(len(value))
	248	+	_, addr, err := bb.Chunk(ctx, rd)
247	249		if err != nil {
248	250		return err
249	251		}
250		-	b.PutStringAddr(idx, t.Addr)
	252	+	b.PutStringAddr(idx, addr)
251	253
252	254		default:
253	255		panic(fmt.Sprintf("unexpected encoding for string (%d)", typ.Enc))
		skipped 56 lines
310	312		}
311	313		buf := bytes.NewBuffer([]byte(s))
312	314
313		-	t, err := tree.NewImmutableTreeFromReader(ctx, buf, ns, tree.DefaultFixedChunkLength)
	315	+	bb := ns.BlobBuilder()
	316	+	bb.Init(len(s))
	317	+	_, addr, err := bb.Chunk(ctx, buf)
314	318		if err != nil {
315	319		return err
316	320		}
317		-	b.PutJSONAddr(idx, t.Addr)
	321	+	b.PutJSONAddr(idx, addr)
318	322		return nil
319	323		}
320	324
		skipped 17 lines
338	342		return err
339	343		}
340	344
341		-	t, err := tree.NewImmutableTreeFromReader(ctx, bytes.NewReader(buf), ns, tree.DefaultFixedChunkLength)
	345	+	bb := ns.BlobBuilder()
	346	+	bb.Init(int(value.Len()))
	347	+	_, addr, err := bb.Chunk(ctx, bytes.NewReader(buf))
342	348		if err != nil {
343	349		return err
344	350		}
		skipped 1 lines
346	352		typ := b.Desc.Types[idx]
347	353		switch typ.Enc {
348	354		case val.BytesAddrEnc:
349		-	b.PutBytesAddr(idx, t.Addr)
	355	+	b.PutBytesAddr(idx, addr)
350	356		case val.StringAddrEnc:
351		-	b.PutStringAddr(idx, t.Addr)
	357	+	b.PutStringAddr(idx, addr)
352	358		}
353	359		return nil
354	360		}
		skipped 34 lines

■ ■ ■ ■ ■ ■

go/libraries/doltcore/schema/schema_impl.go

		skipped 366 lines
367	367		_ = si.GetPKCols().Iter(func(tag uint64, col Column) (stop bool, err error) {
368	368		sqlType := col.TypeInfo.ToSqlType()
369	369		queryType := sqlType.Type()
370		-	tt = append(tt, val.Type{
371		-	Enc: val.Encoding(EncodingFromSqlType(queryType)),
372		-	Nullable: columnMissingNotNullConstraint(col),
373		-	})
374		-	if queryType == query.Type_CHAR \|\| queryType == query.Type_VARCHAR {
	370	+	var t val.Type
	371	+	if queryType == query.Type_BLOB {
	372	+	t = val.Type{
	373	+	Enc: val.Encoding(EncodingFromSqlType(query.Type_VARBINARY)),
	374	+	Nullable: columnMissingNotNullConstraint(col),
	375	+	}
	376	+	} else if queryType == query.Type_TEXT {
	377	+	t = val.Type{
	378	+	Enc: val.Encoding(EncodingFromSqlType(query.Type_VARCHAR)),
	379	+	Nullable: columnMissingNotNullConstraint(col),
	380	+	}
	381	+	} else {
	382	+	t = val.Type{
	383	+	Enc: val.Encoding(EncodingFromSqlType(queryType)),
	384	+	Nullable: columnMissingNotNullConstraint(col),
	385	+	}
	386	+	}
	387	+	tt = append(tt, t)
	388	+	if queryType == query.Type_CHAR \|\| queryType == query.Type_VARCHAR \|\| queryType == query.Type_TEXT {
375	389		useCollations = true
376	390		collations = append(collations, sqlType.(sql.StringType).Collation())
377	391		} else {
		skipped 108 lines

■ ■ ■ ■ ■ ■

go/libraries/doltcore/sqle/database.go

		skipped 850 lines
851	851		}
852	852		if strings.ToLower(tableName) == doltdb.DocTableName {
853	853		// validate correct schema
854		-	if !dtables.DoltDocsSqlSchema.Equals(sch.Schema) {
	854	+	if !dtables.DoltDocsSqlSchema.Equals(sch.Schema) && !dtables.OldDoltDocsSqlSchema.Equals(sch.Schema) {
855	855		return fmt.Errorf("incorrect schema for dolt_docs table")
856	856		}
857	857		} else if doltdb.HasDoltPrefix(tableName) {
		skipped 14 lines
872	872		}
873	873		if strings.ToLower(tableName) == doltdb.DocTableName {
874	874		// validate correct schema
875		-	if !dtables.DoltDocsSqlSchema.Equals(sch.Schema) {
	875	+	if !dtables.DoltDocsSqlSchema.Equals(sch.Schema) && !dtables.OldDoltDocsSqlSchema.Equals(sch.Schema) {
876	876		return fmt.Errorf("incorrect schema for dolt_docs table")
877	877		}
878	878		} else if doltdb.HasDoltPrefix(tableName) {
		skipped 558 lines

■ ■ ■ ■ ■ ■

go/libraries/doltcore/sqle/dtables/docs_table.go

		skipped 21 lines
22	22		)
23	23
24	24		var DoltDocsSqlSchema sql.PrimaryKeySchema
	25	+	var OldDoltDocsSqlSchema sql.PrimaryKeySchema
25	26
26	27		func init() {
27	28		DoltDocsSqlSchema, _ = sqlutil.FromDoltSchema(doltdb.DocTableName, doltdb.DocsSchema)
	29	+	OldDoltDocsSqlSchema, _ = sqlutil.FromDoltSchema(doltdb.DocTableName, doltdb.OldDocsSchema)
28	30		}
29	31

■ ■ ■ ■ ■ ■

go/libraries/doltcore/sqle/enginetest/dolt_queries.go

		skipped 8240 lines
8241	8241		ExpectedErr: sql.ErrUniqueKeyViolation,
8242	8242		},
8243	8243		{
	8244	+	Query: "insert into t values (99, 'ABC', 'ABCDE')",
	8245	+	ExpectedErrStr: "duplicate unique key given: [ABC,ABCDE]",
	8246	+	},
	8247	+	{
8244	8248		Query: "insert into t values (99, 'ABC123', 'ABCDE123')",
8245	8249		ExpectedErr: sql.ErrUniqueKeyViolation,
	8250	+	},
	8251	+	{
	8252	+	Query: "insert into t values (99, 'ABC123', 'ABCDE123')",
	8253	+	ExpectedErrStr: "duplicate unique key given: [ABC,ABCDE]",
8246	8254		},
8247	8255		{
8248	8256		Query: "select * from t where v1 = 'A'",
		skipped 120 lines
8369	8377		{
8370	8378		Query: "select * from t",
8371	8379		Expected: []sql.Row{},
	8380	+	},
	8381	+	},
	8382	+	},
	8383	+	// TODO: these should eventually go in GMS, but it doesn't currently support index rewrite on column modify
	8384	+	{
	8385	+	Name: "drop prefix lengths when modifying column to non string type",
	8386	+	SetUpScript: []string{
	8387	+	"create table t (j varchar(100), index (j(10)))",
	8388	+	},
	8389	+	Assertions: []queries.ScriptTestAssertion{
	8390	+	{
	8391	+	Query: "alter table t modify column j int",
	8392	+	Expected: []sql.Row{{sql.OkResult{}}},
	8393	+	},
	8394	+	{
	8395	+	Query: "show create table t",
	8396	+	Expected: []sql.Row{{"t", "CREATE TABLE `t` (\n `j` int,\n KEY `j` (`j`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_bin"}},
	8397	+	},
	8398	+	},
	8399	+	},
	8400	+	{
	8401	+	Name: "drop prefix length when modifying columns to invalid string type",
	8402	+	SetUpScript: []string{
	8403	+	"create table t (j varchar(100), index (j(10)))",
	8404	+	},
	8405	+	Assertions: []queries.ScriptTestAssertion{
	8406	+	{
	8407	+	Query: "alter table t modify column j varchar(2)",
	8408	+	Expected: []sql.Row{{sql.OkResult{}}},
	8409	+	},
	8410	+	{
	8411	+	Query: "show create table t",
	8412	+	Expected: []sql.Row{{"t", "CREATE TABLE `t` (\n `j` varchar(2),\n KEY `j` (`j`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_bin"}},
	8413	+	},
	8414	+	},
	8415	+	},
	8416	+	{
	8417	+	Name: "preserve prefix length when modifying column to valid string type",
	8418	+	SetUpScript: []string{
	8419	+	"create table t (j varchar(100), index (j(10)))",
	8420	+	},
	8421	+	Assertions: []queries.ScriptTestAssertion{
	8422	+	{
	8423	+	Query: "alter table t modify column j varchar(200)",
	8424	+	Expected: []sql.Row{{sql.OkResult{}}},
	8425	+	},
	8426	+	{
	8427	+	Query: "show create table t",
	8428	+	Expected: []sql.Row{{"t", "CREATE TABLE `t` (\n `j` varchar(200),\n KEY `j` (`j`(10))\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_bin"}},
	8429	+	},
	8430	+	},
	8431	+	},
	8432	+	{
	8433	+	Name: "preserve prefix lengths when there are other unchanged prefix lengths",
	8434	+	SetUpScript: []string{
	8435	+	"create table t (i varchar(100), j varchar(100), index (i(10), j(10)))",
	8436	+	},
	8437	+	Assertions: []queries.ScriptTestAssertion{
	8438	+	{
	8439	+	Query: "alter table t modify column j int",
	8440	+	Expected: []sql.Row{{sql.OkResult{}}},
	8441	+	},
	8442	+	{
	8443	+	Query: "show create table t",
	8444	+	Expected: []sql.Row{{"t", "CREATE TABLE `t` (\n `i` varchar(100),\n `j` int,\n KEY `ij` (`i`(10),`j`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_bin"}},
	8445	+	},
	8446	+	},
	8447	+	},
	8448	+	{
	8449	+	Name: "prefix length too long",
	8450	+	SetUpScript: []string{
	8451	+	"create table t (i blob, index(i(3072)))",
	8452	+	},
	8453	+	Assertions: []queries.ScriptTestAssertion{
	8454	+	{
	8455	+	Query: "alter table t modify column i text",
	8456	+	ExpectedErr: sql.ErrKeyTooLong,
8372	8457		},
8373	8458		},
8374	8459		},
		skipped 2 lines

■ ■ ■ ■ ■ ■

go/libraries/doltcore/sqle/index/dolt_index.go

		skipped 63 lines
64	64		return nil, nil
65	65		}
66	66
67		-	// TODO: do this for other indexes?
68	67		tableRows, err := t.GetRowData(ctx)
69	68		if err != nil {
70	69		return nil, err
71	70		}
72	71		keyBld := maybeGetKeyBuilder(tableRows)
73	72
74		-	// TODO: two primary keys???
75	73		cols := sch.GetPKCols().GetColumns()
76	74
77	75		// add to_ prefix
		skipped 19 lines
97	95		order: sql.IndexOrderAsc,
98	96		constrainedToLookupExpression: false,
99	97		}
100		-
101		-	// TODO: need to add from_ columns
102	98
103	99		return append(indexes, &toIndex), nil
104	100		}
		skipped 290 lines
395	391
396	392		// CanSupport implements sql.Index
397	393		func (di *doltIndex) CanSupport(...sql.Range) bool {
398		-	// TODO (james): don't use and prefix indexes if there's a prefix on a text/blob column
399		-	if len(di.prefixLengths) > 0 {
400		-	hasTextBlob := false
401		-	colColl := di.indexSch.GetAllCols()
402		-	colColl.Iter(func(tag uint64, col schema.Column) (stop bool, err error) {
403		-	if sql.IsTextBlob(col.TypeInfo.ToSqlType()) {
404		-	hasTextBlob = true
405		-	return true, nil
406		-	}
407		-	return false, nil
408		-	})
409		-	return !hasTextBlob
410		-	}
411	394		return true
412	395		}
413	396
		skipped 579 lines

■ ■ ■ ■ ■ ■

go/libraries/doltcore/sqle/index/prolly_fields.go

		skipped 193 lines
194	194		if err != nil {
195	195		return err
196	196		}
197		-	h, err := serializeBytesToAddr(ctx, ns, bytes.NewReader(buf))
	197	+	h, err := serializeBytesToAddr(ctx, ns, bytes.NewReader(buf), len(buf))
198	198		if err != nil {
199	199		return err
200	200		}
201	201		tb.PutJSONAddr(i, h)
202	202		case val.BytesAddrEnc:
203		-	h, err := serializeBytesToAddr(ctx, ns, bytes.NewReader(v.([]byte)))
	203	+	h, err := serializeBytesToAddr(ctx, ns, bytes.NewReader(v.([]byte)), len(v.([]byte)))
204	204		if err != nil {
205	205		return err
206	206		}
207	207		tb.PutBytesAddr(i, h)
208	208		case val.StringAddrEnc:
209	209		//todo: v will be []byte after daylon's changes
210		-	h, err := serializeBytesToAddr(ctx, ns, bytes.NewReader([]byte(v.(string))))
	210	+	h, err := serializeBytesToAddr(ctx, ns, bytes.NewReader([]byte(v.(string))), len(v.(string)))
211	211		if err != nil {
212	212		return err
213	213		}
		skipped 91 lines
305	305		}
306	306		}
307	307
308		-	func serializeBytesToAddr(ctx context.Context, ns tree.NodeStore, r io.Reader) (hash.Hash, error) {
309		-	tree, err := tree.NewImmutableTreeFromReader(ctx, r, ns, tree.DefaultFixedChunkLength)
	308	+	func serializeBytesToAddr(ctx context.Context, ns tree.NodeStore, r io.Reader, dataSize int) (hash.Hash, error) {
	309	+	bb := ns.BlobBuilder()
	310	+	bb.Init(dataSize)
	311	+	_, addr, err := bb.Chunk(ctx, r)
310	312		if err != nil {
311	313		return hash.Hash{}, err
312	314		}
313		-	return tree.Addr, nil
	315	+	return addr, nil
314	316		}
315	317
316	318		func convJson(v interface{}) (buf []byte, err error) {
		skipped 7 lines

■ ■ ■ ■ ■ ■

go/libraries/doltcore/sqle/sqlselect_test.go

		skipped 1291 lines
1292	1292		Name: "select from dolt_docs",
1293	1293		AdditionalSetup: CreateTableFn("dolt_docs", doltdb.DocsSchema,
1294	1294		"INSERT INTO dolt_docs VALUES ('LICENSE.md','A license')"),
1295		-	Query: "select * from dolt_docs",
1296		-	ExpectedRows: ToSqlRows(CompressSchema(doltdb.DocsSchema),
1297		-	NewRow(types.String("LICENSE.md"), types.String("A license"))),
	1295	+	Query: "select * from dolt_docs",
	1296	+	ExpectedRows: []sql.Row{{"LICENSE.md", "A license"}},
1298	1297		ExpectedSchema: CompressSchema(doltdb.DocsSchema),
1299	1298		},
1300	1299		{
		skipped 443 lines

■ ■ ■ ■ ■ ■ ■

go/libraries/doltcore/sqle/tables.go

		skipped 1334 lines
1335	1335		// table to rewrite it we also truncate all the indexes. Much easier to get right.
1336	1336		for _, index := range oldSch.Indexes().AllIndexes() {
1337	1337		var colNames []string
1338		-	for _, colName := range index.ColumnNames() {
	1338	+	prefixLengths := index.PrefixLengths()
	1339	+	for i, colName := range index.ColumnNames() {
1339	1340		if strings.ToLower(oldColumn.Name) == strings.ToLower(colName) {
1340	1341		colNames = append(colNames, newColumn.Name)
	1342	+	if len(prefixLengths) > 0 {
	1343	+	if !sql.IsText(newColumn.Type) {
	1344	+	// drop prefix lengths if column is not a string type
	1345	+	prefixLengths[i] = 0
	1346	+	} else if uint32(prefixLengths[i]) > newColumn.Type.MaxTextResponseByteLength() {
	1347	+	// drop prefix length if prefixLength is too long
	1348	+	prefixLengths[i] = 0
	1349	+	}
	1350	+	}
1341	1351		} else {
1342	1352		colNames = append(colNames, colName)
1343	1353		}
1344	1354		}
	1355	+
	1356	+	// check if prefixLengths should be dropped entirely
	1357	+	var nonZeroPrefixLength bool
	1358	+	for _, prefixLength := range prefixLengths {
	1359	+	if prefixLength > 0 {
	1360	+	nonZeroPrefixLength = true
	1361	+	break
	1362	+	}
	1363	+	}
	1364	+	if !nonZeroPrefixLength {
	1365	+	prefixLengths = nil
	1366	+	}
	1367	+
1345	1368		newSch.Indexes().AddIndexByColNames(
1346	1369		index.Name(),
1347	1370		colNames,
1348		-	index.PrefixLengths(),
	1371	+	prefixLengths,
1349	1372		schema.IndexProperties{
1350	1373		IsUnique: index.IsUnique(),
1351	1374		IsUserDefined: index.IsUserDefined(),
		skipped 1361 lines

■ ■ ■ ■ ■ ■

go/performance/import_benchmarker/testdata/blob.yaml

1	+	opts:
2	+	seed: 0
3	+	tests:
4	+	- name: "sql"
5	+	repos:
6	+	- name: dolt
7	+	server:
8	+	port: 3308
9	+	args: [ "--port", "3308" ]
10	+	# - name: mysql # mysqld --port 3308 --local-infile=1 --socket=/tmp/mysqld2.sock
11	+	# external-server:
12	+	# name: test
13	+	# host: 127.0.0.1
14	+	# user: root
15	+	# password:
16	+	# port: 3309
17	+	tables:
18	+	- name: "1 blob"
19	+	rows: 1
20	+	schema: \|
21	+	create table xy (
22	+	x int primary key,
23	+	y blob,
24	+	z varchar(30),
25	+	w varchar(30)
26	+	);
27	+	- name: "1 blob"
28	+	rows: 10
29	+	schema: \|
30	+	create table xy (
31	+	x int primary key,
32	+	y blob,
33	+	z varchar(30),
34	+	w varchar(30)
35	+	);
36	+	- name: "1 blob"
37	+	rows: 20
38	+	schema: \|
39	+	create table xy (
40	+	x int primary key,
41	+	y blob,
42	+	z varchar(30),
43	+	w varchar(30)
44	+	);
45	+	- name: "1 blob"
46	+	rows: 40
47	+	schema: \|
48	+	create table xy (
49	+	x int primary key,
50	+	y blob,
51	+	z varchar(30),
52	+	w varchar(30)
53	+	);
54	+	- name: "1 blob"
55	+	rows: 50
56	+	schema: \|
57	+	create table xy (
58	+	x int primary key,
59	+	y blob,
60	+	z varchar(30),
61	+	w varchar(30)
62	+	);
63	+	- name: "1 blob"
64	+	rows: 60
65	+	schema: \|
66	+	create table xy (
67	+	x int primary key,
68	+	y blob,
69	+	z varchar(30),
70	+	w varchar(30)
71	+	);
72	+	- name: "1 blob"
73	+	rows: 80
74	+	schema: \|
75	+	create table xy (
76	+	x int primary key,
77	+	y blob,
78	+	z varchar(30),
79	+	w varchar(30)
80	+	);
81	+	- name: "1 blob"
82	+	rows: 100
83	+	schema: \|
84	+	create table xy (
85	+	x int primary key,
86	+	y blob,
87	+	z varchar(30),
88	+	w varchar(30)
89	+	);
90	+	- name: "1 blob"
91	+	rows: 200
92	+	schema: \|
93	+	create table xy (
94	+	x int primary key,
95	+	y blob,
96	+	z varchar(30),
97	+	w varchar(30)
98	+	);
99	+	- name: "1 blob"
100	+	rows: 400
101	+	schema: \|
102	+	create table xy (
103	+	x int primary key,
104	+	y blob,
105	+	z varchar(30),
106	+	w varchar(30)
107	+	);
108	+	- name: "1 blob"
109	+	rows: 600
110	+	schema: \|
111	+	create table xy (
112	+	x int primary key,
113	+	y blob,
114	+	z varchar(30),
115	+	w varchar(30)
116	+	);
117	+	- name: "1 blob"
118	+	rows: 800
119	+	schema: \|
120	+	create table xy (
121	+	x int primary key,
122	+	y blob,
123	+	z varchar(30),
124	+	w varchar(30)
125	+	);
126	+	- name: "1 blob"
127	+	rows: 1000
128	+	schema: \|
129	+	create table xy (
130	+	x int primary key,
131	+	y blob,
132	+	z varchar(30),
133	+	w varchar(30)
134	+	);
135	+	- name: "1 blob"
136	+	rows: 10000
137	+	schema: \|
138	+	create table xy (
139	+	x int primary key,
140	+	y blob,
141	+	z varchar(30),
142	+	w varchar(30)
143	+	);
144	+	- name: "1 blob"
145	+	rows: 100000
146	+	schema: \|
147	+	create table xy (
148	+	x int primary key,
149	+	y blob,
150	+	z varchar(30),
151	+	w varchar(30)
152	+	);
153	+

■ ■ ■ ■ ■ ■

go/store/prolly/tree/blob_bench_test.go

1	+	// Copyright 2022 Dolthub, Inc.
2	+	//
3	+	// Licensed under the Apache License, Version 2.0 (the "License");
4	+	// you may not use this file except in compliance with the License.
5	+	// You may obtain a copy of the License at
6	+	//
7	+	// http://www.apache.org/licenses/LICENSE-2.0
8	+	//
9	+	// Unless required by applicable law or agreed to in writing, software
10	+	// distributed under the License is distributed on an "AS IS" BASIS,
11	+	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12	+	// See the License for the specific language governing permissions and
13	+	// limitations under the License.
14	+
15	+	package tree
16	+
17	+	import (
18	+	"bytes"
19	+	"context"
20	+	"fmt"
21	+	"math"
22	+	"testing"
23	+
24	+	"github.com/dolthub/dolt/go/store/hash"
25	+	)
26	+
27	+	var result hash.Hash
28	+
29	+	func BenchmarkBlobBuilder(b *testing.B) {
30	+	var r hash.Hash
31	+	var err error
32	+	dataSizes := []int{1e3, 1e4, 1e5, 1e6}
33	+	for _, d := range dataSizes {
34	+	b.Run(fmt.Sprintf("datasize: %.0f", math.Log10(float64(d))), func(b *testing.B) {
35	+	ns := NewTestNodeStore()
36	+	bb := mustNewBlobBuilder(DefaultFixedChunkLength)
37	+	bb.SetNodeStore(ns)
38	+	buf := make([]byte, d)
39	+	for i := 0; i < d; i++ {
40	+	buf[i] = uint8(i)
41	+	}
42	+
43	+	b.ResetTimer()
44	+	for n := 0; n < b.N; n++ {
45	+	// always record the result to prevent
46	+	// the compiler eliminating the function call.
47	+	bb.Init(d)
48	+	_, r, err = bb.Chunk(context.Background(), bytes.NewReader(buf))
49	+	if err != nil {
50	+	b.Fatal(err)
51	+	}
52	+	bb.Reset()
53	+	}
54	+	// always store the result to a package level variable
55	+	// so the compiler cannot eliminate the Benchmark itself.
56	+	result = r
57	+	})
58	+	}
59	+	}
60	+

■ ■ ■ ■ ■ ■

go/store/prolly/tree/blob_builder.go

1	+	// Copyright 2022 Dolthub, Inc.
2	+	//
3	+	// Licensed under the Apache License, Version 2.0 (the "License");
4	+	// you may not use this file except in compliance with the License.
5	+	// You may obtain a copy of the License at
6	+	//
7	+	// http://www.apache.org/licenses/LICENSE-2.0
8	+	//
9	+	// Unless required by applicable law or agreed to in writing, software
10	+	// distributed under the License is distributed on an "AS IS" BASIS,
11	+	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12	+	// See the License for the specific language governing permissions and
13	+	// limitations under the License.
14	+
15	+	package tree
16	+
17	+	import (
18	+	"bytes"
19	+	"context"
20	+	"encoding/json"
21	+	"errors"
22	+	"io"
23	+
24	+	"github.com/dolthub/go-mysql-server/sql"
25	+
26	+	"github.com/dolthub/dolt/go/store/hash"
27	+	"github.com/dolthub/dolt/go/store/prolly/message"
28	+	)
29	+
30	+	const DefaultFixedChunkLength = 4000
31	+
32	+	var ErrInvalidChunkSize = errors.New("invalid chunkSize; value must be a multiple of 20")
33	+
34	+	func mustNewBlobBuilder(chunkSize int) *BlobBuilder {
35	+	b, _ := NewBlobBuilder(chunkSize)
36	+	return b
37	+	}
38	+
39	+	// NewBlobBuilder writes the contents of \|reader\| as an append-only
40	+	// tree, returning the root node or an error if applicable. \|chunkSize\|
41	+	// fixes the split size of leaf and intermediate node chunks.
42	+	func NewBlobBuilder(chunkSize int) (*BlobBuilder, error) {
43	+	if chunkSize%hash.ByteLen != 0 {
44	+	return nil, ErrInvalidChunkSize
45	+	}
46	+
47	+	keys := make([][]byte, chunkSize/hash.ByteLen)
48	+	for i := range keys {
49	+	keys[i] = zeroKey
50	+	}
51	+	return &BlobBuilder{
52	+	chunkSize: chunkSize,
53	+	keys: keys,
54	+	}, nil
55	+	}
56	+
57	+	type blobNodeWriter interface {
58	+	Write(ctx context.Context, r io.Reader) (hash.Hash, uint64, error)
59	+	}
60	+
61	+	type BlobBuilder struct {
62	+	ns NodeStore
63	+	S message.Serializer
64	+	chunkSize int
65	+	keys [][]byte
66	+	wr blobNodeWriter
67	+	lastN Node
68	+	topLevel int
69	+
70	+	levelCap int
71	+	buf []byte
72	+	vals [][]byte
73	+	subtrees []uint64
74	+	}
75	+
76	+	func (b *BlobBuilder) SetNodeStore(ns NodeStore) {
77	+	b.ns = ns
78	+	b.S = message.NewBlobSerializer(ns.Pool())
79	+	}
80	+
81	+	// Reset clears the BlobBuilder for re-use.
82	+	func (b *BlobBuilder) Reset() {
83	+	b.wr = nil
84	+	b.topLevel = 0
85	+	}
86	+
87	+	// Init calculates tree dimensions for a given blob.
88	+	func (b *BlobBuilder) Init(dataSize int) {
89	+	b.Reset()
90	+
91	+	if dataSize == 0 {
92	+	return
93	+	}
94	+
95	+	if dataSize <= b.chunkSize {
96	+	b.wr = &blobLeafWriter{
97	+	bb: b,
98	+	buf: make([]byte, dataSize),
99	+	}
100	+	return
101	+	}
102	+
103	+	b.wr = &blobLeafWriter{
104	+	bb: b,
105	+	buf: make([]byte, b.chunkSize),
106	+	}
107	+
108	+	numAddrs := b.chunkSize / hash.ByteLen
109	+	dataSize = dataSize / b.chunkSize
110	+	for dataSize > 0 {
111	+	dataSize = dataSize / numAddrs
112	+	b.topLevel += 1
113	+	}
114	+
115	+	// Allocate everything we need in batch, slice them up down below.
116	+	if b.levelCap < b.topLevel {
117	+	b.expand(numAddrs)
118	+	b.levelCap = b.topLevel
119	+	}
120	+
121	+	writers := make([]blobLevelWriter, b.topLevel)
122	+	for i, addrs := 0, 0; i < b.topLevel; i, addrs = i+1, addrs+numAddrs {
123	+	wr := &writers[i]
124	+	wr.bb = b
125	+	wr.child = b.wr
126	+	wr.buf = b.buf[addrshash.ByteLen : (addrs+numAddrs)hash.ByteLen]
127	+	wr.vals = b.vals[addrs : addrs+numAddrs]
128	+	wr.subtrees = b.subtrees[addrs : addrs+numAddrs]
129	+	wr.level = i + 1
130	+	wr.sz = numAddrs
131	+	b.wr = wr
132	+	}
133	+	}
134	+
135	+	func (b *BlobBuilder) expand(numAddrs int) {
136	+	b.buf = make([]byte, b.topLevelnumAddrshash.ByteLen)
137	+	b.vals = make([][]byte, numAddrs*b.topLevel)
138	+	b.subtrees = make([]uint64, numAddrs*b.topLevel)
139	+	}
140	+
141	+	// Chunk builds the blob tree by passing the Reader to the chain of level
142	+	// writers, terminated in a leaf writer. The leaf writer reads chunks from the
143	+	// Reader and writes them, returning their hashes to its parent level writer.
144	+	// When the parent level writer fills up with addresses, it writes a chunk and
145	+	// returns that address to its parent. This continues until the Reader returns
146	+	// io.EOF, when every writer in the chain completes its chunk and we return the
147	+	// root node.
148	+	func (b *BlobBuilder) Chunk(ctx context.Context, r io.Reader) (Node, hash.Hash, error) {
149	+	if b.wr == nil {
150	+	return Node{}, hash.Hash{}, nil
151	+	}
152	+	h, _, err := b.wr.Write(ctx, r)
153	+	if err != nil && err != io.EOF {
154	+	return Node{}, hash.Hash{}, err
155	+	}
156	+	return b.lastN, h, nil
157	+	}
158	+
159	+	// blobLeafWriter writes leaf chunks of the blob, with max capacity len(buf),
160	+	// for every call to Write().
161	+	type blobLeafWriter struct {
162	+	bb *BlobBuilder
163	+	buf []byte
164	+	}
165	+
166	+	var zeroKey = []byte{0}
167	+	var zeroKeys = [][]byte{zeroKey}
168	+	var leafSubtrees = []uint64{1}
169	+
170	+	func (lw *blobLeafWriter) Write(ctx context.Context, r io.Reader) (hash.Hash, uint64, error) {
171	+	n, err := r.Read(lw.buf)
172	+	if err != nil {
173	+	return hash.Hash{}, 0, err
174	+	}
175	+	h, err := lw.bb.write(ctx, zeroKeys, [][]byte{lw.buf[:n]}, leafSubtrees, 0)
176	+	return h, 1, err
177	+	}
178	+
179	+	// blobLevelWriters writes internal chunks of a blob, using its \|child\| to
180	+	// write the level below it. On a call to \|Write\|, it repeatedly calls
181	+	// \|child.Write\|, accumulating addresses to its children, until it fills up or
182	+	// the Reader is exhausted. In either case, it then writes its node and
183	+	// returns.
184	+	type blobLevelWriter struct {
185	+	bb *BlobBuilder
186	+	child blobNodeWriter
187	+	buf []byte
188	+	vals [][]byte
189	+	subtrees []uint64
190	+	sz int
191	+	level int
192	+	}
193	+
194	+	func (lw *blobLevelWriter) Write(ctx context.Context, r io.Reader) (hash.Hash, uint64, error) {
195	+	i, off, totalCount := 0, 0, uint64(0)
196	+	for {
197	+	// Sketchy hack to elide a copy here...
198	+	//h := (*hash.Hash)(unsafe.Pointer(&lw.buf[off]))
199	+	//var n uint64
200	+	//var err error
201	+	h, n, err := lw.child.Write(ctx, r)
202	+	if err != nil && err != io.EOF {
203	+	return hash.Hash{}, 0, err
204	+	}
205	+	if n != 0 {
206	+	totalCount += n
207	+	copy(lw.buf[off:], h[:])
208	+	lw.subtrees[i] = n
209	+	lw.vals[i] = lw.buf[off : off+hash.ByteLen]
210	+	i += 1
211	+	off += hash.ByteLen
212	+	}
213	+	if i >= lw.sz \|\| err == io.EOF {
214	+	h, nerr := lw.bb.write(ctx, lw.bb.keys[:i], lw.vals[:i], lw.subtrees[:i], lw.level)
215	+	if nerr != nil {
216	+	return hash.Hash{}, 0, nerr
217	+	}
218	+	return h, totalCount, err
219	+	}
220	+	}
221	+	}
222	+
223	+	// Write the blob node. Called by level and leaf writers. Will store lastN if
224	+	// the level corresponds to our root level.
225	+	func (b *BlobBuilder) write(ctx context.Context, keys, vals [][]byte, subtrees []uint64, level int) (hash.Hash, error) {
226	+	msg := b.S.Serialize(keys, vals, subtrees, level)
227	+	node, err := NodeFromBytes(msg)
228	+	if err != nil {
229	+	return hash.Hash{}, err
230	+	}
231	+	h, err := b.ns.Write(ctx, node)
232	+	if err != nil {
233	+	return hash.Hash{}, err
234	+	}
235	+	if level == b.topLevel {
236	+	b.lastN = node
237	+	}
238	+	return h, nil
239	+	}
240	+
241	+	const bytePeekLength = 128
242	+
243	+	type ByteArray struct {
244	+	ImmutableTree
245	+	}
246	+
247	+	func NewByteArray(addr hash.Hash, ns NodeStore) *ByteArray {
248	+	return &ByteArray{ImmutableTree{Addr: addr, ns: ns}}
249	+	}
250	+
251	+	func (b *ByteArray) ToBytes(ctx context.Context) ([]byte, error) {
252	+	return b.bytes(ctx)
253	+	}
254	+
255	+	func (b *ByteArray) ToString(ctx context.Context) (string, error) {
256	+	buf, err := b.bytes(ctx)
257	+	if err != nil {
258	+	return "", err
259	+	}
260	+	toShow := bytePeekLength
261	+	if len(buf) < toShow {
262	+	toShow = len(buf)
263	+	}
264	+	return string(buf[:toShow]), nil
265	+	}
266	+
267	+	type JSONDoc struct {
268	+	ImmutableTree
269	+	}
270	+
271	+	func NewJSONDoc(addr hash.Hash, ns NodeStore) *JSONDoc {
272	+	return &JSONDoc{ImmutableTree{Addr: addr, ns: ns}}
273	+	}
274	+
275	+	func (b *JSONDoc) ToJSONDocument(ctx context.Context) (sql.JSONDocument, error) {
276	+	buf, err := b.bytes(ctx)
277	+	if err != nil {
278	+	return sql.JSONDocument{}, err
279	+	}
280	+	var doc sql.JSONDocument
281	+	err = json.Unmarshal(buf, &doc.Val)
282	+	if err != nil {
283	+	return sql.JSONDocument{}, err
284	+	}
285	+	return doc, err
286	+	}
287	+
288	+	func (b *JSONDoc) ToString(ctx context.Context) (string, error) {
289	+	buf, err := b.bytes(ctx)
290	+	if err != nil {
291	+	return "", err
292	+	}
293	+	toShow := bytePeekLength
294	+	if len(buf) < toShow {
295	+	toShow = len(buf)
296	+	}
297	+	return string(buf[:toShow]), nil
298	+	}
299	+
300	+	type TextStorage struct {
301	+	ImmutableTree
302	+	}
303	+
304	+	func NewTextStorage(addr hash.Hash, ns NodeStore) *TextStorage {
305	+	return &TextStorage{ImmutableTree{Addr: addr, ns: ns}}
306	+	}
307	+
308	+	func (b *TextStorage) ToBytes(ctx context.Context) ([]byte, error) {
309	+	return b.bytes(ctx)
310	+	}
311	+
312	+	func (b *TextStorage) ToString(ctx context.Context) (string, error) {
313	+	buf, err := b.bytes(ctx)
314	+	if err != nil {
315	+	return "", err
316	+	}
317	+	return string(buf), nil
318	+	}
319	+
320	+	type ImmutableTree struct {
321	+	Addr hash.Hash
322	+	buf []byte
323	+	ns NodeStore
324	+	}
325	+
326	+	func (t *ImmutableTree) load(ctx context.Context) error {
327	+	if t.Addr.IsEmpty() {
328	+	t.buf = []byte{}
329	+	return nil
330	+	}
331	+	n, err := t.ns.Read(ctx, t.Addr)
332	+	if err != nil {
333	+	return err
334	+	}
335	+
336	+	return WalkNodes(ctx, n, t.ns, func(ctx context.Context, n Node) error {
337	+	if n.IsLeaf() {
338	+	t.buf = append(t.buf, n.GetValue(0)...)
339	+	}
340	+	return nil
341	+	})
342	+	}
343	+
344	+	func (t *ImmutableTree) bytes(ctx context.Context) ([]byte, error) {
345	+	if t.buf == nil {
346	+	err := t.load(ctx)
347	+	if err != nil {
348	+	return nil, err
349	+	}
350	+	}
351	+	return t.buf[:], nil
352	+	}
353	+
354	+	func (t *ImmutableTree) next() (Node, error) {
355	+	panic("not implemented")
356	+	}
357	+
358	+	func (t *ImmutableTree) close() error {
359	+	panic("not implemented")
360	+	}
361	+
362	+	func (t *ImmutableTree) Read(_ bytes.Buffer) (int, error) {
363	+	panic("not implemented")
364	+	}
365	+

■ ■ ■ ■ ■ ■

go/store/prolly/tree/immutable_tree_test.go go/store/prolly/tree/blob_builder_test.go

		skipped 34 lines
35	35		tests := []struct {
36	36		inputSize int
37	37		chunkSize int
38		-	err error
	38	+	execErr error
	39	+	initErr error
39	40		checkSum bool
40	41		}{
41	42		{
		skipped 6 lines
48	49		},
49	50		{
50	51		inputSize: 100,
51		-	chunkSize: 101,
	52	+	chunkSize: 100,
52	53		},
53	54		{
54	55		inputSize: 255,
		skipped 25 lines
80	81		},
81	82		{
82	83		inputSize: 1_000,
83		-	chunkSize: 47,
	84	+	chunkSize: 40,
84	85		checkSum: false,
85	86		},
86	87		{
87	88		inputSize: 1_000,
88		-	chunkSize: 53,
	89	+	chunkSize: 60,
89	90		checkSum: false,
90	91		},
91	92		{
92	93		inputSize: 1_000,
93		-	chunkSize: 67,
	94	+	chunkSize: 80,
94	95		checkSum: false,
95	96		},
96	97		{
97	98		inputSize: 10_000,
98		-	chunkSize: 89,
	99	+	chunkSize: 100,
99	100		checkSum: false,
100	101		},
101	102		{
		skipped 7 lines
109	110		checkSum: false,
110	111		},
111	112		{
112		-	inputSize: 50_000_000,
113		-	chunkSize: 33_000,
114		-	err: ErrInvalidChunkSize,
115		-	},
116		-	{
117		-	inputSize: 10,
118		-	chunkSize: 1,
119		-	err: ErrInvalidChunkSize,
	113	+	inputSize: 0,
	114	+	chunkSize: 40,
120	115		},
121	116		{
122		-	inputSize: 10,
123		-	chunkSize: -1,
124		-	err: ErrInvalidChunkSize,
125		-	},
126		-	{
127		-	inputSize: 10,
128		-	chunkSize: 39,
129		-	err: ErrInvalidChunkSize,
	117	+	inputSize: 100,
	118	+	chunkSize: 41,
	119	+	initErr: ErrInvalidChunkSize,
130	120		},
131	121		}
132	122
		skipped 6 lines
139	129		ctx := context.Background()
140	130		r := bytes.NewReader(buf)
141	131		ns := NewTestNodeStore()
142		-	serializer := message.NewBlobSerializer(ns.Pool())
143		-	root, err := buildImmutableTree(ctx, r, ns, serializer, tt.chunkSize)
144		-	if tt.err != nil {
145		-	require.True(t, errors.Is(err, tt.err))
	132	+	//serializer := message.NewBlobSerializer(ns.Pool())
	133	+
	134	+	b, err := NewBlobBuilder(tt.chunkSize)
	135	+	if tt.initErr != nil {
	136	+	require.True(t, errors.Is(err, tt.initErr))
	137	+	return
	138	+	}
	139	+	b.SetNodeStore(ns)
	140	+	b.Init(tt.inputSize)
	141	+	root, _, err := b.Chunk(ctx, r)
	142	+
	143	+	if tt.execErr != nil {
	144	+	require.True(t, errors.Is(err, tt.execErr))
146	145		return
147	146		}
148	147		require.NoError(t, err)
		skipped 9 lines
158	157		sum := 0
159	158		byteCnt := 0
160	159		WalkNodes(ctx, root, ns, func(ctx context.Context, n Node) error {
	160	+	if n.empty() {
	161	+	return nil
	162	+	}
161	163		var keyCnt int
162	164		leaf := n.IsLeaf()
163	165		if leaf {
		skipped 78 lines
242	244		}
243	245
244	246		func expectedUnfilled(size, chunk int) int {
245		-	if size == chunk {
	247	+	if size == chunk \|\| size == 0 {
246	248		return 0
247	249		} else if size < chunk {
248	250		return 1
		skipped 27 lines
276	278		}{
277	279		{
278	280		blobLen: 250,
279		-	chunkSize: 41,
	281	+	chunkSize: 60,
280	282		keyCnt: 4,
281	283		},
282	284		{
		skipped 3 lines
286	288		},
287	289		{
288	290		blobLen: 378,
289		-	chunkSize: 43,
	291	+	chunkSize: 60,
290	292		keyCnt: 12,
291	293		},
292	294		{
		skipped 7 lines
300	302		keyCnt: 6,
301	303		},
302	304		{
303		-	blobLen: 0,
304		-	chunkSize: 40,
305		-	keyCnt: 6,
306		-	},
307		-	{
308	305		blobLen: 50_000_000,
309	306		chunkSize: 4000,
310	307		keyCnt: 1,
311	308		},
312	309		{
313	310		blobLen: 10_000,
314		-	chunkSize: 83,
	311	+	chunkSize: 80,
315	312		keyCnt: 6,
316	313		},
317	314		}
		skipped 44 lines
362	359		keyBld.PutUint32(0, uint32(i))
363	360		tuples[i][0] = keyBld.Build(sharedPool)
364	361
365		-	b := mustNewBlob(ctx, ns, blobLen, chunkSize)
366		-	valBld.PutBytesAddr(0, b.Addr)
	362	+	addr := mustNewBlob(ctx, ns, blobLen, chunkSize)
	363	+	valBld.PutBytesAddr(0, addr)
367	364		tuples[i][1] = valBld.Build(sharedPool)
368	365		}
369	366
		skipped 9 lines
379	376		return root
380	377		}
381	378
382		-	func mustNewBlob(ctx context.Context, ns NodeStore, len, chunkSize int) *ImmutableTree {
	379	+	func mustNewBlob(ctx context.Context, ns NodeStore, len, chunkSize int) hash.Hash {
383	380		buf := make([]byte, len)
384	381		for i := range buf {
385	382		buf[i] = byte(i)
386	383		}
387	384		r := bytes.NewReader(buf)
388		-	root, err := NewImmutableTreeFromReader(ctx, r, ns, chunkSize)
	385	+	b, err := NewBlobBuilder(chunkSize)
389	386		if err != nil {
390	387		panic(err)
391	388		}
392		-	return root
	389	+	b.SetNodeStore(ns)
	390	+	b.Init(len)
	391	+	_, addr, err := b.Chunk(ctx, r)
	392	+	if err != nil {
	393	+	panic(err)
	394	+	}
	395	+	return addr
393	396		}
394	397
395	398		func getBlobValues(msg serial.Message) []byte {
		skipped 8 lines

■ ■ ■ ■ ■ ■

go/store/prolly/tree/immutable_tree.go

1	-	// Copyright 2022 Dolthub, Inc.
2	-	//
3	-	// Licensed under the Apache License, Version 2.0 (the "License");
4	-	// you may not use this file except in compliance with the License.
5	-	// You may obtain a copy of the License at
6	-	//
7	-	// http://www.apache.org/licenses/LICENSE-2.0
8	-	//
9	-	// Unless required by applicable law or agreed to in writing, software
10	-	// distributed under the License is distributed on an "AS IS" BASIS,
11	-	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12	-	// See the License for the specific language governing permissions and
13	-	// limitations under the License.
14	-
15	-	package tree
16	-
17	-	import (
18	-	"bytes"
19	-	"context"
20	-	"encoding/json"
21	-	"errors"
22	-	"io"
23	-
24	-	"github.com/dolthub/go-mysql-server/sql"
25	-
26	-	"github.com/dolthub/dolt/go/store/hash"
27	-	"github.com/dolthub/dolt/go/store/prolly/message"
28	-	)
29	-
30	-	const DefaultFixedChunkLength = 4000
31	-
32	-	var ErrInvalidChunkSize = errors.New("invalid chunkSize; value must be > 1")
33	-
34	-	// buildImmutableTree writes the contents of \|reader\| as an append-only
35	-	// tree, returning the root node or an error if applicable. \|chunkSize\|
36	-	// fixes the split size of leaf and intermediate node chunks.
37	-	func buildImmutableTree(ctx context.Context, r io.Reader, ns NodeStore, S message.Serializer, chunkSize int) (Node, error) {
38	-	if chunkSize < hash.ByteLen*2 \|\| chunkSize > int(message.MaxVectorOffset)/2 {
39	-	// internal nodes must fit at least two 20-byte hashes
40	-	return Node{}, ErrInvalidChunkSize
41	-	}
42	-
43	-	var levels [][]novelNode
44	-	var levelCnts []int
45	-	var finalize bool
46	-
47	-	// We use lookahead to check whether the reader has
48	-	// more bytes. The reader will only EOF when reading
49	-	// zero bytes into the lookahead buffer, but we want
50	-	// to know at the beginning of a loop whether we are
51	-	// finished.
52	-	lookahead := make([]byte, chunkSize)
53	-	lookaheadN, err := r.Read(lookahead)
54	-	if err != nil {
55	-	return Node{}, err
56	-	}
57	-
58	-	buf := make([]byte, chunkSize)
59	-	for {
60	-	copy(buf, lookahead)
61	-	curN := lookaheadN
62	-	lookaheadN, err = r.Read(lookahead)
63	-	if err == io.EOF {
64	-	finalize = true
65	-	} else if err != nil {
66	-	return Node{}, err
67	-	}
68	-
69	-	novel, err := _newLeaf(ctx, ns, S, buf[:curN])
70	-	if err != nil {
71	-	return Node{}, err
72	-	}
73	-
74	-	i := 0
75	-	for {
76	-	// Three cases for building tree
77	-	// 1) reached new level => create new level
78	-	// 2) add novel node to current level
79	-	// 3) we didn't fill the current level => break
80	-	// 4) we filled current level, chunk and recurse into parent
81	-	//
82	-	// Two cases for finalizing tree
83	-	// 1) we haven't hit root, so we add the final chunk, finalize level, and continue upwards
84	-	// 2) we overshot root finalizing chunks, and we return the single root in the lower level
85	-	if i > len(levels)-1 {
86	-	levels = append(levels, make([]novelNode, chunkSize))
87	-	levelCnts = append(levelCnts, 0)
88	-	}
89	-
90	-	levels[i][levelCnts[i]] = novel
91	-	levelCnts[i]++
92	-	// note: the size of an internal node will be the key count times key length (hash)
93	-	if levelCnts[i]*hash.ByteLen < chunkSize {
94	-	// current level is not full
95	-	if !finalize {
96	-	// only continue and chunk this level if finalizing all in-progress nodes
97	-	break
98	-	}
99	-	}
100	-
101	-	nodes := levels[i][:levelCnts[i]]
102	-	if len(nodes) == 1 && i == len(levels)-1 {
103	-	// this is necessary and only possible if we're finalizing
104	-	// note: this is the only non-error return
105	-	return nodes[0].node, nil
106	-	}
107	-
108	-	// chunk the current level
109	-	novel, err = _newInternal(ctx, ns, S, nodes, i+1, chunkSize)
110	-	if err != nil {
111	-	return Node{}, err
112	-	}
113	-	levelCnts[i] = 0
114	-	i++
115	-	}
116	-	}
117	-	}
118	-
119	-	func _newInternal(ctx context.Context, ns NodeStore, s message.Serializer, nodes []novelNode, level int, chunkSize int) (novelNode, error) {
120	-	keys := make([][]byte, len(nodes))
121	-	vals := make([][]byte, len(nodes))
122	-	subtrees := make([]uint64, len(nodes))
123	-	treeCnt := uint64(0)
124	-	for i := range nodes {
125	-	keys[i] = []byte{0}
126	-	vals[i] = nodes[i].addr[:]
127	-	subtrees[i] = nodes[i].treeCount
128	-	treeCnt += nodes[i].treeCount
129	-	}
130	-	msg := s.Serialize(keys, vals, subtrees, level)
131	-	node, err := NodeFromBytes(msg)
132	-	if err != nil {
133	-	return novelNode{}, err
134	-	}
135	-	addr, err := ns.Write(ctx, node)
136	-	if err != nil {
137	-	return novelNode{}, err
138	-	}
139	-	return novelNode{
140	-	addr: addr,
141	-	node: node,
142	-	lastKey: []byte{0},
143	-	treeCount: treeCnt,
144	-	}, nil
145	-	}
146	-
147	-	func _newLeaf(ctx context.Context, ns NodeStore, s message.Serializer, buf []byte) (novelNode, error) {
148	-	msg := s.Serialize([][]byte{{0}}, [][]byte{buf}, []uint64{1}, 0)
149	-	node, err := NodeFromBytes(msg)
150	-	if err != nil {
151	-	return novelNode{}, err
152	-	}
153	-	addr, err := ns.Write(ctx, node)
154	-	if err != nil {
155	-	return novelNode{}, err
156	-	}
157	-	return novelNode{
158	-	addr: addr,
159	-	node: node,
160	-	lastKey: []byte{0},
161	-	treeCount: 1,
162	-	}, nil
163	-	}
164	-
165	-	const bytePeekLength = 128
166	-
167	-	type ByteArray struct {
168	-	ImmutableTree
169	-	}
170	-
171	-	func NewByteArray(addr hash.Hash, ns NodeStore) *ByteArray {
172	-	return &ByteArray{ImmutableTree{Addr: addr, ns: ns}}
173	-	}
174	-
175	-	func (b *ByteArray) ToBytes(ctx context.Context) ([]byte, error) {
176	-	return b.bytes(ctx)
177	-	}
178	-
179	-	func (b *ByteArray) ToString(ctx context.Context) (string, error) {
180	-	buf, err := b.bytes(ctx)
181	-	if err != nil {
182	-	return "", err
183	-	}
184	-	toShow := bytePeekLength
185	-	if len(buf) < toShow {
186	-	toShow = len(buf)
187	-	}
188	-	return string(buf[:toShow]), nil
189	-	}
190	-
191	-	type JSONDoc struct {
192	-	ImmutableTree
193	-	}
194	-
195	-	func NewJSONDoc(addr hash.Hash, ns NodeStore) *JSONDoc {
196	-	return &JSONDoc{ImmutableTree{Addr: addr, ns: ns}}
197	-	}
198	-
199	-	func (b *JSONDoc) ToJSONDocument(ctx context.Context) (sql.JSONDocument, error) {
200	-	buf, err := b.bytes(ctx)
201	-	if err != nil {
202	-	return sql.JSONDocument{}, err
203	-	}
204	-	var doc sql.JSONDocument
205	-	err = json.Unmarshal(buf, &doc.Val)
206	-	if err != nil {
207	-	return sql.JSONDocument{}, err
208	-	}
209	-	return doc, err
210	-	}
211	-
212	-	func (b *JSONDoc) ToString(ctx context.Context) (string, error) {
213	-	buf, err := b.bytes(ctx)
214	-	if err != nil {
215	-	return "", err
216	-	}
217	-	toShow := bytePeekLength
218	-	if len(buf) < toShow {
219	-	toShow = len(buf)
220	-	}
221	-	return string(buf[:toShow]), nil
222	-	}
223	-
224	-	type TextStorage struct {
225	-	ImmutableTree
226	-	}
227	-
228	-	func NewTextStorage(addr hash.Hash, ns NodeStore) *TextStorage {
229	-	return &TextStorage{ImmutableTree{Addr: addr, ns: ns}}
230	-	}
231	-
232	-	func (b *TextStorage) ToBytes(ctx context.Context) ([]byte, error) {
233	-	return b.bytes(ctx)
234	-	}
235	-
236	-	func (b *TextStorage) ToString(ctx context.Context) (string, error) {
237	-	buf, err := b.bytes(ctx)
238	-	if err != nil {
239	-	return "", err
240	-	}
241	-	return string(buf), nil
242	-	}
243	-
244	-	type ImmutableTree struct {
245	-	Addr hash.Hash
246	-	buf []byte
247	-	ns NodeStore
248	-	}
249	-
250	-	func NewImmutableTreeFromReader(ctx context.Context, r io.Reader, ns NodeStore, chunkSize int) (*ImmutableTree, error) {
251	-	s := message.NewBlobSerializer(ns.Pool())
252	-	root, err := buildImmutableTree(ctx, r, ns, s, chunkSize)
253	-	if errors.Is(err, io.EOF) {
254	-	return &ImmutableTree{Addr: hash.Hash{}}, nil
255	-	} else if err != nil {
256	-	return nil, err
257	-	}
258	-	return &ImmutableTree{Addr: root.HashOf()}, nil
259	-	}
260	-
261	-	func (t *ImmutableTree) load(ctx context.Context) error {
262	-	if t.Addr.IsEmpty() {
263	-	t.buf = []byte{}
264	-	return nil
265	-	}
266	-	n, err := t.ns.Read(ctx, t.Addr)
267	-	if err != nil {
268	-	return err
269	-	}
270	-
271	-	return WalkNodes(ctx, n, t.ns, func(ctx context.Context, n Node) error {
272	-	if n.IsLeaf() {
273	-	t.buf = append(t.buf, n.GetValue(0)...)
274	-	}
275	-	return nil
276	-	})
277	-	}
278	-
279	-	func (t *ImmutableTree) bytes(ctx context.Context) ([]byte, error) {
280	-	if t.buf == nil {
281	-	err := t.load(ctx)
282	-	if err != nil {
283	-	return nil, err
284	-	}
285	-	}
286	-	return t.buf[:], nil
287	-	}
288	-
289	-	func (t *ImmutableTree) next() (Node, error) {
290	-	panic("not implemented")
291	-	}
292	-
293	-	func (t *ImmutableTree) close() error {
294	-	panic("not implemented")
295	-	}
296	-
297	-	func (t *ImmutableTree) Read(buf bytes.Buffer) (int, error) {
298	-	panic("not implemented")
299	-	}
300	-

■ ■ ■ ■ ■ ■

go/store/prolly/tree/node.go

		skipped 112 lines
113	113		})
114	114		}
115	115
	116	+	type nodeArena []Node
	117	+
	118	+	const nodeArenaSize = 10000
	119	+
	120	+	func (a *nodeArena) Get() Node {
	121	+	if len(*a) == 0 {
	122	+	*a = make([]Node, nodeArenaSize)
	123	+	}
	124	+	n := (a)[len(a)-1]
	125	+	a = (a)[:len(*a)-1]
	126	+	return n
	127	+	}
	128	+
	129	+	func (a *nodeArena) NodeFromBytes(msg []byte) (Node, error) {
	130	+	keys, values, level, count, err := message.UnpackFields(msg)
	131	+	if err != nil {
	132	+	return Node{}, err
	133	+	}
	134	+	n := a.Get()
	135	+	n.keys = keys
	136	+	n.values = values
	137	+	n.count = count
	138	+	n.level = level
	139	+	n.msg = msg
	140	+	return n, nil
	141	+	}
	142	+
116	143		func NodeFromBytes(msg []byte) (Node, error) {
117	144		keys, values, level, count, err := message.UnpackFields(msg)
118	145		return Node{
		skipped 147 lines

■ ■ ■ ■ ■ ■

go/store/prolly/tree/node_store.go

		skipped 43 lines
44	44
45	45		// Format returns the types.NomsBinFormat of this NodeStore.
46	46		Format() *types.NomsBinFormat
	47	+
	48	+	BlobBuilder() *BlobBuilder
47	49		}
48	50
49	51		type nodeStore struct {
50	52		store chunks.ChunkStore
51	53		cache nodeCache
52	54		bp pool.BuffPool
	55	+	bbp *sync.Pool
53	56		}
54	57
55	58		var _ NodeStore = nodeStore{}
		skipped 2 lines
58	61
59	62		var sharedPool = pool.NewBuffPool()
60	63
	64	+	var blobBuilderPool = sync.Pool{
	65	+	New: func() any {
	66	+	return mustNewBlobBuilder(DefaultFixedChunkLength)
	67	+	},
	68	+	}
	69	+
61	70		// NewNodeStore makes a new NodeStore.
62	71		func NewNodeStore(cs chunks.ChunkStore) NodeStore {
63	72		return nodeStore{
64	73		store: cs,
65	74		cache: sharedCache,
66	75		bp: sharedPool,
	76	+	bbp: &blobBuilderPool,
67	77		}
68	78		}
69	79
		skipped 77 lines
147	157		// Pool implements NodeStore.
148	158		func (ns nodeStore) Pool() pool.BuffPool {
149	159		return ns.bp
	160	+	}
	161	+
	162	+	// BlobBuilder implements NodeStore.
	163	+	func (ns nodeStore) BlobBuilder() *BlobBuilder {
	164	+	bb := ns.bbp.Get().(*BlobBuilder)
	165	+	if bb.ns == nil {
	166	+	bb.SetNodeStore(ns)
	167	+	}
	168	+	return bb
150	169		}
151	170
152	171		func (ns nodeStore) Format() *types.NomsBinFormat {
		skipped 7 lines

■ ■ ■ ■ ■ ■

go/store/prolly/tree/testutils.go

		skipped 20 lines
21	21		"math"
22	22		"math/rand"
23	23		"sort"
	24	+	"sync"
24	25
25	26		"github.com/dolthub/dolt/go/store/chunks"
26	27		"github.com/dolthub/dolt/go/store/hash"
		skipped 213 lines
240	241		testRand.Read(buf)
241	242		tb.PutCommitAddr(idx, hash.New(buf))
242	243		case val.BytesAddrEnc, val.StringAddrEnc, val.JSONAddrEnc:
243		-	buf := make([]byte, (testRand.Int63()%40)+10)
	244	+	len := (testRand.Int63() % 40) + 10
	245	+	buf := make([]byte, len)
244	246		testRand.Read(buf)
245		-	tree, err := NewImmutableTreeFromReader(context.Background(), bytes.NewReader(buf), ns, DefaultFixedChunkLength)
	247	+	bb := ns.BlobBuilder()
	248	+	bb.Init(int(len))
	249	+	_, addr, err := bb.Chunk(context.Background(), bytes.NewReader(buf))
246	250		if err != nil {
247	251		panic("failed to write bytes tree")
248	252		}
249		-	tb.PutBytesAddr(idx, tree.Addr)
	253	+	tb.PutBytesAddr(idx, addr)
250	254		default:
251	255		panic("unknown encoding")
252	256		}
		skipped 2 lines
255	259		func NewTestNodeStore() NodeStore {
256	260		ts := &chunks.TestStorage{}
257	261		ns := NewNodeStore(ts.NewViewWithFormat(types.Format_DOLT.VersionString()))
258		-	return nodeStoreValidator{ns: ns}
	262	+	bb := &blobBuilderPool
	263	+	return nodeStoreValidator{ns: ns, bb: bb}
259	264		}
260	265
261	266		type nodeStoreValidator struct {
262	267		ns NodeStore
	268	+	bb *sync.Pool
263	269		}
264	270
265	271		func (v nodeStoreValidator) Read(ctx context.Context, ref hash.Hash) (Node, error) {
		skipped 41 lines
307	313
308	314		func (v nodeStoreValidator) Pool() pool.BuffPool {
309	315		return v.ns.Pool()
	316	+	}
	317	+
	318	+	func (v nodeStoreValidator) BlobBuilder() *BlobBuilder {
	319	+	bb := v.bb.Get().(*BlobBuilder)
	320	+	if bb.ns == nil {
	321	+	bb.SetNodeStore(v)
	322	+	}
	323	+	return bb
310	324		}
311	325
312	326		func (v nodeStoreValidator) Format() *types.NomsBinFormat {
		skipped 3 lines

■ ■ ■ ■ ■ ■

integration-tests/MySQLDockerfile

		skipped 24 lines
25	25		pkg-config \
26	26		mysql-client \
27	27		libmysqlclient-dev \
28		-	openjdk-8-jdk \
	28	+	openjdk-17-jdk \
29	29		ant \
30	30		ca-certificates-java \
31	31		bats \
		skipped 54 lines
86	86		RUN pip install mysql-connector-python PyMySQL sqlalchemy
87	87
88	88		# Setup JAVA_HOME -- useful for docker commandline
89		-	ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64/
	89	+	ENV JAVA_HOME /usr/lib/jvm/java-17-openjdk-amd64/
90	90
91	91		# install mysql connector java
92	92		RUN mkdir -p /mysql-client-tests/java
		skipped 44 lines

■ ■ ■ ■ ■ ■

integration-tests/ORMDockerfile

		skipped 15 lines
16	16		git \
17	17		mysql-client \
18	18		libmysqlclient-dev \
	19	+	# weird issue: installing openjdk-17-jdk errors if `maven` or possibly any other package is not installed after it
	20	+	openjdk-17-jdk \
	21	+	# currently, `apt install maven` installs v3.6.0 which does not work with openjdk-17-jdk
	22	+	maven \
19	23		bats && \
20	24		update-ca-certificates -f
21	25
		skipped 12 lines
34	38
35	39		# install mysql connector and pymsql
36	40		RUN pip3 install mysql-connector-python PyMySQL sqlalchemy
	41	+
	42	+	# Setup JAVA_HOME -- useful for docker commandline
	43	+	ENV JAVA_HOME /usr/lib/jvm/java-17-openjdk-amd64/
	44	+
	45	+	# install the current latest maven version, `v3.6.3`, because apt installed one does not work with jdk 17
	46	+	ADD https://apache.osuosl.org/maven/maven-3/3.8.6/binaries/apache-maven-3.8.6-bin.tar.gz apache-maven-3.8.6-bin.tar.gz
	47	+	RUN tar zxvf apache-maven-3.8.6-bin.tar.gz && \
	48	+	cp -r apache-maven-3.8.6 /opt && \
	49	+	rm -rf apache-maven-3.8.6 apache-maven-3.8.6-bin.tar.gz
	50	+
	51	+	# add maven binary
	52	+	ENV PATH /opt/apache-maven-3.8.6/bin:$PATH
37	53
38	54		# install dolt from source
39	55		WORKDIR /root/building
		skipped 10 lines

■ ■ ■ ■ ■ ■

integration-tests/bats/docs.bats

		skipped 110 lines
111	111		dolt sql <<SQL
112	112		CREATE TABLE dolt_docs (
113	113		doc_name varchar(16383) NOT NULL,
114		-	doc_text varchar(16383),
	114	+	doc_text longtext,
115	115		PRIMARY KEY (doc_name)
116	116		);
117	117		SQL
		skipped 38 lines

■ ■ ■ ■ ■ ■

integration-tests/orm-tests/hibernate/DoltHibernateSmokeTest/pom.xml

1	+	<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
2	+	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
3	+	<modelVersion>4.0.0</modelVersion>
4	+	<groupId>com.dolt.hibernate</groupId>
5	+	<artifactId>DoltHibernateSmokeTest</artifactId>
6	+	<packaging>jar</packaging>
7	+	<version>1.0.0</version>
8	+	<name>DoltHibernateSmokeTest</name>
9	+	<url>http://maven.apache.org</url>
10	+
11	+	<properties>
12	+	<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
13	+	</properties>
14	+
15	+	<dependencies>
16	+	<!-- Hibernate -->
17	+	<!-- [5.6.14,) fails as it gets the latest version but including beta builds that does not work -->
18	+	<dependency>
19	+	<groupId>org.hibernate</groupId>
20	+	<artifactId>hibernate-core</artifactId>
21	+	<version>5.6.14.Final</version>
22	+	</dependency>
23	+	<dependency>
24	+	<groupId>org.hibernate</groupId>
25	+	<artifactId>hibernate-entitymanager</artifactId>
26	+	<version>5.6.14.Final</version>
27	+	</dependency>
28	+
29	+	<!-- MySQL -->
30	+	<dependency>
31	+	<groupId>com.mysql</groupId>
32	+	<artifactId>mysql-connector-j</artifactId>
33	+	<version>8.0.31</version>
34	+	</dependency>
35	+	</dependencies>
36	+
37	+	<build>
38	+	<pluginManagement>
39	+	<plugins>
40	+	<plugin>
41	+	<groupId>org.apache.maven.plugins</groupId>
42	+	<artifactId>maven-compiler-plugin</artifactId>
43	+	<version>3.10.1</version>
44	+	<configuration>
45	+	<source>1.8</source>
46	+	<target>1.8</target>
47	+	</configuration>
48	+	</plugin>
49	+	<plugin>
50	+	<groupId>org.codehaus.mojo</groupId>
51	+	<artifactId>exec-maven-plugin</artifactId>
52	+	<version>3.1.0</version>
53	+	<configuration>
54	+	<mainClass>com.dolt.hibernate.Test</mainClass>
55	+	</configuration>
56	+	</plugin>
57	+	</plugins>
58	+	</pluginManagement>
59	+	</build>
60	+	</project>
61	+

■ ■ ■ ■ ■ ■

integration-tests/orm-tests/hibernate/DoltHibernateSmokeTest/src/main/java/com/dolt/hibernate/Test.java

1	+	package com.dolt.hibernate;
2	+
3	+	import java.util.List;
4	+	import org.hibernate.Session;
5	+	import com.dolt.hibernate.model.Student;
6	+
7	+	/**
8	+	* Class used to perform CRUD operation on database with Hibernate API's
9	+	*
10	+	*/
11	+	public class Test {
12	+
13	+	@SuppressWarnings("unused")
14	+	public static void main(String[] args) {
15	+
16	+	Test application = new Test();
17	+
18	+	/*
19	+	* Save few objects with hibernate
20	+	*/
21	+	int studentId1 = application.saveStudent("Sam", "Disilva", "Maths");
22	+	int studentId2 = application.saveStudent("Joshua", "Brill", "Science");
23	+	int studentId3 = application.saveStudent("Peter", "Pan", "Physics");
24	+	int studentId4 = application.saveStudent("Bill", "Laurent", "Maths");
25	+
26	+	/*
27	+	* Retrieve all saved objects
28	+	*/
29	+	List<Student> students = application.getAllStudents();
30	+	System.out.println("List of all persisted students >>>");
31	+	for (Student student : students) {
32	+	System.out.println("Persisted Student :" + student);
33	+	}
34	+
35	+	/*
36	+	* Update an object
37	+	*/
38	+	application.updateStudent(studentId4, "ARTS");
39	+
40	+	/*
41	+	* Deletes an object
42	+	*/
43	+	application.deleteStudent(studentId2);
44	+
45	+	/*
46	+	* Retrieve all saved objects
47	+	*/
48	+	List<Student> remaingStudents = application.getAllStudents();
49	+	System.out.println("List of all remained persisted students >>>");
50	+	for (Student student : remaingStudents) {
51	+	System.out.println("Persisted Student :" + student);
52	+	}
53	+
54	+	}
55	+
56	+	/**
57	+	* This method saves a Student object in database
58	+	*/
59	+	public int saveStudent(String firstName, String lastName, String section) {
60	+	Student student = new Student();
61	+	student.setFirstName(firstName);
62	+	student.setLastName(lastName);
63	+	student.setSection(section);
64	+
65	+	Session session = Util.getSessionFactory().openSession();
66	+	session.beginTransaction();
67	+
68	+	int id = (Integer) session.save(student);
69	+	session.getTransaction().commit();
70	+	session.close();
71	+	return id;
72	+	}
73	+
74	+	/**
75	+	* This method returns list of all persisted Student objects/tuples from
76	+	* database
77	+	*/
78	+	public List<Student> getAllStudents() {
79	+	Session session = Util.getSessionFactory().openSession();
80	+	session.beginTransaction();
81	+
82	+	@SuppressWarnings("unchecked")
83	+	List<Student> employees = (List<Student>) session.createQuery(
84	+	"FROM Student s ORDER BY s.firstName ASC").list();
85	+
86	+	session.getTransaction().commit();
87	+	session.close();
88	+	return employees;
89	+	}
90	+
91	+	/**
92	+	* This method updates a specific Student object
93	+	*/
94	+	public void updateStudent(int id, String section) {
95	+	Session session = Util.getSessionFactory().openSession();
96	+	session.beginTransaction();
97	+
98	+	Student student = (Student) session.get(Student.class, id);
99	+	student.setSection(section);
100	+	//session.update(student);//No need to update manually as it will be updated automatically on transaction close.
101	+	session.getTransaction().commit();
102	+	session.close();
103	+	}
104	+
105	+	/**
106	+	* This method deletes a specific Student object
107	+	*/
108	+	public void deleteStudent(int id) {
109	+	Session session = Util.getSessionFactory().openSession();
110	+	session.beginTransaction();
111	+
112	+	Student student = (Student) session.get(Student.class, id);
113	+	session.delete(student);
114	+	session.getTransaction().commit();
115	+	session.close();
116	+	}
117	+	}
118	+

■ ■ ■ ■ ■ ■

integration-tests/orm-tests/hibernate/DoltHibernateSmokeTest/src/main/java/com/dolt/hibernate/Util.java

1	+	package com.dolt.hibernate;
2	+
3	+	import org.hibernate.SessionFactory;
4	+	import org.hibernate.cfg.Configuration;
5	+
6	+	@SuppressWarnings("deprecation")
7	+	public class Util {
8	+
9	+	private static final SessionFactory sessionFactory;
10	+
11	+	static{
12	+	try{
13	+	sessionFactory = new Configuration().configure().buildSessionFactory();
14	+
15	+	}catch (Throwable ex) {
16	+	System.err.println("Session Factory could not be created." + ex);
17	+	throw new ExceptionInInitializerError(ex);
18	+	}
19	+	}
20	+
21	+	public static SessionFactory getSessionFactory() {
22	+	return sessionFactory;
23	+	}
24	+
25	+	}
26	+

■ ■ ■ ■ ■ ■

integration-tests/orm-tests/hibernate/DoltHibernateSmokeTest/src/main/java/com/dolt/hibernate/model/Student.java

1	+	package com.dolt.hibernate.model;
2	+
3	+	import java.io.Serializable;
4	+
5	+	import javax.persistence.Column;
6	+	import javax.persistence.Entity;
7	+	import javax.persistence.GeneratedValue;
8	+	import javax.persistence.GenerationType;
9	+	import javax.persistence.Id;
10	+	import javax.persistence.Table;
11	+
12	+	@Entity
13	+	@Table(name = "STUDENT")
14	+	public class Student implements Serializable {
15	+
16	+	@Id
17	+	@GeneratedValue(strategy = GenerationType.IDENTITY)
18	+	private int id;
19	+
20	+	@Column(name = "FIRST_NAME", nullable = false)
21	+	private String firstName;
22	+
23	+	@Column(name = "LAST_NAME", nullable = false)
24	+	private String lastName;
25	+
26	+	@Column(name = "SECTION", nullable = false)
27	+	private String section;
28	+
29	+	public int getId() {
30	+	return id;
31	+	}
32	+
33	+	public void setId(int id) {
34	+	this.id = id;
35	+	}
36	+
37	+	public String getFirstName() {
38	+	return firstName;
39	+	}
40	+
41	+	public void setFirstName(String firstName) {
42	+	this.firstName = firstName;
43	+	}
44	+
45	+	public String getLastName() {
46	+	return lastName;
47	+	}
48	+
49	+	public void setLastName(String lastName) {
50	+	this.lastName = lastName;
51	+	}
52	+
53	+	public String getSection() {
54	+	return section;
55	+	}
56	+
57	+	public void setSection(String section) {
58	+	this.section = section;
59	+	}
60	+
61	+	@Override
62	+	public int hashCode() {
63	+	final int prime = 31;
64	+	int result = 1;
65	+	result = prime * result + id;
66	+	return result;
67	+	}
68	+
69	+	@Override
70	+	public boolean equals(Object obj) {
71	+	if (this == obj)
72	+	return true;
73	+	if (obj == null)
74	+	return false;
75	+	if (!(obj instanceof Student))
76	+	return false;
77	+	Student other = (Student) obj;
78	+	if (id != other.id)
79	+	return false;
80	+	return true;
81	+	}
82	+
83	+	@Override
84	+	public String toString() {
85	+	return "Student [id=" + id + ", firstName=" + firstName + ", lastName="
86	+	+ lastName + ", section=" + section + "]";
87	+	}
88	+
89	+	}
90	+

■ ■ ■ ■ ■ ■

integration-tests/orm-tests/hibernate/DoltHibernateSmokeTest/src/main/resources/hibernate.cfg.xml

1	+	<?xml version="1.0" encoding="utf-8"?>
2	+	<!DOCTYPE hibernate-configuration SYSTEM "http://www.hibernate.org/dtd/hibernate-configuration-3.0.dtd">
3	+
4	+
5	+	<hibernate-configuration>
6	+	<session-factory>
7	+	<property name="hibernate.dialect">org.hibernate.dialect.MySQLDialect</property>
8	+	<property name="hibernate.connection.driver_class">com.mysql.jdbc.Driver</property>
9	+	<property name="hibernate.connection.username">dolt</property>
10	+	<property name="hibernate.connection.password"></property>
11	+	<property name="hibernate.connection.url">jdbc:mysql://localhost:3306/dolt</property>
12	+	<property name="show_sql">true</property>
13	+	<property name="format_sql">false</property>
14	+	<mapping class="com.dolt.hibernate.model.Student"/>
15	+	</session-factory>
16	+	</hibernate-configuration>
17	+

■ ■ ■ ■ ■ ■

integration-tests/orm-tests/hibernate/README.md

1	+	# Hibernate-ORM Smoke Test
2	+
3	+	The smoke test is run by Maven using MySQL JDBC driver. To install Maven, go to `https://maven.apache.org/install.html`.
4	+
5	+	Database settings are inside `hibernate.cfg.xml` file and is configured to hit a Dolt sql-server
6	+	on the default port, for the user "dolt", with no password, for the database named "dolt".
7	+
8	+	`Test.java` file is the main entry point and will insert a new record into the database, then print the data
9	+	before changes, and update and delete rows, and print the data again after changes. Exit with a zero exit code.
10	+	If any errors are encountered, they are logged, and the process exits with a non-zero exit code.
11	+
12	+	To run this smoke test project run these commands:
13	+	1. `cd DoltHibernateSmokeTest`
14	+	2. `mvn clean install`
15	+	3. `mvn clean package`
16	+	4. `mvn exec:java`
17	+

■ ■ ■ ■ ■ ■

integration-tests/orm-tests/orm-tests.bats

		skipped 86 lines
87	87		npm start
88	88		}
89	89
	90	+	# Hibernate is an ORM for Java applications using JDBC driver. This is a simple smoke test to make sure
	91	+	# Dolt can support the most basic Hibernate operations.
	92	+	@test "Hibernate smoke test" {
	93	+	# need to create tables for it before running the test
	94	+	mysql --protocol TCP -u dolt -e "create database dolt; use dolt; create table STUDENT (id INT NOT NULL auto_increment PRIMARY KEY, first_name VARCHAR(30) NOT NULL, last_name VARCHAR(30) NOT NULL, section VARCHAR(30) NOT NULL);"
	95	+
	96	+	cd hibernate/DoltHibernateSmokeTest
	97	+	mvn clean install
	98	+	mvn clean package
	99	+	mvn exec:java
	100	+	}
	101	+
90	102		# Turn this test on to prevent the container from exiting if you need to exec a shell into
91	103		# the container to debug failed tests.
92	104		#@test "Pause container for an hour to debug failures" {
		skipped 2 lines

Merge remote-tracking branch 'origin/main' into taylor/shallow-gc