STRLCPY/syft

[wip] adding file index to directory resolver

Signed-off-by: Alex Goodman <[email protected]>

Alex Goodman committed 1 year ago

5b9ba7d0

1 parent 39290211

Revision indexing in progress... (symbol navigation in revisions will be accurate after indexed)

Total 11 files Show one by one

■ ■ ■ ■ ■ ■

go.mod

		skipped 157 lines
158	158		v0.53.1 // Published accidentally with incorrect license in depdencies
159	159		)
160	160
	161	+	replace github.com/anchore/stereoscope => ../stereoscope
	162	+

■ ■ ■ ■ ■ ■

syft/file/all_regular_files.go

1	1		package file
2	2
3	3		import (
	4	+	"github.com/anchore/stereoscope/pkg/file"
4	5		"github.com/anchore/syft/internal/log"
5	6		"github.com/anchore/syft/syft/source"
6	7		)
		skipped 13 lines
20	21		continue
21	22		}
22	23
23		-	if metadata.Type != source.RegularFile {
	24	+	if metadata.Type != file.TypeReg {
24	25		continue
25	26		}
26	27		locations = append(locations, resolvedLocation)
		skipped 5 lines

■ ■ ■ ■ ■ ■

syft/file/digest_cataloger.go

		skipped 3 lines
4	4		"crypto"
5	5		"errors"
6	6		"fmt"
	7	+	"github.com/anchore/stereoscope/pkg/file"
7	8		"hash"
8	9		"io"
9	10		"strings"
		skipped 55 lines
65	66		}
66	67
67	68		// we should only attempt to report digests for files that are regular files (don't attempt to resolve links)
68		-	if meta.Type != source.RegularFile {
	69	+	if meta.Type != file.TypeReg {
69	70		return nil, errUndigestableFile
70	71		}
71	72
		skipped 71 lines

■ ■ ■ ■ ■ ■

syft/formats/formats.go

		skipped 37 lines
38	38		for _, f := range Formats() {
39	39		if err := f.Validate(bytes.NewReader(by)); err != nil {
40	40		if !errors.Is(err, sbom.ErrValidationNotSupported) {
41		-	log.Debugf("format %s returned err: %+v", f.ID(), err)
	41	+	log.WithFields("error", err).Tracef("format validation for %s failed", f.ID())
42	42		}
43	43		continue
44	44		}
		skipped 86 lines

■ ■ ■ ■ ■ ■

syft/source/directory_resolver.go

		skipped 2 lines
3	3		import (
4	4		"errors"
5	5		"fmt"
	6	+	"github.com/anchore/stereoscope/pkg/query"
6	7		"io"
7	8		"io/fs"
8	9		"os"
		skipped 31 lines
40	41		currentWdRelativeToRoot string
41	42		currentWd string
42	43		fileTree *filetree.FileTree
43		-	metadata map[file.ID]FileMetadata
	44	+	fileIndex *file.Index
44	45		// TODO: wire up to report these paths in the json report
45	46		pathFilterFns []pathFilterFn
46	47		refsByMIMEType map[string][]file.Reference
		skipped 32 lines
79	80		currentWd: cleanCWD,
80	81		currentWdRelativeToRoot: currentWdRelRoot,
81	82		fileTree: filetree.NewFileTree(),
82		-	metadata: make(map[file.ID]FileMetadata),
	83	+	fileIndex: file.NewIndex(),
83	84		pathFilterFns: append([]pathFilterFn{isUnallowableFileType, isUnixSystemRuntimePath}, pathFilters...),
84	85		refsByMIMEType: make(map[string][]file.Reference),
85	86		errPaths: make(map[string]error),
86	87		}
87	88
	89	+	log.WithFields("path", cleanRoot).Debug("indexing full filetree")
88	90		return &resolver, indexAllRoots(cleanRoot, resolver.indexTree)
89	91		}
90	92
91	93		func (r directoryResolver) indexTree(root string, stager progress.Stage) ([]string, error) {
92		-	log.Debugf("indexing filesystem path=%q", root)
	94	+	log.WithFields("path", root).Trace("indexing filetree")
93	95
94	96		var roots []string
95	97		var err error
		skipped 110 lines
206	208
207	209		// cases like "/" will be in the tree, but not been indexed yet (a special case). We want to capture
208	210		// these cases as new paths to index.
209		-	_, exists = r.metadata[ref.ID()]
210		-	return exists
	211	+	return r.fileIndex.Exists(*ref)
211	212		}
212	213
213	214		func (r directoryResolver) addDirectoryToIndex(p string, info os.FileInfo) error {
		skipped 4 lines
218	219
219	220		location := NewLocationFromDirectory(p, *ref)
220	221		metadata := fileMetadataFromPath(p, info, r.isInIndex(location))
221		-	r.addFileMetadataToIndex(ref, metadata)
	222	+	r.addFileToFileIndex(ref, metadata)
222	223
223	224		return nil
224	225		}
		skipped 6 lines
231	232
232	233		location := NewLocationFromDirectory(p, *ref)
233	234		metadata := fileMetadataFromPath(p, info, r.isInIndex(location))
234		-	r.addFileMetadataToIndex(ref, metadata)
	235	+	r.addFileToFileIndex(ref, metadata)
235	236
236	237		return nil
237	238		}
		skipped 26 lines
264	265		location.VirtualPath = p
265	266		metadata := fileMetadataFromPath(p, usedInfo, r.isInIndex(location))
266	267		metadata.LinkDestination = linkTarget
267		-	r.addFileMetadataToIndex(ref, metadata)
	268	+	r.addFileToFileIndex(ref, metadata)
268	269
269	270		return targetAbsPath, nil
270	271		}
271	272
272		-	func (r directoryResolver) addFileMetadataToIndex(ref *file.Reference, metadata FileMetadata) {
	273	+	func (r directoryResolver) addFileToFileIndex(ref *file.Reference, metadata file.Metadata) {
273	274		if ref != nil {
274	275		if metadata.MIMEType != "" {
275	276		r.refsByMIMEType[metadata.MIMEType] = append(r.refsByMIMEType[metadata.MIMEType], *ref)
276	277		}
277		-	r.metadata[ref.ID()] = metadata
	278	+	r.fileIndex.Add(*ref, metadata, nil)
278	279		}
279	280		}
280	281
		skipped 54 lines
335	336		continue
336	337		}
337	338
338		-	// we should be resolving symlinks and preserving this information as a VirtualPath to the real file
339		-	evaluatedPath, err := filepath.EvalSymlinks(userStrPath)
	339	+	tree := r.fileTree
	340	+	_, ref, err := tree.File(file.Path(userStrPath), filetree.FollowBasenameLinks)
340	341		if err != nil {
341		-	log.Tracef("unable to evaluate symlink for path=%q : %+v", userPath, err)
342		-	continue
	342	+	return nil, err
343	343		}
344		-
345		-	// TODO: why not use stored metadata?
346		-	fileMeta, err := os.Stat(evaluatedPath)
347		-	if errors.Is(err, os.ErrNotExist) {
348		-	// note: there are other kinds of errors other than os.ErrNotExist that may be given that is platform
349		-	// specific, but essentially hints at the same overall problem (that the path does not exist). Such an
350		-	// error could be syscall.ENOTDIR (see https://github.com/golang/go/issues/18974).
351		-	continue
352		-	} else if err != nil {
353		-	// we don't want to consider any other syscalls that may hint at non-existence of the file/dir as
354		-	// invalid paths. This logging statement is meant to raise IO or permissions related problems.
355		-	var pathErr *os.PathError
356		-	if !errors.As(err, &pathErr) {
357		-	log.Warnf("path is not valid (%s): %+v", evaluatedPath, err)
358		-	}
	344	+	if ref == nil {
	345	+	// no file found, keep looking through layers
359	346		continue
360	347		}
361	348
362		-	// don't consider directories
363		-	if fileMeta.IsDir() {
	349	+	// don't consider directories (special case: there is no path information for /)
	350	+	if ref.RealPath == "/" {
364	351		continue
	352	+	} else if r.fileIndex.Exists(*ref) {
	353	+	metadata, err := r.fileIndex.Get(*ref)
	354	+	if err != nil {
	355	+	return nil, fmt.Errorf("unable to get file metadata for path=%q: %w", ref.RealPath, err)
	356	+	}
	357	+	if metadata.Metadata.IsDir {
	358	+	continue
	359	+	}
365	360		}
366	361
367		-	if runtime.GOOS == WindowsOS {
368		-	userStrPath = windowsToPosix(userStrPath)
369		-	}
370		-
371		-	exists, ref, err := r.fileTree.File(file.Path(userStrPath), filetree.FollowBasenameLinks)
372		-	if err == nil && exists {
373		-	loc := NewVirtualLocationFromDirectory(
374		-	r.responsePath(string(ref.RealPath)), // the actual path relative to the resolver root
375		-	r.responsePath(userStrPath), // the path used to access this file, relative to the resolver root
376		-	*ref,
377		-	)
378		-	references = append(references, loc)
379		-	}
	362	+	loc := NewVirtualLocationFromDirectory(
	363	+	r.responsePath(string(ref.RealPath)), // the actual path relative to the resolver root
	364	+	r.responsePath(userStrPath), // the path used to access this file, relative to the resolver root
	365	+	*ref,
	366	+	)
	367	+	references = append(references, loc)
380	368		}
381	369
382	370		return references, nil
		skipped 25 lines
408	396		result := make([]Location, 0)
409	397
410	398		for _, extension := range extensions {
411		-	// TODO: is there a faster way to do this?
412		-	globResults, err := r.fileTree.FilesByGlob("*/"+extension, filetree.FollowBasenameLinks)
	399	+
	400	+	refs, err := query.FilesByExtension(r.fileTree, r.fileIndex, extension)
413	401		if err != nil {
414	402		return nil, err
415	403		}
416		-	for _, globResult := range globResults {
417		-	loc := NewVirtualLocationFromDirectory(
418		-	r.responsePath(string(globResult.Reference.RealPath)), // the actual path relative to the resolver root
419		-	r.responsePath(string(globResult.MatchPath)), // the path used to access this file, relative to the resolver root
420		-	globResult.Reference,
	404	+	for _, ref := range refs {
	405	+	loc := NewLocationFromDirectory(
	406	+	r.responsePath(string(ref.RealPath)),
	407	+	ref,
421	408		)
422	409		result = append(result, loc)
423	410		}
		skipped 6 lines
430	417		result := make([]Location, 0)
431	418
432	419		for _, filename := range filenames {
433		-	// TODO: is there a faster way to do this?
434		-	globResults, err := r.fileTree.FilesByGlob("**/"+filename, filetree.FollowBasenameLinks)
	420	+	refs, err := query.FilesByBasename(r.fileTree, r.fileIndex, filename)
435	421		if err != nil {
436	422		return nil, err
437	423		}
438		-	for _, globResult := range globResults {
439		-	loc := NewVirtualLocationFromDirectory(
440		-	r.responsePath(string(globResult.Reference.RealPath)), // the actual path relative to the resolver root
441		-	r.responsePath(string(globResult.MatchPath)), // the path used to access this file, relative to the resolver root
442		-	globResult.Reference,
	424	+	for _, ref := range refs {
	425	+	loc := NewLocationFromDirectory(
	426	+	r.responsePath(string(ref.RealPath)),
	427	+	ref,
443	428		)
444	429		result = append(result, loc)
445	430		}
		skipped 2 lines
448	433		return result, nil
449	434		}
450	435
451		-	// TODO: duplicate code with FilesByBasename
452	436		func (r directoryResolver) FilesByBasenameGlob(globs ...string) ([]Location, error) {
453	437		result := make([]Location, 0)
454	438
455	439		for _, glob := range globs {
456		-	// TODO: is there a faster way to do this?
457		-	globResults, err := r.fileTree.FilesByGlob("**/"+glob, filetree.FollowBasenameLinks)
	440	+	refs, err := query.FilesByBasenameGlob(r.fileTree, r.fileIndex, glob)
458	441		if err != nil {
459	442		return nil, err
460	443		}
461		-	for _, globResult := range globResults {
462		-	loc := NewVirtualLocationFromDirectory(
463		-	r.responsePath(string(globResult.Reference.RealPath)), // the actual path relative to the resolver root
464		-	r.responsePath(string(globResult.MatchPath)), // the path used to access this file, relative to the resolver root
465		-	globResult.Reference,
	444	+	for _, ref := range refs {
	445	+	loc := NewLocationFromDirectory(
	446	+	r.responsePath(string(ref.RealPath)),
	447	+	ref,
466	448		)
467	449		result = append(result, loc)
468	450		}
		skipped 57 lines
526	508		return results
527	509		}
528	510
529		-	func (r *directoryResolver) FileMetadataByLocation(location Location) (FileMetadata, error) {
530		-	metadata, exists := r.metadata[location.ref.ID()]
531		-	if !exists {
532		-	return FileMetadata{}, fmt.Errorf("location: %+v : %w", location, os.ErrNotExist)
	511	+	func (r *directoryResolver) FileMetadataByLocation(location Location) (file.Metadata, error) {
	512	+	indexEntry, err := r.fileIndex.Get(location.ref)
	513	+	if err != nil {
	514	+	return file.Metadata{}, fmt.Errorf("location: %+v : %w", location, err)
533	515		}
534	516
535		-	return metadata, nil
	517	+	return indexEntry.Metadata, nil
536	518		}
537	519
538	520		func (r *directoryResolver) FilesByMIMEType(types ...string) ([]Location, error) {
		skipped 115 lines

■ ■ ■ ■ ■ ■

syft/source/file_metadata.go

		skipped 7 lines
8	8		"github.com/anchore/syft/internal/log"
9	9		)
10	10
11		-	type FileMetadata struct {
12		-	Mode os.FileMode
13		-	Type FileType
14		-	UserID int
15		-	GroupID int
16		-	LinkDestination string
17		-	Size int64
18		-	MIMEType string
19		-	}
	11	+	type FileMetadata = file.Metadata
20	12
21	13		func fileMetadataByLocation(img *image.Image, location Location) (FileMetadata, error) {
22	14		entry, err := img.FileCatalog.Get(location.ref)
		skipped 1 lines
24	16		return FileMetadata{}, err
25	17		}
26	18
27		-	return FileMetadata{
28		-	Mode: entry.Metadata.Mode,
29		-	Type: newFileTypeFromTarHeaderTypeFlag(entry.Metadata.TypeFlag),
30		-	UserID: entry.Metadata.UserID,
31		-	GroupID: entry.Metadata.GroupID,
32		-	LinkDestination: entry.Metadata.Linkname,
33		-	Size: entry.Metadata.Size,
34		-	MIMEType: entry.Metadata.MIMEType,
35		-	}, nil
	19	+	return entry.Metadata, nil
36	20		}
37	21
38		-	func fileMetadataFromPath(path string, info os.FileInfo, withMIMEType bool) FileMetadata {
	22	+	func fileMetadataFromPath(path string, info os.FileInfo, withMIMEType bool) file.Metadata {
39	23		var mimeType string
40	24		uid, gid := GetXid(info)
41	25
		skipped 13 lines
55	39		mimeType = file.MIMEType(f)
56	40		}
57	41
58		-	return FileMetadata{
	42	+	return file.Metadata{
59	43		Mode: info.Mode(),
60		-	Type: newFileTypeFromMode(info.Mode()),
	44	+	Type: file.TypeFromMode(info.Mode()),
61	45		// unsupported across platforms
62	46		UserID: uid,
63	47		GroupID: gid,
64	48		Size: info.Size(),
65	49		MIMEType: mimeType,
	50	+	// note: link destination is intentionally not set here, this requires knowledge of the filetree
66	51		}
67	52		}
68	53

■ ■ ■ ■ ■ ■

syft/source/file_type.go

		skipped 1 lines
2	2
3	3		import (
4	4		"archive/tar"
	5	+	"github.com/anchore/stereoscope/pkg/file"
5	6		"os"
6	7		)
7	8
8	9		const (
9		-	RegularFile FileType = "RegularFile"
	10	+	RegularFile = file.TypeReg
10	11		// IrregularFile is how syft defines files that are neither regular, symbolic or directory.
11	12		// For ref: the seven standard Unix file types are regular, directory, symbolic link,
12	13		// FIFO special, block special, character special, and socket as defined by POSIX.
13		-	IrregularFile FileType = "IrregularFile"
14		-	HardLink FileType = "HardLink"
15		-	SymbolicLink FileType = "SymbolicLink"
16		-	CharacterDevice FileType = "CharacterDevice"
17		-	BlockDevice FileType = "BlockDevice"
18		-	Directory FileType = "Directory"
19		-	FIFONode FileType = "FIFONode"
20		-	Socket FileType = "Socket"
	14	+	IrregularFile = file.TypeIrregular
	15	+	HardLink = file.TypeHardLink
	16	+	SymbolicLink = file.TypeSymlink
	17	+	CharacterDevice = file.TypeCharacterDevice
	18	+	BlockDevice = file.TypeBlockDevice
	19	+	Directory = file.TypeDir
	20	+	FIFONode = file.TypeFifo
	21	+	Socket = file.TypeSocket
21	22		)
22	23
23		-	type FileType string
	24	+	type FileType = file.Type
24	25
25	26		func newFileTypeFromTarHeaderTypeFlag(flag byte) FileType {
26	27		switch flag {
		skipped 45 lines

■ ■ ■ ■ ■ ■

syft/source/image_all_layers_resolver.go

1	1		package source
2	2
3	3		import (
4		-	"archive/tar"
5	4		"fmt"
6	5		"io"
7	6
		skipped 48 lines
56	55		return nil, fmt.Errorf("unable to fetch metadata (ref=%+v): %w", ref, err)
57	56		}
58	57
59		-	if entry.Metadata.TypeFlag == tar.TypeLink \|\| entry.Metadata.TypeFlag == tar.TypeSymlink {
	58	+	if entry.Metadata.Type == file.TypeHardLink \|\| entry.Metadata.Type == file.TypeSymlink {
60	59		// a link may resolve in this layer or higher, assuming a squashed tree is used to search
61	60		// we should search all possible resolutions within the valid source
62	61		for _, subLayerIdx := range r.layers[layerIdx:] {
		skipped 99 lines
162	161		// RelativeFileByPath fetches a single file at the given path relative to the layer squash of the given reference.
163	162		// This is helpful when attempting to find a file that is in the same layer or lower as another file.
164	163		func (r imageAllLayersResolver) RelativeFileByPath(location Location, path string) Location {
165		-	entry, err := r.img.FileCatalog.Get(location.ref)
166		-	if err != nil {
	164	+	layer := r.img.FileCatalog.Layer(location.ref)
	165	+	if layer == nil {
167	166		return nil
168	167		}
169	168
170		-	exists, relativeRef, err := entry.Layer.SquashedTree.File(file.Path(path), filetree.FollowBasenameLinks)
	169	+	exists, relativeRef, err := layer.SquashedTree.File(file.Path(path), filetree.FollowBasenameLinks)
171	170		if err != nil {
172	171		log.Errorf("failed to find path=%q in squash: %+w", path, err)
173	172		return nil
		skipped 15 lines
189	188		return nil, fmt.Errorf("unable to get metadata for path=%q from file catalog: %w", location.RealPath, err)
190	189		}
191	190
192		-	switch entry.Metadata.TypeFlag {
193		-	case tar.TypeSymlink, tar.TypeLink:
	191	+	switch entry.Metadata.Type {
	192	+	case file.TypeSymlink, file.TypeHardLink:
194	193		// the location we are searching may be a symlink, we should always work with the resolved file
195	194		newLocation := r.RelativeFileByPath(location, location.VirtualPath)
196	195		if newLocation == nil {
		skipped 90 lines
287	286		defer close(results)
288	287		for _, layerIdx := range r.layers {
289	288		tree := r.img.Layers[layerIdx].Tree
290		-	for _, ref := range tree.AllFiles(file.AllTypes...) {
	289	+	for _, ref := range tree.AllFiles(file.AllTypes()...) {
291	290		results <- NewLocationFromImage(string(ref.RealPath), ref, r.img)
292	291		}
293	292		}
		skipped 8 lines

■ ■ ■ ■ ■ ■

syft/source/image_squash_resolver.go

1	1		package source
2	2
3	3		import (
4		-	"archive/tar"
5	4		"fmt"
6	5		"io"
7	6
		skipped 135 lines
143	142		return nil, fmt.Errorf("unable to get metadata for path=%q from file catalog: %w", location.RealPath, err)
144	143		}
145	144
146		-	switch entry.Metadata.TypeFlag {
147		-	case tar.TypeSymlink, tar.TypeLink:
	145	+	switch entry.Metadata.Type {
	146	+	case file.TypeSymlink, file.TypeHardLink:
148	147		// the location we are searching may be a symlink, we should always work with the resolved file
149	148		locations, err := r.FilesByPath(location.RealPath)
150	149		if err != nil {
		skipped 17 lines
168	167		results := make(chan Location)
169	168		go func() {
170	169		defer close(results)
171		-	for _, ref := range r.img.SquashedTree().AllFiles(file.AllTypes...) {
	170	+	for _, ref := range r.img.SquashedTree().AllFiles(file.AllTypes()...) {
172	171		results <- NewLocationFromImage(string(ref.RealPath), ref, r.img)
173	172		}
174	173		}()
		skipped 70 lines

■ ■ ■ ■ ■ ■

syft/source/location.go

		skipped 4 lines
5	5
6	6		"github.com/anchore/stereoscope/pkg/file"
7	7		"github.com/anchore/stereoscope/pkg/image"
8		-	"github.com/anchore/syft/internal/log"
9	8		)
10	9
11	10		// Location represents a path relative to a particular filesystem resolved to a specific file.Reference. This struct is used as a key
		skipped 34 lines
46	45
47	46		// NewLocationFromImage creates a new Location representing the given path (extracted from the ref) relative to the given image.
48	47		func NewLocationFromImage(virtualPath string, ref file.Reference, img *image.Image) Location {
49		-	entry, err := img.FileCatalog.Get(ref)
50		-	if err != nil {
51		-	log.Warnf("unable to find file catalog entry for ref=%+v", ref)
52		-	return Location{
53		-	Coordinates: Coordinates{
54		-	RealPath: string(ref.RealPath),
55		-	},
56		-	VirtualPath: virtualPath,
57		-	ref: ref,
58		-	}
	48	+	var digest string
	49	+	layer := img.FileCatalog.Layer(ref)
	50	+	if layer != nil {
	51	+	digest = layer.Metadata.Digest
59	52		}
60	53
61	54		return Location{
62	55		Coordinates: Coordinates{
63	56		RealPath: string(ref.RealPath),
64		-	FileSystemID: entry.Layer.Metadata.Digest,
	57	+	FileSystemID: digest,
65	58		},
66	59		VirtualPath: virtualPath,
67	60		ref: ref,
		skipped 58 lines

■ ■ ■ ■ ■ ■

syft/source/mock_resolver.go

		skipped 1 lines
2	2
3	3		import (
4	4		"fmt"
	5	+	"github.com/anchore/stereoscope/pkg/file"
5	6		"io"
6	7		"os"
7	8		"path"
		skipped 151 lines
159	160		}
160	161
161	162		// other types not supported
162		-	ty := RegularFile
	163	+	ty := file.TypeReg
163	164		if info.IsDir() {
164		-	ty = Directory
	165	+	ty = file.TypeDir
165	166		}
166	167
167	168		return FileMetadata{
		skipped 37 lines