STRLCPY/bearer

feat: redo risks
vjerci committed 1 year ago

da023484

1 parent d63df395

Revision indexing in progress... (symbol navigation in revisions will be accurate after indexed)

■ ■ ■ ■ ■ ■

pkg/report/output/dataflow/risks/risks.go

		skipped 2 lines
3	3		import (
4	4		"bytes"
5	5		"encoding/json"
6		-	"sort"
7	6
8	7		"github.com/bearer/bearer/pkg/classification/db"
9	8		"github.com/bearer/bearer/pkg/commands/process/settings"
		skipped 8 lines
18	17		)
19	18
20	19		type Holder struct {
21		-	detectors map[string]detectorHolder // group datatypeHolders by name
22		-	config settings.Config
23		-	isInternal bool
	20	+	detectors map[string]detectorHolder // group detections by detector name
	21	+	config settings.Config
24	22		}
25	23
26	24		type detectorHolder struct {
27		-	id string
28		-	datatypes map[string]*datatypeHolder // group detectors by detectorName
	25	+	id string
	26	+	files map[string]fileHolder // group detectors by file name
29	27		}
30	28
31		-	type datatypeHolder struct {
32		-	name string
33		-	uuid string
34		-	categoryUUID string
35		-	files map[string]map[int]*fileHolder // group files by filename
	29	+	type fileHolder struct {
	30	+	name string
	31	+	lineNumber map[int]lineHolder // group detections by line number
36	32		}
37	33
38		-	type fileHolder struct {
39		-	name string
40		-	lineNumber int
	34	+	type lineHolder struct {
	35	+	lineNumber int
	36	+	dataTypeCategory map[string]dataTypeCategoryHolder // group detections by datatype category
	37	+	}
	38	+
	39	+	type dataTypeCategoryHolder struct {
	40	+	name string
	41	+	isPresence bool
	42	+	dataType map[string]dataTypeHolder
	43	+	}
	44	+
	45	+	type dataTypeHolder struct {
41	46		parent *schema.Parent
42		-	fieldName string
43		-	objectName string
	47	+	content *string
	48	+	fieldName *string
	49	+	objectName *string
44	50		subjectName *string
45	51		}
46	52
47	53		func New(config settings.Config, isInternal bool) *Holder {
48	54		return &Holder{
49		-	detectors: make(map[string]detectorHolder),
50		-	config: config,
51		-	isInternal: isInternal,
	55	+	detectors: make(map[string]detectorHolder),
	56	+	config: config,
52	57		}
53	58		}
54	59
55	60		func (holder *Holder) AddRiskPresence(detection detections.Detection) {
56	61		// create entry if it doesn't exist
57	62		ruleName := string(detection.DetectorType)
58		-	if _, exists := holder.detectors[ruleName]; !exists {
59		-	holder.detectors[ruleName] = detectorHolder{
60		-	id: ruleName,
61		-	datatypes: make(map[string]*datatypeHolder),
62		-	}
63		-	}
64		-
65		-
66		-	detector := holder.detectors[ruleName]
67		-
68		-	if _, exists := detector[]
69		-
70		-	riskLocation := &types.RiskLocation{
71		-	Filename: detection.Source.Filename,
72		-	LineNumber: *detection.Source.LineNumber,
73		-	}
74		-
75		-	// add parent information if possible
	63	+	fileName := detection.Source.Filename
	64	+	lineNumber := *detection.Source.LineNumber
	65	+	// can be nil
76	66		parent := extractCustomRiskParent(detection.Value)
77		-	if parent != nil {
78		-	riskLocation.Parent = parent
79		-	}
	67	+	var content string
80	68
81	69		if detection.DetectorType == detectors.DetectorGitleaks {
82	70		value := detection.Value.(map[string]interface{})["description"]
83		-
84		-	holder.presentRisks[ruleName].Locations = append(holder.presentRisks[ruleName].Locations, types.RiskDetectionLocation{
85		-	RiskLocation: riskLocation,
86		-	Content: value.(string),
87		-	})
	71	+	content = value.(string)
88	72		} else {
89		-	holder.presentRisks[ruleName].Locations = append(holder.presentRisks[ruleName].Locations, types.RiskDetectionLocation{
90		-	RiskLocation: riskLocation,
91		-	Content: *detection.Source.Text,
92		-	})
	73	+	content = *detection.Source.Text
93	74		}
	75	+
	76	+	holder.addDatatype(ruleName, &db.DataType{Name: content}, nil, fileName, lineNumber, schema.Schema{Parent: parent}, true)
94	77		}
95	78
96	79		func (holder *Holder) AddSchema(detection detections.Detection) error {
		skipped 15 lines
112	95		detection.Source.Filename,
113	96		*detection.Source.LineNumber,
114	97		schema,
	98	+	false,
115	99		)
116	100		}
117	101
		skipped 1 lines
119	103		}
120	104
121	105		// addDatatype adds detector to hash list and at the same time blocks duplicates
122		-	func (holder Holder) addDatatype(ruleName string, datatype db.DataType, subjectName *string, fileName string, lineNumber int, schema schema.Schema) {
	106	+	func (holder Holder) addDatatype(ruleName string, datatype db.DataType, subjectName *string, fileName string, lineNumber int, schema schema.Schema, isPresence bool) {
123	107		if datatype == nil {
124	108		// FIXME: we end up with empty field Name and no datatype with the new code
125	109		// Might be related to the bug with the Unique Identifier classification
		skipped 3 lines
129	113		// create detector entry if it doesn't exist
130	114		if _, exists := holder.detectors[ruleName]; !exists {
131	115		holder.detectors[ruleName] = detectorHolder{
132		-	id: ruleName,
133		-	datatypes: make(map[string]*datatypeHolder),
	116	+	id: ruleName,
	117	+	files: make(map[string]fileHolder),
134	118		}
135	119		}
136	120
137	121		detector := holder.detectors[ruleName]
138		-	// create datatype entry if it doesn't exist
139		-	if _, exists := detector.datatypes[datatype.Name]; !exists {
140		-	if holder.isInternal {
141		-	detector.datatypes[datatype.Name] = &datatypeHolder{
142		-	name: datatype.Name,
143		-	uuid: datatype.UUID,
144		-	categoryUUID: datatype.CategoryUUID,
145		-	files: make(map[string]map[int]*fileHolder),
146		-	}
147		-	} else {
148		-	detector.datatypes[datatype.Name] = &datatypeHolder{
149		-	name: datatype.Name,
150		-	files: make(map[string]map[int]*fileHolder),
151		-	}
	122	+	// create file entry if it doesn't exist
	123	+	if _, exists := detector.files[fileName]; !exists {
	124	+	detector.files[fileName] = fileHolder{
	125	+	name: fileName,
	126	+	lineNumber: make(map[int]lineHolder),
152	127		}
153	128		}
154	129
155		-	detectorDatatype := detector.datatypes[datatype.Name]
156		-	// create file entry if it doesn't exist
157		-	if _, exists := detectorDatatype.files[fileName]; !exists {
158		-	detectorDatatype.files[fileName] = make(map[int]*fileHolder, 0)
	130	+	file := detector.files[fileName]
	131	+	// create line number entry if it doesn't exist
	132	+	if _, exists := file.lineNumber[lineNumber]; !exists {
	133	+	file.lineNumber[lineNumber] = lineHolder{
	134	+	lineNumber: lineNumber,
	135	+	dataTypeCategory: make(map[string]dataTypeCategoryHolder),
	136	+	}
	137	+	}
	138	+
	139	+	line := file.lineNumber[lineNumber]
	140	+	// create datatype category entry if it doesn't exist
	141	+	if _, exists := line.dataTypeCategory[datatype.Name]; !exists {
	142	+	line.dataTypeCategory[datatype.Name] = dataTypeCategoryHolder{
	143	+	name: datatype.Name,
	144	+	isPresence: isPresence,
	145	+	dataType: make(map[string]dataTypeHolder),
	146	+	}
159	147		}
160	148
161		-	detectorDatatype.files[fileName][lineNumber] = &fileHolder{
162		-	name: fileName,
163		-	lineNumber: lineNumber,
164		-	parent: schema.Parent,
165		-	fieldName: schema.FieldName,
166		-	objectName: schema.ObjectName,
167		-	subjectName: subjectName,
	149	+	if isPresence {
	150	+	datatypeCategory := line.dataTypeCategory[datatype.Name]
	151	+	if schema.Parent != nil {
	152	+	datatypeKey := schema.Parent.Content
	153	+	// create datatype if it doesn't exists
	154	+	if _, exists := datatypeCategory.dataType[datatypeKey]; !exists {
	155	+	datatypeCategory.dataType[datatypeKey] = dataTypeHolder{
	156	+	content: &datatypeKey,
	157	+	}
	158	+	}
	159	+	}
	160	+	} else {
	161	+	datatypeCategory := line.dataTypeCategory[datatype.Name]
	162	+	datatypeKey := schema.FieldName + schema.ObjectName
	163	+	// create datatype if it doesn't exists
	164	+	if _, exists := datatypeCategory.dataType[datatypeKey]; !exists {
	165	+	datatypeCategory.dataType[datatypeKey] = dataTypeHolder{
	166	+	parent: schema.Parent,
	167	+	fieldName: &schema.FieldName,
	168	+	objectName: &schema.ObjectName,
	169	+	subjectName: subjectName,
	170	+	content: nil,
	171	+	}
	172	+	}
168	173		}
169	174		}
170	175
171	176		func (holder *Holder) ToDataFlow() []interface{} {
172	177		data := make([]interface{}, 0)
173	178
174		-	detectors := maputil.ToSortedSlice(holder.detectors)
	179	+	for _, detector := range maputil.ToSortedSlice(holder.detectors) {
	180	+	stored := false
	181	+	if customDetector, isCustomDetector := holder.config.Rules[detector.id]; isCustomDetector {
	182	+	stored = customDetector.Stored
	183	+	}
175	184
176		-	for _, detector := range detectors {
177	185		constructedDetector := types.RiskDetector{
178	186		DetectorID: detector.id,
179		-	DataTypes: make([]types.RiskDatatype, 0),
180	187		}
	188	+	locations := []types.RiskLocation{}
181	189
182		-	datatypes := maputil.ToSortedSlice(detector.datatypes)
	190	+	for _, file := range maputil.ToSortedSlice(detector.files) {
183	191
184		-	for _, datatype := range datatypes {
185		-	stored := false
186		-	if customDetector, isCustomDetector := holder.config.Rules[detector.id]; isCustomDetector {
187		-	stored = customDetector.Stored
188		-	}
	192	+	for _, line := range maputil.ToSortedSlice(file.lineNumber) {
	193	+	location := types.RiskLocation{
	194	+	Filename: file.name,
	195	+	LineNumber: line.lineNumber,
	196	+	}
189	197
190		-	constructedDatatype := types.RiskDatatype{
191		-	Name: datatype.name,
192		-	UUID: datatype.uuid,
193		-	CategoryUUID: datatype.categoryUUID,
194		-	Stored: stored,
195		-	Locations: make([]types.RiskLocation, 0),
196		-	}
	198	+	for _, dataTypeCategory := range maputil.ToSortedSlice(line.dataTypeCategory) {
	199	+	category := types.RiskDatatypeCategory{
	200	+	Name: dataTypeCategory.name,
	201	+	IsPresence: dataTypeCategory.isPresence,
	202	+	}
	203	+
	204	+	for _, dataType := range maputil.ToSortedSlice(dataTypeCategory.dataType) {
	205	+	riskDatatype := types.RiskDatatype{
	206	+	Parent: dataType.parent,
	207	+	SubjectName: dataType.subjectName,
	208	+	Stored: stored,
	209	+	}
197	210
198		-	files := maputil.ToSortedSlice(datatype.files)
199		-	for _, locations := range files {
200		-	sortedLocations := maputil.ToSortedSlice(locations)
201		-	for _, location := range sortedLocations {
202		-	constructedDatatype.Locations = append(constructedDatatype.Locations, types.RiskLocation{
203		-	Filename: location.name,
204		-	LineNumber: location.lineNumber,
205		-	Parent: location.parent,
206		-	FieldName: location.fieldName,
207		-	ObjectName: location.objectName,
208		-	SubjectName: location.subjectName,
209		-	})
	211	+	if dataType.content != nil {
	212	+	riskDatatype.Content = *dataType.content
	213	+	}
	214	+	if dataType.fieldName != nil {
	215	+	riskDatatype.FieldName = *dataType.fieldName
	216	+	}
	217	+	if dataType.objectName != nil {
	218	+	riskDatatype.ObjectName = *dataType.objectName
	219	+	}
	220	+
	221	+	category.DataTypes = append(category.DataTypes, riskDatatype)
	222	+	}
	223	+
	224	+	location.DataTypeCategories = append(location.DataTypeCategories, category)
210	225		}
	226	+
	227	+	locations = append(locations, location)
211	228		}
212		-	constructedDetector.DataTypes = append(constructedDetector.DataTypes, constructedDatatype)
	229	+
213	230		}
	231	+	constructedDetector.Locations = locations
214	232
215	233		data = append(data, constructedDetector)
216		-	}
217		-
218		-	sortedRisks := maputil.ToSortedSlice(holder.presentRisks)
219		-	for _, presentRisk := range sortedRisks {
220		-	sortLocations(presentRisk)
221		-	data = append(data, presentRisk)
222	234		}
223	235
224	236		return data
		skipped 19 lines
244	256		return &parent
245	257		}
246	258
247		-	func sortLocations(risk *types.RiskDetection) {
248		-	sort.Slice(risk.Locations, func(i, j int) bool {
249		-	locationA := risk.Locations[i]
250		-	locationB := risk.Locations[j]
251		-
252		-	if locationA.Filename < locationB.Filename {
253		-	return true
254		-	}
255		-	if locationA.Filename > locationB.Filename {
256		-	return false
257		-	}
258		-
259		-	if locationA.LineNumber < locationB.LineNumber {
260		-	return true
261		-	}
262		-	if locationA.LineNumber > locationB.LineNumber {
263		-	return false
264		-	}
265		-
266		-	if locationA.Parent == nil && locationB.Parent != nil {
267		-	return true
268		-	}
269		-	if locationA.Parent != nil && locationB.Parent == nil {
270		-	return false
271		-	}
272		-	if locationA.Parent != nil {
273		-	if locationA.Parent.LineNumber < locationB.Parent.LineNumber {
274		-	return true
275		-	}
276		-	if locationA.Parent.LineNumber > locationB.Parent.LineNumber {
277		-	return false
278		-	}
279		-
280		-	if locationA.Parent.Content < locationB.Parent.Content {
281		-	return true
282		-	}
283		-	if locationA.Parent.Content > locationB.Parent.Content {
284		-	return false
285		-	}
286		-	}
287		-
288		-	return locationA.Content < locationB.Content
289		-	})
290		-	}
291		-

■ ■ ■ ■ ■ ■

pkg/report/output/dataflow/risks/risks_test.go

		skipped 38 lines
39	39		Want: []interface{}{
40	40		types.RiskDetector{
41	41		DetectorID: "detect_ruby_logger",
42		-	DataTypes: []types.RiskDatatype{
	42	+	Locations: []types.RiskLocation{
43	43		{
44		-	Name: "Username",
45		-	Stored: false,
46		-	Locations: []types.RiskLocation{
47		-	{Filename: "./users.rb", LineNumber: 25, FieldName: "User_name"},
	44	+	Filename: "./users.rb",
	45	+	LineNumber: 25,
	46	+	DataTypeCategories: []types.RiskDatatypeCategory{
	47	+	{
	48	+	Name: "Username",
	49	+	IsPresence: false,
	50	+	DataTypes: []types.RiskDatatype{
	51	+	{
	52	+	FieldName: "User_name",
	53	+	},
	54	+	},
	55	+	},
48	56		},
49	57		},
50	58		},
		skipped 14 lines
65	73		Want: []interface{}{
66	74		types.RiskDetector{
67	75		DetectorID: "detect_ruby_logger",
68		-	DataTypes: []types.RiskDatatype{
	76	+	Locations: []types.RiskLocation{
69	77		{
70		-	Name: "Username",
71		-	Stored: false,
72		-	Locations: []types.RiskLocation{
73		-	{Filename: "./users.rb", LineNumber: 25, FieldName: "User_name"},
	78	+	Filename: "./users.rb",
	79	+	LineNumber: 25,
	80	+	DataTypeCategories: []types.RiskDatatypeCategory{
	81	+	{
	82	+	Name: "Username",
	83	+	IsPresence: false,
	84	+	DataTypes: []types.RiskDatatype{
	85	+	{
	86	+	FieldName: "User_name",
	87	+	},
	88	+	},
	89	+	},
74	90		},
75	91		},
76	92		},
		skipped 7 lines
84	100		Want: []interface{}{
85	101		types.RiskDetector{
86	102		DetectorID: "ruby_leak",
87		-	DataTypes: []types.RiskDatatype{
	103	+	Locations: []types.RiskLocation{
88	104		{
89		-	Name: "Username",
90		-	Stored: true,
91		-	Locations: []types.RiskLocation{
92		-	{Filename: "./users.rb", LineNumber: 25, FieldName: "User_name"},
	105	+	Filename: "./users.rb",
	106	+	LineNumber: 25,
	107	+	DataTypeCategories: []types.RiskDatatypeCategory{
	108	+	{
	109	+	Name: "Username",
	110	+	IsPresence: false,
	111	+	DataTypes: []types.RiskDatatype{
	112	+	{
	113	+	Stored: true,
	114	+	FieldName: "User_name",
	115	+	},
	116	+	},
	117	+	},
93	118		},
94	119		},
95	120		},
		skipped 4 lines
100	125		Name: "single detection - multiple occurences - deterministic output",
101	126		Config: config,
102	127		FileContent: `{"id": "1", "type": "custom_classified", "detector_type":"detect_ruby_logger", "source": {"filename": "./users.rb", "line_number": 25}, "value": {"field_name": "User_name", "classification": {"data_type": {"name": "Username", "uuid": "123", "category_uuid": "456"} ,"decision":{"state": "valid"}}}}
103		-	{"id": "2", "type": "custom_classified", "detector_type":"detect_ruby_logger", "source": {"filename": "./users.rb", "line_number": 2}, "value": {"field_name": "User_name", "classification": {"data_type": {"name": "Username", "uuid": "123", "category_uuid": "456"} ,"decision":{"state": "valid"}}}}`,
	128	+	{"id": "2", "type": "custom_classified", "detector_type":"detect_ruby_logger", "source": {"filename": "./users.rb", "line_number": 25}, "value": {"field_name": "User_name", "classification": {"data_type": {"name": "Username", "uuid": "123", "category_uuid": "456"} ,"decision":{"state": "valid"}}}}`,
104	129		Want: []interface{}{
105	130		types.RiskDetector{
106	131		DetectorID: "detect_ruby_logger",
107		-	DataTypes: []types.RiskDatatype{
	132	+	Locations: []types.RiskLocation{
108	133		{
109		-	Name: "Username",
110		-	Stored: false,
111		-	Locations: []types.RiskLocation{
112		-	{Filename: "./users.rb", LineNumber: 2, FieldName: "User_name"},
113		-	{Filename: "./users.rb", LineNumber: 25, FieldName: "User_name"},
	134	+	Filename: "./users.rb",
	135	+	LineNumber: 25,
	136	+	DataTypeCategories: []types.RiskDatatypeCategory{
	137	+	{
	138	+	Name: "Username",
	139	+	IsPresence: false,
	140	+	DataTypes: []types.RiskDatatype{
	141	+	{
	142	+	FieldName: "User_name",
	143	+	},
	144	+	},
	145	+	},
114	146		},
115	147		},
116	148		},
		skipped 4 lines
121	153		Name: "multiple detections - same detector - deterministic output",
122	154		Config: config,
123	155		FileContent: `{"id": "1", "type": "custom_classified", "detector_type":"detect_ruby_logger", "source": {"filename": "./users.rb", "line_number": 25}, "value": {"field_name": "User_name", "classification": {"data_type": {"name": "Username", "uuid": "123", "category_uuid": "456"} ,"decision":{"state": "valid"}}}}
124		-	{"id": "2", "type": "custom_classified", "detector_type":"detect_ruby_logger", "source": {"filename": "./address.rb", "line_number": 2}, "value": {"field_name": "address", "classification": {"data_type": {"name": "Physical Address", "uuid": "123", "category_uuid": "456"} ,"decision":{"state": "valid"}}}}`,
	156	+	{"id": "2", "type": "custom_classified", "detector_type":"detect_ruby_logger", "source": {"filename": "./users.rb", "line_number": 25}, "value": {"field_name": "address", "classification": {"data_type": {"name": "Physical Address", "uuid": "123", "category_uuid": "456"} ,"decision":{"state": "valid"}}}}`,
125	157		Want: []interface{}{
126	158		types.RiskDetector{
127	159		DetectorID: "detect_ruby_logger",
128		-	DataTypes: []types.RiskDatatype{
129		-	{
130		-	Name: "Physical Address",
131		-	Stored: false,
132		-	Locations: []types.RiskLocation{
133		-	{Filename: "./address.rb", LineNumber: 2, FieldName: "address"},
134		-	},
135		-	},
	160	+	Locations: []types.RiskLocation{
136	161		{
137		-	Name: "Username",
138		-	Stored: false,
139		-	Locations: []types.RiskLocation{
140		-	{Filename: "./users.rb", LineNumber: 25, FieldName: "User_name"},
	162	+	Filename: "./users.rb",
	163	+	LineNumber: 25,
	164	+	DataTypeCategories: []types.RiskDatatypeCategory{
	165	+	{
	166	+	Name: "Physical Address",
	167	+	IsPresence: false,
	168	+	DataTypes: []types.RiskDatatype{
	169	+	{
	170	+	FieldName: "address",
	171	+	},
	172	+	},
	173	+	},
	174	+	{
	175	+	Name: "Username",
	176	+	IsPresence: false,
	177	+	DataTypes: []types.RiskDatatype{
	178	+	{
	179	+	FieldName: "User_name",
	180	+	},
	181	+	},
	182	+	},
141	183		},
142	184		},
143	185		},
		skipped 39 lines

■ ■ ■ ■ ■ ■

pkg/report/output/dataflow/types/risks.go

		skipped 7 lines
8	8		}
9	9
10	10		type RiskLocation struct {
11		-	Filename string `json:"filename" yaml:"filename"`
12		-	LineNumber int `json:"line_number" yaml:"line_number"`
13		-	Parent *schema.Parent `json:"parent,omitempty" yaml:"parent,omitempty"`
14		-	Datatypes []RiskDatatype `json:"data_types,omitempty" yaml:"data_types,omitempty"`
	11	+	Filename string `json:"filename" yaml:"filename"`
	12	+	LineNumber int `json:"line_number" yaml:"line_number"`
	13	+	DataTypeCategories []RiskDatatypeCategory `json:"data_type_categories,omitempty" yaml:"data_type_categories,omitempty"`
	14	+	}
	15	+
	16	+	type RiskDatatypeCategory struct {
	17	+	Name string `json:"name,omitempty" yaml:"name,omitempty"`
	18	+	DataTypes []RiskDatatype `json:"data_types,omitempty" yaml:"data_types,omitempty"`
	19	+	IsPresence bool `json:"is_presence,omitempty" yaml:"is_presence,omitempty"`
15	20		}
16	21
17	22		type RiskDatatype struct {
18		-	Content string `json:"content,omitempty" yaml:"content,omitempty"`
19		-	FieldName string `json:"field_name,omitempty" yaml:"field_name,omitempty"`
20		-	ObjectName string `json:"object_name,omitempty" yaml:"object_name,omitempty"`
21		-	SubjectName *string `json:"subject_name,omitempty" yaml:"subject_name,omitempty"`
22		-	Name string `json:"name,omitempty" yaml:"name,omitempty"`
23		-	UUID string `json:"uuid,omitempty" yaml:"uuid,omitempty"`
24		-	CategoryUUID string `json:"category_uuid,omitempty" yaml:"category_uuid,omitempty"`
25		-	Stored bool `json:"stored" yaml:"stored"`
	23	+	Parent *schema.Parent `json:"parent,omitempty" yaml:"parent,omitempty"`
	24	+	Content string `json:"content,omitempty" yaml:"content,omitempty"`
	25	+	FieldName string `json:"field_name,omitempty" yaml:"field_name,omitempty"`
	26	+	ObjectName string `json:"object_name,omitempty" yaml:"object_name,omitempty"`
	27	+	SubjectName *string `json:"subject_name,omitempty" yaml:"subject_name,omitempty"`
	28	+	Stored bool `json:"stored" yaml:"stored"`
26	29		}
27	30

feat: redo risks