STRLCPY/scorecard

Fix a bug in cron API data exporting (#2112)

Co-authored-by: Azeem Shaikh <[email protected]>

Azeem Shaikh committed with GitHub 2 years ago

69eb1ccf

1 parent 89163cc4

Revision indexing in progress... (symbol navigation in revisions will be accurate after indexed)

Total 4 files Show one by one

■ ■ ■ ■ ■ ■

cron/internal/config/config.go

		skipped 78 lines
79	79		// Raw results.
80	80		RawResultDataBucketURL string `yaml:"raw-result-data-bucket-url"`
81	81		RawBigQueryTable string `yaml:"raw-bigquery-table"`
82		-	ExportResultsBucketURL string `yaml:"export-results-bucket-url"`
	82	+	APIResultsBucketURL string `yaml:"api-results-bucket-url"`
83	83		}
84	84
85	85		func getParsedConfigFromFile(byteValue []byte) (config, error) {
		skipped 148 lines
234	234		return getStringConfigValue(metricExporter, configYAML, "MetricExporter", "metric-exporter")
235	235		}
236	236
237		-	// GetBQExportResultsBucketURL returns the bucket URL for storing cron job results.
238		-	func GetBQExportResultsBucketURL() (string, error) {
	237	+	// GetAPIResultsBucketURL returns the bucket URL for storing cron job results.
	238	+	func GetAPIResultsBucketURL() (string, error) {
239	239		return getStringConfigValue(apiResultsBucketURL, configYAML,
240		-	"ExportResultsBucketURL", "export-results-bucket-url")
	240	+	"APIResultsBucketURL", "api-results-bucket-url")
241	241		}
242	242

■ ■ ■ ■ ■ ■

cron/internal/config/config.yaml

		skipped 28 lines
29	29		# Raw results.
30	30		raw-result-data-bucket-url: gs://ossf-scorecard-rawdata
31	31		raw-bigquery-table: scorecard-rawdata
32		-	# export-bucket
33		-	export-results-bucket-url: gs://ossf-scorecard-cron-releasetest-results
	32	+	# API results bucket
	33	+	api-results-bucket-url: gs://ossf-scorecard-cron-results
34	34

■ ■ ■ ■ ■ ■

cron/internal/config/config_test.go

		skipped 36 lines
37	37		prodShardSize int = 10
38	38		prodMetricExporter string = "stackdriver"
39	39		// Raw results.
40		-	prodRawBucket = "gs://ossf-scorecard-rawdata"
41		-	prodRawBigQueryTable = "scorecard-rawdata"
42		-	prodBigQueryExportsBucketURL = "gs://ossf-scorecard-cron-releasetest-results"
	40	+	prodRawBucket = "gs://ossf-scorecard-rawdata"
	41	+	prodRawBigQueryTable = "scorecard-rawdata"
	42	+	prodAPIBucketURL = "gs://ossf-scorecard-cron-results"
43	43		)
44	44
45	45		func getByteValueFromFile(filename string) ([]byte, error) {
		skipped 29 lines
75	75		MetricExporter: prodMetricExporter,
76	76		RawResultDataBucketURL: prodRawBucket,
77	77		RawBigQueryTable: prodRawBigQueryTable,
78		-	ExportResultsBucketURL: prodBigQueryExportsBucketURL,
	78	+	APIResultsBucketURL: prodAPIBucketURL,
79	79		},
80	80		},
81	81
		skipped 266 lines
348	348		}
349	349
350	350		//nolint:paralleltest // Since os.Setenv is used.
351		-	func TestGetBigQueryExportsBucketURL(t *testing.T) {
	351	+	func TestGetAPIResultsBucketURL(t *testing.T) {
352	352		t.Run("GetBigQueryExportsBucketURL", func(t *testing.T) {
353	353		bigqueryExportsBucketURL := apiResultsBucketURL
354	354		os.Unsetenv(bigqueryExportsBucketURL)
355		-	bucket, err := GetBQExportResultsBucketURL()
	355	+	bucket, err := GetAPIResultsBucketURL()
356	356		if err != nil {
357	357		t.Errorf("failed to get production bucket URL from config: %v", err)
358	358		}
359		-	if bucket != prodBigQueryExportsBucketURL {
360		-	t.Errorf("test failed: expected - %s, got = %s", prodBigQueryExportsBucketURL, bucket)
	359	+	if bucket != prodAPIBucketURL {
	360	+	t.Errorf("test failed: expected - %s, got = %s", prodAPIBucketURL, bucket)
361	361		}
362	362		})
363	363		}
		skipped 1 lines

■ ■ ■ ■ ■ ■

cron/internal/worker/main.go

		skipped 42 lines
43	43		"github.com/ossf/scorecard/v4/stats"
44	44		)
45	45
	46	+	const (
	47	+	resultsFile = "results.json"
	48	+	rawResultsFile = "raw.json"
	49	+	)
	50	+
46	51		var ignoreRuntimeErrors = flag.Bool("ignoreRuntimeErrors", false, "if set to true any runtime errors will be ignored")
47	52
48	53		// nolint: gocognit
49	54		func processRequest(ctx context.Context,
50	55		batchRequest *data.ScorecardBatchRequest,
51		-	blacklistedChecks []string, bucketURL, rawBucketURL, exportBucketURL string,
	56	+	blacklistedChecks []string, bucketURL, rawBucketURL, apiBucketURL string,
52	57		checkDocs docs.Doc,
53	58		repoClient clients.RepoClient, ossFuzzRepoClient clients.RepoClient,
54	59		ciiClient clients.CIIBestPracticesClient,
		skipped 84 lines
139	144		if err := format.AsRawJSON(&result, &exportRawBuffer); err != nil {
140	145		return fmt.Errorf("error during result.AsRawJSON for export: %w", err)
141	146		}
142		-	exportPath := fmt.Sprintf("%s/result.json", repo.URI())
143		-	exportCommitSHAPath := fmt.Sprintf("%s/%s/result.json", repo.URI(), result.Repo.CommitSHA)
144		-	exportRawPath := fmt.Sprintf("%s/raw.json", repo.URI())
145		-	exportRawCommitSHAPath := fmt.Sprintf("%s/%s/raw.json", repo.URI(), result.Repo.CommitSHA)
	147	+	exportPath := fmt.Sprintf("%s/%s", repo.URI(), resultsFile)
	148	+	exportCommitSHAPath := fmt.Sprintf("%s/%s/%s", repo.URI(), result.Repo.CommitSHA, resultsFile)
	149	+	exportRawPath := fmt.Sprintf("%s/%s", repo.URI(), rawResultsFile)
	150	+	exportRawCommitSHAPath := fmt.Sprintf("%s/%s/%s", repo.URI(), result.Repo.CommitSHA, rawResultsFile)
146	151
147	152		// Raw result.
148	153		if err := format.AsRawJSON(&result, &rawBuffer); err != nil {
		skipped 1 lines
150	155		}
151	156
152	157		// These are results without the commit SHA which represents the latest commit.
153		-	if err := data.WriteToBlobStore(ctx, exportBucketURL, exportPath, exportBuffer.Bytes()); err != nil {
	158	+	if err := data.WriteToBlobStore(ctx, apiBucketURL, exportPath, exportBuffer.Bytes()); err != nil {
154	159		return fmt.Errorf("error during writing to exportBucketURL: %w", err)
155	160		}
156	161		// Export result based on commitSHA.
157		-	if err := data.WriteToBlobStore(ctx, exportBucketURL, exportCommitSHAPath, exportBuffer.Bytes()); err != nil {
	162	+	if err := data.WriteToBlobStore(ctx, apiBucketURL, exportCommitSHAPath, exportBuffer.Bytes()); err != nil {
158	163		return fmt.Errorf("error during exportBucketURL with commit SHA: %w", err)
159	164		}
160	165		// Export raw result.
161		-	if err := data.WriteToBlobStore(ctx, exportBucketURL, exportRawPath, exportRawBuffer.Bytes()); err != nil {
	166	+	if err := data.WriteToBlobStore(ctx, apiBucketURL, exportRawPath, exportRawBuffer.Bytes()); err != nil {
162	167		return fmt.Errorf("error during writing to exportBucketURL for raw results: %w", err)
163	168		}
164		-	if err := data.WriteToBlobStore(ctx, exportBucketURL, exportRawCommitSHAPath, exportRawBuffer.Bytes()); err != nil {
	169	+	if err := data.WriteToBlobStore(ctx, apiBucketURL, exportRawCommitSHAPath, exportRawBuffer.Bytes()); err != nil {
165	170		return fmt.Errorf("error during exportBucketURL for raw results with commit SHA: %w", err)
166	171		}
167	172		}
		skipped 70 lines
238	243		panic(err)
239	244		}
240	245
241		-	exportBucketURL, err := config.GetBQExportResultsBucketURL()
	246	+	apiBucketURL, err := config.GetAPIResultsBucketURL()
242	247		if err != nil {
243	248		panic(err)
244	249		}
		skipped 33 lines
278	283		break
279	284		}
280	285		if err := processRequest(ctx, req, blacklistedChecks,
281		-	bucketURL, rawBucketURL, exportBucketURL, checkDocs,
	286	+	bucketURL, rawBucketURL, apiBucketURL, checkDocs,
282	287		repoClient, ossFuzzRepoClient, ciiClient, vulnsClient, logger); err != nil {
283	288		// TODO(log): Previously Warn. Consider logging an error here.
284	289		logger.Info(fmt.Sprintf("error processing request: %v", err))
		skipped 14 lines