STRLCPY/scorecard

🌱 cron: pass config as an argument to binaries (4/n) (#2279)

* Explicitly read config file instead of embedding it.

Signed-off-by: Spencer Schrock <[email protected]>

* Add CLI config arg and ReadConfig() to existing cron binaries.

Signed-off-by: Spencer Schrock <[email protected]>

* Volume mount config

Signed-off-by: Spencer Schrock <[email protected]>

* Ignore CLI flag args when reading local filenames in controller.

Signed-off-by: Spencer Schrock <[email protected]>

* Hide --config in the config package.

Signed-off-by: Spencer Schrock <[email protected]>

* Add config param to k8s files.

Signed-off-by: Spencer Schrock <[email protected]>

* Fix test

Signed-off-by: Spencer Schrock <[email protected]>

* Allow fallback to embedded config if no config is passed as arg. Intended to be temporary to help with GKE rollout.

Signed-off-by: Spencer Schrock <[email protected]>

Signed-off-by: Spencer Schrock <[email protected]>

Spencer Schrock committed with GitHub 2 years ago

a7a503ae

1 parent 97df43be

Revision indexing in progress... (symbol navigation in revisions will be accurate after indexed)

Total 17 files Show one by one

■ ■ ■ ■ ■ ■

cron/internal/bq/main.go

		skipped 17 lines
18	18		import (
19	19		"bytes"
20	20		"context"
	21	+	"flag"
21	22		"fmt"
22	23		"io"
23	24		"log"
		skipped 126 lines
150	151
151	152		func main() {
152	153		ctx := context.Background()
	154	+
	155	+	flag.Parse()
	156	+	if err := config.ReadConfig(); err != nil {
	157	+	panic(err)
	158	+	}
	159	+
153	160		bucketURL, err := config.GetResultDataBucketURL()
154	161		if err != nil {
155	162		panic(err)
		skipped 26 lines

■ ■ ■ ■ ■ ■

cron/internal/cii/main.go

		skipped 17 lines
18	18		import (
19	19		"context"
20	20		"encoding/json"
	21	+	"flag"
21	22		"fmt"
22	23		"io"
23	24		"net/http"
		skipped 58 lines
82	83		func main() {
83	84		ctx := context.Background()
84	85		fmt.Println("Starting...")
	86	+
	87	+	flag.Parse()
	88	+	if err := config.ReadConfig(); err != nil {
	89	+	panic(err)
	90	+	}
85	91
86	92		ciiDataBucket, err := config.GetCIIDataBucketURL()
87	93		if err != nil {
		skipped 19 lines

■ ■ ■ ■ ■ ■ ■

cron/internal/config/config.go

		skipped 18 lines
19	19		// Used to embed config.yaml.
20	20		_ "embed"
21	21		"errors"
	22	+	"flag"
22	23		"fmt"
23	24		"os"
24	25		"reflect"
		skipped 10 lines
35	36		ShardNumFilename string = ".shard_num"
36	37		// TransferStatusFilename file identifies if shard transfer to BigQuery is completed.
37	38		TransferStatusFilename string = ".transfer_complete"
	39	+
	40	+	configFlag string = "config"
	41	+	configDefault string = ""
	42	+	configUsage string = "Location of config file. Required"
	43	+
38	44		projectID string = "SCORECARD_PROJECT_ID"
39	45		requestTopicURL string = "SCORECARD_REQUEST_TOPIC_URL"
40	46		requestSubscriptionURL string = "SCORECARD_REQUEST_SUBSCRIPTION_URL"
		skipped 14 lines
55	61		ErrorEmptyConfigValue = errors.New("config value set to empty")
56	62		// ErrorValueConversion indicates an unexpected type was found for the value of the config option.
57	63		ErrorValueConversion = errors.New("unexpected type, cannot convert value")
	64	+	// ErrorNoConfig indicates no config file was provided, or flag.Parse() was not called.
	65	+	ErrorNoConfig = errors.New("no configuration file provided with --" + configFlag)
58	66		//go:embed config.yaml
59		-	configYAML []byte
	67	+	configYAML []byte
	68	+	configFilename = flag.String(configFlag, configDefault, configUsage)
60	69		)
61	70
62	71		//nolint:govet
		skipped 130 lines
193	202		return "", err
194	203		}
195	204		return s[key], nil
	205	+	}
	206	+
	207	+	// ReadConfig reads the contents of a configuration file specified with --config for later use by getters.
	208	+	// This function must be called before any other exported function, and after flag.Parse() is called.
	209	+	func ReadConfig() error {
	210	+	var err error
	211	+	if configFilename == nil \|\| *configFilename == "" {
	212	+	return nil
	213	+	}
	214	+	configYAML, err = os.ReadFile(*configFilename)
	215	+	if err != nil {
	216	+	return fmt.Errorf("config file: %w", err)
	217	+	}
	218	+	return nil
196	219		}
197	220
198	221		// GetProjectID returns the cloud projectID for the cron job.
		skipped 107 lines

■ ■ ■ ■ ■ ■

cron/internal/config/config_test.go

		skipped 15 lines
16	16
17	17		import (
18	18		"errors"
	19	+	"log"
19	20		"os"
20	21		"testing"
21	22
		skipped 43 lines
65	66		}
66	67		//nolint
67	68		return os.ReadFile(filename)
	69	+	}
	70	+
	71	+	// runs once before all tests, to initialize the config file for testing purposes.
	72	+	func TestMain(m *testing.M) {
	73	+	// TODO change to config.yaml when removing built-in embedding
	74	+	*configFilename = ""
	75	+	if err := ReadConfig(); err != nil {
	76	+	log.Fatalf("failed to read config: %v", err)
	77	+	}
	78	+	os.Exit(m.Run())
68	79		}
69	80
70	81		func TestYAMLParsing(t *testing.T) {
		skipped 396 lines

■ ■ ■ ■ ■ ■

cron/internal/controller/main.go

		skipped 17 lines
18	18		import (
19	19		"bytes"
20	20		"context"
	21	+	"flag"
21	22		"fmt"
22	23		"os"
23	24		"time"
		skipped 119 lines
143	144		ctx := context.Background()
144	145		t := time.Now()
145	146
	147	+	flag.Parse()
	148	+	if err := config.ReadConfig(); err != nil {
	149	+	panic(err)
	150	+	}
	151	+
146	152		topic, err := config.GetRequestTopicURL()
147	153		if err != nil {
148	154		panic(err)
		skipped 19 lines
168	174		}
169	175
170	176		var reader data.Iterator
171		-	if useLocalFiles := len(os.Args) > 1; useLocalFiles {
172		-	reader = localFiles(os.Args[1:])
	177	+	if useLocalFiles := len(flag.Args()) > 0; useLocalFiles {
	178	+	reader = localFiles(flag.Args())
173	179		} else {
174	180		reader = bucketFiles(ctx)
175	181		}
		skipped 34 lines

■ ■ ■ ■ ■ ■

cron/internal/monitoring/exporter.go

		skipped 47 lines
48	48		}
49	49
50	50		// GetExporter defines a factory for returning opencensus Exporter.
	51	+	// Ensure config.ReadConfig() is called at some point before this function.
51	52		func GetExporter() (Exporter, error) {
52	53		exporter, err := config.GetMetricExporter()
53	54		if err != nil {
		skipped 32 lines

■ ■ ■ ■ ■ ■

cron/internal/worker/main.go

		skipped 49 lines
50	50
51	51		var ignoreRuntimeErrors = flag.Bool("ignoreRuntimeErrors", false, "if set to true any runtime errors will be ignored")
52	52
53		-	// nolint: gocognit
	53	+	//nolint:gocognit
54	54		func processRequest(ctx context.Context,
55	55		batchRequest *data.ScorecardBatchRequest,
56	56		blacklistedChecks []string, bucketURL, rawBucketURL, apiBucketURL string,
		skipped 151 lines
208	208		ctx := context.Background()
209	209
210	210		flag.Parse()
	211	+	if err := config.ReadConfig(); err != nil {
	212	+	panic(err)
	213	+	}
211	214
212	215		checkDocs, err := docs.Read()
213	216		if err != nil {
		skipped 91 lines

■ ■ ■ ■ ■ ■ ■

cron/k8s/README.md

		skipped 10 lines
11	11
12	12		The cluster name is `openssf` which is in zone `us-central1-c`.
13	13
14		-	## Uploading a configuration file
	14	+	## Uploading a cronjob/pod configuration file
15	15
16	16		1. Verify you're working on the `openssf` cluster with `kubectl config current-context`
17	17		2. Run `kubectl apply -f FILENAME` to apply a new configuration
18	18
	19	+
	20	+	## Creating or updating the ConfigMap using the config.yaml file
	21	+
	22	+	We use [ConfigMaps](https://kubernetes.io/docs/concepts/configuration/configmap/) to store our config file (`cron/internal/config/config.yaml`).
	23	+	The file can be created for the first time, or updated, with the same command:
	24	+	```
	25	+	kubectl create configmap scorecard-config --from-file=config.yaml -o yaml --dry-run=client \| kubectl apply -f -
	26	+	```
	27	+
	28	+	### Accessing the config.yaml through ConfigMap
	29	+	The ConfigMap is then volume mounted, so the config file is accessible by any cronjob that specifies the mounting in its yaml.
	30	+

■ ■ ■ ■ ■ ■

cron/k8s/cii.yaml

		skipped 27 lines
28	28		containers:
29	29		- name: cii-worker
30	30		image: gcr.io/openssf/scorecard-cii-worker:stable
	31	+	args: ["--config=/etc/scorecard/config.yaml"]
31	32		imagePullPolicy: Always
	33	+	volumeMounts:
	34	+	- name: config-volume
	35	+	mountPath: /etc/scorecard
	36	+	readOnly: true
	37	+	volumes:
	38	+	- name: config-volume
	39	+	configMap:
	40	+	name: scorecard-config
32	41

■ ■ ■ ■ ■ ■

cron/k8s/controller.release.yaml

		skipped 51 lines
52	52		containers:
53	53		- name: controller
54	54		image: gcr.io/openssf/scorecard-batch-controller:latest
55		-	args: ["cron/internal/data/projects.release.csv"]
	55	+	args: ["--config=/etc/scorecard/config.yaml", "cron/internal/data/projects.release.csv"]
56	56		imagePullPolicy: Always
57	57		env:
58	58		- name: SCORECARD_REQUEST_TOPIC_URL
		skipped 10 lines
69	69		memory: 1Gi
70	70		requests:
71	71		memory: 1Gi
	72	+	volumeMounts:
	73	+	- name: config-volume
	74	+	mountPath: /etc/scorecard
	75	+	readOnly: true
	76	+	volumes:
	77	+	- name: config-volume
	78	+	configMap:
	79	+	name: scorecard-config
72	80		- name: worker-update
73	81		image: bitnami/kubectl@sha256:44468c0f5b348e6dcf5e11feb6fdcc969c874bba2856150fe50eb1aacb3bdfee
74	82		command:
		skipped 5 lines

■ ■ ■ ■ ■ ■

cron/k8s/controller.yaml

		skipped 51 lines
52	52		containers:
53	53		- name: controller
54	54		image: gcr.io/openssf/scorecard-batch-controller:stable
55		-	args: ["cron/internal/data/projects.csv"]
	55	+	args: ["--config=/etc/scorecard/config.yaml", "cron/internal/data/projects.csv"]
56	56		imagePullPolicy: Always
57	57		resources:
58	58		limits:
59	59		memory: 1Gi
60	60		requests:
61	61		memory: 1Gi
	62	+	volumeMounts:
	63	+	- name: config-volume
	64	+	mountPath: /etc/scorecard
	65	+	readOnly: true
	66	+	volumes:
	67	+	- name: config-volume
	68	+	configMap:
	69	+	name: scorecard-config
62	70		- name: worker-update
63	71		image: bitnami/kubectl@sha256:44468c0f5b348e6dcf5e11feb6fdcc969c874bba2856150fe50eb1aacb3bdfee
64	72		command:
		skipped 5 lines

■ ■ ■ ■ ■ ■

cron/k8s/transfer-raw.yaml

		skipped 26 lines
27	27		containers:
28	28		- name: bq-rawdata-transfer
29	29		image: gcr.io/openssf/scorecard-bq-transfer:latest
	30	+	args: ["--config=/etc/scorecard/config.yaml"]
30	31		imagePullPolicy: Always
31	32		resources:
32	33		limits:
		skipped 5 lines
38	39		value: "scorecard-rawdata"
39	40		- name: SCORECARD_DATA_BUCKET_URL
40	41		value: "gs://ossf-scorecard-rawdata"
	42	+	volumeMounts:
	43	+	- name: config-volume
	44	+	mountPath: /etc/scorecard
	45	+	readOnly: true
	46	+	volumes:
	47	+	- name: config-volume
	48	+	configMap:
	49	+	name: scorecard-config
41	50		restartPolicy: OnFailure
42	51

■ ■ ■ ■ ■ ■

cron/k8s/transfer.release-raw.yaml

		skipped 27 lines
28	28		containers:
29	29		- name: bq-transfer-rawdata-releasetest
30	30		image: gcr.io/openssf/scorecard-bq-transfer:latest
	31	+	args: ["--config=/etc/scorecard/config.yaml"]
31	32		imagePullPolicy: Always
32	33		env:
33	34		- name: SCORECARD_DATA_BUCKET_URL
		skipped 7 lines
41	42		memory: 1Gi
42	43		requests:
43	44		memory: 1Gi
	45	+	volumeMounts:
	46	+	- name: config-volume
	47	+	mountPath: /etc/scorecard
	48	+	readOnly: true
	49	+	volumes:
	50	+	- name: config-volume
	51	+	configMap:
	52	+	name: scorecard-config
44	53

■ ■ ■ ■ ■ ■

cron/k8s/transfer.release.yaml

		skipped 27 lines
28	28		containers:
29	29		- name: bq-transfer-releasetest-v2
30	30		image: gcr.io/openssf/scorecard-bq-transfer:latest
	31	+	args: ["--config=/etc/scorecard/config.yaml"]
31	32		imagePullPolicy: Always
32	33		env:
33	34		- name: SCORECARD_DATA_BUCKET_URL
		skipped 9 lines
43	44		memory: 1Gi
44	45		requests:
45	46		memory: 1Gi
	47	+	volumeMounts:
	48	+	- name: config-volume
	49	+	mountPath: /etc/scorecard
	50	+	readOnly: true
	51	+	volumes:
	52	+	- name: config-volume
	53	+	configMap:
	54	+	name: scorecard-config
46	55

■ ■ ■ ■ ■ ■

cron/k8s/transfer.yaml

		skipped 26 lines
27	27		containers:
28	28		- name: bq-transfer-v2
29	29		image: gcr.io/openssf/scorecard-bq-transfer:latest
	30	+	args: ["--config=/etc/scorecard/config.yaml"]
30	31		imagePullPolicy: Always
31	32		resources:
32	33		limits:
33	34		memory: 1Gi
34	35		requests:
35	36		memory: 1Gi
	37	+	volumeMounts:
	38	+	- name: config-volume
	39	+	mountPath: /etc/scorecard
	40	+	readOnly: true
	41	+	volumes:
	42	+	- name: config-volume
	43	+	configMap:
	44	+	name: scorecard-config
36	45		restartPolicy: OnFailure
37	46

■ ■ ■ ■ ■ ■

cron/k8s/worker.release.yaml

		skipped 28 lines
29	29		containers:
30	30		- name: worker
31	31		image: gcr.io/openssf/scorecard-batch-worker:latest
32		-	args: ["--ignoreRuntimeErrors=false"]
	32	+	args: ["--ignoreRuntimeErrors=false", "--config=/etc/scorecard/config.yaml"]
33	33		imagePullPolicy: Always
34	34		env:
35	35		- name: SCORECARD_DATA_BUCKET_URL
		skipped 15 lines
51	51		limits:
52	52		memory: 12Gi
53	53		ephemeral-storage: 500Gi
	54	+	volumeMounts:
	55	+	- name: config-volume
	56	+	mountPath: /etc/scorecard
	57	+	readOnly: true
	58	+	volumes:
	59	+	- name: config-volume
	60	+	configMap:
	61	+	name: scorecard-config
54	62		strategy:
55	63		type: "RollingUpdate"
56	64		rollingUpdate:
		skipped 3 lines

■ ■ ■ ■ ■ ■

cron/k8s/worker.yaml

		skipped 28 lines
29	29		containers:
30	30		- name: worker
31	31		image: gcr.io/openssf/scorecard-batch-worker:stable
32		-	args: ["--ignoreRuntimeErrors=true"]
	32	+	args: ["--ignoreRuntimeErrors=true", "--config=/etc/scorecard/config.yaml"]
33	33		imagePullPolicy: Always
34	34		env:
35	35		- name: GITHUB_AUTH_SERVER
		skipped 5 lines
41	41		limits:
42	42		memory: 12Gi
43	43		ephemeral-storage: 500Gi
	44	+	volumeMounts:
	45	+	- name: config-volume
	46	+	mountPath: /etc/scorecard
	47	+	readOnly: true
	48	+	volumes:
	49	+	- name: config-volume
	50	+	configMap:
	51	+	name: scorecard-config
44	52		strategy:
45	53		type: "RollingUpdate"
46	54		rollingUpdate:
		skipped 3 lines