STRLCPY/scorecard

✨ Gitlab support: RepoClient (#2655)

* Add make targets and E2E test target for GitLab only

Signed-off-by: Raghav Kaul <[email protected]>

* Add GitLab support to RepoClient

Signed-off-by: Raghav Kaul <[email protected]>

* Build

* Make target for e2e-gitlab-token
* Only run Gitlab tests in CI that don't require a token

Signed-off-by: Raghav Kaul <[email protected]>

* Add tests

Signed-off-by: Raghav Kaul <[email protected]>

* Remove spurious printf

Signed-off-by: Raghav Kaul <[email protected]>

* 🐛 Check OSS Fuzz build file for Fuzzing check (#2719)

* Check OSS-Fuzz using project list

Signed-off-by: Spencer Schrock <[email protected]>

* Use clients.RepoClient interface to perform the new OSS Fuzz check

Signed-off-by: Spencer Schrock <[email protected]>

* wip: add eager client for better repeated lookup of projects

Signed-off-by: Spencer Schrock <[email protected]>

* Split lazy and eager behavior into different implementations.

Signed-off-by: Spencer Schrock <[email protected]>

* Add tests and benchmarks

Signed-off-by: Spencer Schrock <[email protected]>

* Switch to always parsing JSON to determine if a project is present. The other approach of looking for a substring match would lead to false positives.

Signed-off-by: Spencer Schrock <[email protected]>

* Add eager constructor to surface status file errors sooner.

Signed-off-by: Spencer Schrock <[email protected]>

* Switch existing users to new OSS Fuzz client

Signed-off-by: Spencer Schrock <[email protected]>

* Mark old method as deprecated in the godoc

Signed-off-by: Spencer Schrock <[email protected]>

* remove unused comment.

Signed-off-by: Spencer Schrock <[email protected]>

* Use new OSS Fuzz client in e2e test.

Signed-off-by: Spencer Schrock <[email protected]>

* fix typo.

Signed-off-by: Spencer Schrock <[email protected]>

* Fix potential path bug with test server.

Signed-off-by: Spencer Schrock <[email protected]>

* Force include the two JSON files which were being ignored by .gitignore

Signed-off-by: Spencer Schrock <[email protected]>

* trim the status json file

Signed-off-by: Spencer Schrock <[email protected]>

---------

Signed-off-by: Spencer Schrock <[email protected]>

---------

Signed-off-by: Raghav Kaul <[email protected]>
Signed-off-by: Spencer Schrock <[email protected]>
Co-authored-by: Spencer Schrock <[email protected]>

raghavkaul committed with GitHub 1 year ago

110e3522

1 parent 5625dda9

Total 27 files Show one by one

■ ■ ■ ■ ■ ■

.github/workflows/integration.yml

		skipped 71 lines
72	72		files: ./e2e-coverage.out
73	73		verbose: true
74	74
	75	+	- name: Run GitLab E2E #using retry because the GitHub token is being throttled.
	76	+	uses: nick-invision/retry@943e742917ac94714d2f408a0e8320f2d1fcafcd
	77	+	with:
	78	+	max_attempts: 3
	79	+	retry_on: error
	80	+	timeout_minutes: 30
	81	+	command: make e2e-gitlab
	82	+
	83	+	- name: codecov
	84	+	uses: codecov/codecov-action@81cd2dc8148241f03f5839d295e000b8f761e378 # 2.1.0
	85	+	with:
	86	+	files: ./e2e-coverage.out
	87	+	verbose: true
	88	+

■ ■ ■ ■ ■ ■

Makefile

		skipped 333 lines
334	334		# Run e2e tests. GITHUB_AUTH_TOKEN set to secrets.GITHUB_TOKEN must be used to run this.
335	335		TOKEN_TYPE="GITHUB_TOKEN" $(GINKGO) --race -p -v -cover -coverprofile=e2e-coverage.out --keep-separate-coverprofiles ./...
336	336
	337	+	e2e-gitlab-token: ## Runs e2e tests that require a GITLAB_TOKEN
	338	+	TOKEN_TYPE="GITLAB_PAT" $(GINKGO) --race -p -vv --focus '.*GitLab Token' ./...
	339	+
	340	+	e2e-gitlab: ## Runs e2e tests for GitLab only. TOKEN_TYPE is not used (since these are public APIs), but must be set to something
	341	+	TOKEN_TYPE="GITLAB_PAT" $(GINKGO) --race -p -vv --focus '.*GitLab' ./...
	342	+
337	343		e2e-attestor: ## Runs e2e tests for scorecard-attestor
338	344		cd attestor/e2e; go test -covermode=atomic -coverprofile=e2e-coverage.out; cd ../..
339	345
		skipped 98 lines

■ ■ ■ ■ ■ ■

checker/client.go

		skipped 16 lines
17	17		import (
18	18		"context"
19	19		"fmt"
	20	+	"os"
20	21
21	22		"github.com/ossf/scorecard/v4/clients"
22	23		ghrepo "github.com/ossf/scorecard/v4/clients/githubrepo"
	24	+	glrepo "github.com/ossf/scorecard/v4/clients/gitlabrepo"
23	25		"github.com/ossf/scorecard/v4/clients/localdir"
24	26		"github.com/ossf/scorecard/v4/clients/ossfuzz"
25	27		"github.com/ossf/scorecard/v4/log"
		skipped 9 lines
35	37		clients.VulnerabilitiesClient, // vulnClient
36	38		error,
37	39		) {
38		-	var githubRepo clients.Repo
	40	+	var repo clients.Repo
	41	+	var makeRepoError error
	42	+
39	43		if localURI != "" {
40	44		localRepo, errLocal := localdir.MakeLocalDirRepo(localURI)
41	45		var retErr error
		skipped 8 lines
50	54		retErr
51	55		}
52	56
53		-	githubRepo, errGitHub := ghrepo.MakeGithubRepo(repoURI)
54		-	if errGitHub != nil {
55		-	return githubRepo,
56		-	nil,
57		-	nil,
58		-	nil,
59		-	nil,
60		-	fmt.Errorf("getting local directory client: %w", errGitHub)
	57	+	_, experimental := os.LookupEnv("SCORECARD_EXPERIMENTAL")
	58	+	var repoClient clients.RepoClient
	59	+
	60	+	//nolint:nestif
	61	+	if experimental && glrepo.DetectGitLab(repoURI) {
	62	+	repo, makeRepoError = glrepo.MakeGitlabRepo(repoURI)
	63	+	if makeRepoError != nil {
	64	+	return repo,
	65	+	nil,
	66	+	nil,
	67	+	nil,
	68	+	nil,
	69	+	fmt.Errorf("getting local directory client: %w", makeRepoError)
	70	+	}
	71	+
	72	+	var err error
	73	+	repoClient, err = glrepo.CreateGitlabClientWithToken(ctx, os.Getenv("GITLAB_AUTH_TOKEN"), repo)
	74	+	if err != nil {
	75	+	return repo,
	76	+	nil,
	77	+	nil,
	78	+	nil,
	79	+	nil,
	80	+	fmt.Errorf("error creating gitlab client: %w", err)
	81	+	}
	82	+	} else {
	83	+	repo, makeRepoError = ghrepo.MakeGithubRepo(repoURI)
	84	+	if makeRepoError != nil {
	85	+	return repo,
	86	+	nil,
	87	+	nil,
	88	+	nil,
	89	+	nil,
	90	+	fmt.Errorf("getting local directory client: %w", makeRepoError)
	91	+	}
	92	+	repoClient = ghrepo.CreateGithubRepoClient(ctx, logger)
61	93		}
62	94
63		-	return githubRepo, /repo/
64		-	ghrepo.CreateGithubRepoClient(ctx, logger), /repoClient/
	95	+	return repo, /repo/
	96	+	repoClient, /repoClient/
65	97		ossfuzz.CreateOSSFuzzClient(ossfuzz.StatusURL), /ossFuzzClient/
66	98		clients.DefaultCIIBestPracticesClient(), /ciiClient/
67	99		clients.DefaultVulnerabilitiesClient(), /vulnClient/
		skipped 3 lines

■ ■ ■ ■ ■ ■

clients/githubrepo/repo.go

		skipped 75 lines
76	76		return fmt.Sprintf("%s/%s/%s", r.host, r.owner, r.repo)
77	77		}
78	78
	79	+	func (r *repoURL) Host() string {
	80	+	return r.host
	81	+	}
	82	+
79	83		// String implements Repo.String.
80	84		func (r *repoURL) String() string {
81	85		return fmt.Sprintf("%s-%s-%s", r.host, r.owner, r.repo)
		skipped 56 lines

■ ■ ■ ■ ■ ■

clients/githubrepo/repo_test.go

		skipped 96 lines
97	97		if !tt.wantErr && !cmp.Equal(tt.expected, r, cmp.AllowUnexported(repoURL{})) {
98	98		t.Errorf("Got diff: %s", cmp.Diff(tt.expected, r))
99	99		}
	100	+
	101	+	if !cmp.Equal(r.Host(), tt.expected.host) {
	102	+	t.Errorf("%s expected host: %s got host %s", tt.inputURL, tt.expected.host, r.Host())
	103	+	}
100	104		})
101	105		}
102	106		}
		skipped 1 lines

■ ■ ■ ■ ■ ■

clients/gitlabrepo/branches.go

		skipped 45 lines
46	46		return
47	47		}
48	48
49		-	proj, _, err := handler.glClient.Projects.GetProject(handler.repourl.projectID, &gitlab.GetProjectOptions{})
	49	+	proj, _, err := handler.glClient.Projects.GetProject(handler.repourl.project, &gitlab.GetProjectOptions{})
50	50		if err != nil {
51	51		handler.errSetup = fmt.Errorf("requirest for project failed with error %w", err)
52	52		return
53	53		}
54	54
55		-	branch, _, err := handler.glClient.Branches.GetBranch(handler.repourl.projectID, proj.DefaultBranch)
	55	+	branch, _, err := handler.glClient.Branches.GetBranch(handler.repourl.project, proj.DefaultBranch)
56	56		if err != nil {
57	57		handler.errSetup = fmt.Errorf("request for default branch failed with error %w", err)
58	58		return
		skipped 1 lines
60	60
61	61		if branch.Protected {
62	62		protectedBranch, resp, err := handler.glClient.ProtectedBranches.GetProtectedBranch(
63		-	handler.repourl.projectID, branch.Name)
	63	+	handler.repourl.project, branch.Name)
64	64		if err != nil && resp.StatusCode != 403 {
65	65		handler.errSetup = fmt.Errorf("request for protected branch failed with error %w", err)
66	66		return
		skipped 3 lines
70	70		}
71	71
72	72		projectStatusChecks, resp, err := handler.glClient.ExternalStatusChecks.ListProjectStatusChecks(
73		-	handler.repourl.projectID, &gitlab.ListOptions{})
	73	+	handler.repourl.project, &gitlab.ListOptions{})
74	74		if err != nil && resp.StatusCode != 404 {
75	75		handler.errSetup = fmt.Errorf("request for external status checks failed with error %w", err)
76	76		return
77	77		}
78	78
79		-	projectApprovalRule, resp, err := handler.glClient.Projects.GetApprovalConfiguration(handler.repourl.projectID)
	79	+	projectApprovalRule, resp, err := handler.glClient.Projects.GetApprovalConfiguration(handler.repourl.project)
80	80		if err != nil && resp.StatusCode != 404 {
81	81		handler.errSetup = fmt.Errorf("request for project approval rule failed with %w", err)
82	82		return
		skipped 22 lines
105	105		}
106	106
107	107		func (handler branchesHandler) getBranch(branch string) (clients.BranchRef, error) {
108		-	bran, _, err := handler.glClient.Branches.GetBranch(handler.repourl.projectID, branch)
	108	+	bran, _, err := handler.glClient.Branches.GetBranch(handler.repourl.project, branch)
109	109		if err != nil {
110	110		return nil, fmt.Errorf("error getting branch in branchsHandler.getBranch: %w", err)
111	111		}
112	112
113	113		if bran.Protected {
114		-	protectedBranch, _, err := handler.glClient.ProtectedBranches.GetProtectedBranch(handler.repourl.projectID, bran.Name)
	114	+	protectedBranch, _, err := handler.glClient.ProtectedBranches.GetProtectedBranch(handler.repourl.project, bran.Name)
115	115		if err != nil {
116	116		return nil, fmt.Errorf("request for protected branch failed with error %w", err)
117	117		}
118	118
119	119		projectStatusChecks, resp, err := handler.glClient.ExternalStatusChecks.ListProjectStatusChecks(
120		-	handler.repourl.projectID, &gitlab.ListOptions{})
	120	+	handler.repourl.project, &gitlab.ListOptions{})
121	121		if err != nil && resp.StatusCode != 404 {
122	122		return nil, fmt.Errorf("request for external status checks failed with error %w", err)
123	123		}
124	124
125		-	projectApprovalRule, resp, err := handler.glClient.Projects.GetApprovalConfiguration(handler.repourl.projectID)
	125	+	projectApprovalRule, resp, err := handler.glClient.Projects.GetApprovalConfiguration(handler.repourl.project)
126	126		if err != nil && resp.StatusCode != 404 {
127	127		return nil, fmt.Errorf("request for project approval rule failed with %w", err)
128	128		}
		skipped 69 lines

■ ■ ■ ■ ■ ■

clients/gitlabrepo/checkruns.go

		skipped 33 lines
34	34
35	35		func (handler *checkrunsHandler) listCheckRunsForRef(ref string) ([]clients.CheckRun, error) {
36	36		pipelines, _, err := handler.glClient.Pipelines.ListProjectPipelines(
37		-	handler.repourl.projectID, &gitlab.ListProjectPipelinesOptions{})
	37	+	handler.repourl.project, &gitlab.ListProjectPipelinesOptions{})
38	38		if err != nil {
39	39		return nil, fmt.Errorf("request for pipelines returned error: %w", err)
40	40		}
		skipped 20 lines

■ ■ ■ ■ ■ ■

clients/gitlabrepo/client.go

		skipped 51 lines
52	52		languages *languagesHandler
53	53		licenses *licensesHandler
54	54		ctx context.Context
55		-	// tarball tarballHandler
56		-	commitDepth int
	55	+	commitDepth int
57	56		}
58	57
59	58		// InitRepo sets up the GitLab project in local storage for improving performance and GitLab token usage efficiency.
		skipped 4 lines
64	63		}
65	64
66	65		// Sanity check.
67		-	repo, _, err := client.glClient.Projects.GetProject(glRepo.projectID, &gitlab.GetProjectOptions{})
	66	+	proj := fmt.Sprintf("%s/%s", glRepo.owner, glRepo.project)
	67	+	repo, _, err := client.glClient.Projects.GetProject(proj, &gitlab.GetProjectOptions{})
68	68		if err != nil {
69		-	return sce.WithMessage(sce.ErrRepoUnreachable, err.Error())
	69	+	return sce.WithMessage(sce.ErrRepoUnreachable, proj+"\t"+err.Error())
70	70		}
71	71		if commitDepth <= 0 {
72	72		client.commitDepth = 30 // default
		skipped 2 lines
75	75		}
76	76		client.repo = repo
77	77		client.repourl = &repoURL{
78		-	hostname: inputRepo.URI(),
79		-	projectID: fmt.Sprint(repo.ID),
	78	+	scheme: glRepo.scheme,
	79	+	host: glRepo.host,
	80	+	project: fmt.Sprint(repo.ID),
80	81		defaultBranch: repo.DefaultBranch,
81	82		commitSHA: commitSHA,
82	83		}
		skipped 44 lines
127	128		// Init languagesHandler
128	129		client.licenses.init(client.repourl)
129	130
130		-	// Init tarballHandler.
131		-	// client.tarball.init(client.ctx, client.repourl, client.repo, commitSHA)
132	131		return nil
133	132		}
134	133
135	134		func (client *Client) URI() string {
136		-	return fmt.Sprintf("%s/%s/%s", client.repourl.hostname, client.repourl.owner, client.repourl.projectID)
	135	+	return fmt.Sprintf("%s/%s/%s", client.repourl.host, client.repourl.owner, client.repourl.project)
137	136		}
138	137
139	138		func (client *Client) LocalPath() (string, error) {
		skipped 82 lines
222	221		}
223	222
224	223		func CreateGitlabClientWithToken(ctx context.Context, token string, repo clients.Repo) (clients.RepoClient, error) {
225		-	client, err := gitlab.NewClient(token, gitlab.WithBaseURL(repo.URI()))
	224	+	client, err := gitlab.NewClient(token, gitlab.WithBaseURL(repo.Host()))
226	225		if err != nil {
227	226		return nil, fmt.Errorf("could not create gitlab client with error: %w", err)
228	227		}
		skipped 40 lines
269	268		languages: &languagesHandler{
270	269		glClient: client,
271	270		},
	271	+	licenses: &licensesHandler{},
272	272		}, nil
273	273		}
274	274
		skipped 2 lines
277	277		return nil, fmt.Errorf("%w, oss fuzz currently only supported for github repos", clients.ErrUnsupportedFeature)
278	278		}
279	279
	280	+	// DetectGitLab: check whether the repoURI is a GitLab URI
	281	+	// Makes HTTP request to GitLab API.
	282	+	func DetectGitLab(repoURI string) bool {
	283	+	var repo repoURL
	284	+	if err := repo.parse(repoURI); err != nil {
	285	+	return false
	286	+	}
	287	+
	288	+	return repo.IsValid() == nil
	289	+	}
	290	+

■ ■ ■ ■ ■ ■

clients/gitlabrepo/commits.go

		skipped 40 lines
41	41		// nolint: gocognit
42	42		func (handler *commitsHandler) setup() error {
43	43		handler.once.Do(func() {
44		-	commits, _, err := handler.glClient.Commits.ListCommits(handler.repourl.projectID, &gitlab.ListCommitsOptions{})
	44	+	commits, _, err := handler.glClient.Commits.ListCommits(handler.repourl.project, &gitlab.ListCommitsOptions{})
45	45		if err != nil {
46	46		handler.errSetup = fmt.Errorf("request for commits failed with %w", err)
47	47		return
		skipped 28 lines
76	76
77	77		// Commits are able to be a part of multiple merge requests, but the only one that will be important
78	78		// here is the earliest one.
79		-	mergeRequests, _, err := handler.glClient.Commits.ListMergeRequestsByCommit(handler.repourl.projectID, commit.ID)
	79	+	mergeRequests, _, err := handler.glClient.Commits.ListMergeRequestsByCommit(handler.repourl.project, commit.ID)
80	80		if err != nil {
81	81		handler.errSetup = fmt.Errorf("unable to find merge requests associated with commit: %w", err)
82	82		return
		skipped 85 lines

■ ■ ■ ■ ■ ■

clients/gitlabrepo/contributors.go

		skipped 45 lines
46	46		return
47	47		}
48	48		contribs, _, err := handler.glClient.Repositories.Contributors(
49		-	handler.repourl.projectID, &gitlab.ListContributorsOptions{})
	49	+	handler.repourl.project, &gitlab.ListContributorsOptions{})
50	50		if err != nil {
51	51		handler.errSetup = fmt.Errorf("error during ListContributors: %w", err)
52	52		return
		skipped 42 lines

■ ■ ■ ■ ■ ■

clients/gitlabrepo/issues.go

		skipped 39 lines
40	40		func (handler *issuesHandler) setup() error {
41	41		handler.once.Do(func() {
42	42		issues, _, err := handler.glClient.Issues.ListProjectIssues(
43		-	handler.repourl.projectID, &gitlab.ListProjectIssuesOptions{})
	43	+	handler.repourl.project, &gitlab.ListProjectIssuesOptions{})
44	44		if err != nil {
45	45		handler.errSetup = fmt.Errorf("unable to find issues associated with the project id: %w", err)
46	46		return
		skipped 2 lines
49	49		// There doesn't seem to be a good way to get user access_levels in gitlab so the following way may seem incredibly
50	50		// barberic, however I couldn't find a better way in the docs.
51	51		projectAccessTokens, resp, err := handler.glClient.ProjectAccessTokens.ListProjectAccessTokens(
52		-	handler.repourl.projectID, &gitlab.ListProjectAccessTokensOptions{})
	52	+	handler.repourl.project, &gitlab.ListProjectAccessTokensOptions{})
53	53		if err != nil && resp.StatusCode != 401 {
54	54		handler.errSetup = fmt.Errorf("unable to find access tokens associated with the project id: %w", err)
55	55		return
		skipped 64 lines

■ ■ ■ ■ ■ ■

clients/gitlabrepo/languages.go

		skipped 39 lines
40	40		func (handler *languagesHandler) setup() error {
41	41		handler.once.Do(func() {
42	42		client := handler.glClient
43		-	languageMap, _, err := client.Projects.GetProjectLanguages(handler.repourl.projectID)
	43	+	languageMap, _, err := client.Projects.GetProjectLanguages(handler.repourl.project)
44	44		if err != nil \|\| languageMap == nil {
45	45		handler.errSetup = fmt.Errorf("request for repo languages failed with %w", err)
46	46		return
		skipped 26 lines

■ ■ ■ ■ ■ ■

clients/gitlabrepo/project.go

		skipped 38 lines
39	39
40	40		func (handler *projectHandler) setup() error {
41	41		handler.once.Do(func() {
42		-	proj, _, err := handler.glClient.Projects.GetProject(handler.repourl.projectID, &gitlab.GetProjectOptions{})
	42	+	proj, _, err := handler.glClient.Projects.GetProject(handler.repourl.project, &gitlab.GetProjectOptions{})
43	43		if err != nil {
44	44		handler.errSetup = fmt.Errorf("request for project failed with error %w", err)
45	45		return
		skipped 25 lines

■ ■ ■ ■ ■ ■

clients/gitlabrepo/releases.go

		skipped 43 lines
44	44		handler.errSetup = fmt.Errorf("%w: ListReleases only supported for HEAD queries", clients.ErrUnsupportedFeature)
45	45		return
46	46		}
47		-	releases, _, err := handler.glClient.Releases.ListReleases(handler.repourl.projectID, &gitlab.ListReleasesOptions{})
	47	+	releases, _, err := handler.glClient.Releases.ListReleases(handler.repourl.project, &gitlab.ListReleasesOptions{})
48	48		if err != nil {
49	49		handler.errSetup = fmt.Errorf("%w: ListReleases failed", err)
50	50		return
		skipped 36 lines

■ ■ ■ ■ ■ ■

clients/gitlabrepo/repo.go

		skipped 17 lines
18	18
19	19		import (
20	20		"fmt"
21		-	"regexp"
	21	+	"net/url"
22	22		"strings"
	23	+
	24	+	"github.com/xanzy/go-gitlab"
23	25
24	26		"github.com/ossf/scorecard/v4/clients"
25	27		sce "github.com/ossf/scorecard/v4/errors"
		skipped 4 lines
30	32		)
31	33
32	34		type repoURL struct {
33		-	hostname string
	35	+	scheme string
	36	+	host string
34	37		owner string
35		-	projectID string
	38	+	project string
36	39		defaultBranch string
37	40		commitSHA string
38	41		metadata []string
		skipped 2 lines
41	44		// Parses input string into repoURL struct
42	45		/*
43	46		* Accepted input string formats are as follows:
44		-	* "gitlab.<companyDomain:string>.com/<owner:string>/<projectID:int>"
45		-	* "https://gitlab.<companyDomain:string>.com/<owner:string>/<projectID:int>"
	47	+	* "gitlab.<companyDomain:string>.com/<owner:string>/<projectID:string>"
	48	+	* "https://gitlab.<companyDomain:string>.com/<owner:string>/<projectID:string>"
	49	+
	50	+	The following input format is not supported:
	51	+	* https://gitlab.<companyDomain:string>.com/projects/<projectID:int>
46	52		*/
47	53		func (r *repoURL) parse(input string) error {
48		-	switch {
49		-	case strings.Contains(input, "https://"):
50		-	input = strings.TrimPrefix(input, "https://")
51		-	case strings.Contains(input, "http://"):
52		-	input = strings.TrimPrefix(input, "http://")
53		-	case strings.Contains(input, "://"):
54		-	return sce.WithMessage(sce.ErrScorecardInternal, "unknown input format")
	54	+	var t string
	55	+	c := strings.Split(input, "/")
	56	+	switch l := len(c); {
	57	+	// owner/repo format is not supported for gitlab, it's github-only
	58	+	case l == 2:
	59	+	return sce.WithMessage(sce.ErrScorecardInternal, fmt.Sprintf("gitlab repo must specify host: %s", input))
	60	+	case l >= 3:
	61	+	t = input
55	62		}
56	63
57		-	stringParts := strings.Split(input, "/")
	64	+	// Allow skipping scheme for ease-of-use, default to https.
	65	+	if !strings.Contains(t, "://") {
	66	+	t = "https://" + t
	67	+	}
58	68
59		-	stringParts[2] = strings.TrimSuffix(stringParts[2], "/")
	69	+	u, e := url.Parse(t)
	70	+	if e != nil {
	71	+	return sce.WithMessage(sce.ErrScorecardInternal, fmt.Sprintf("url.Parse: %v", e))
	72	+	}
60	73
61		-	r.hostname, r.owner, r.projectID = stringParts[0], stringParts[1], stringParts[2]
	74	+	const splitLen = 2
	75	+	split := strings.SplitN(strings.Trim(u.Path, "/"), "/", splitLen)
	76	+	if len(split) != splitLen {
	77	+	return sce.WithMessage(sce.ErrorInvalidURL, fmt.Sprintf("%v. Expected full repository url", input))
	78	+	}
	79	+
	80	+	r.scheme, r.host, r.owner, r.project = u.Scheme, u.Host, split[0], split[1]
62	81		return nil
63	82		}
64	83
65	84		// URI implements Repo.URI().
66		-	// TODO: there may be a reason the string was originally in format "%s/%s/%s", hostname, owner, projectID,
67		-	// however I changed it to be more "userful".
68	85		func (r *repoURL) URI() string {
69		-	return fmt.Sprintf("https://%s", r.hostname)
	86	+	return fmt.Sprintf("%s/%s/%s", r.host, r.owner, r.project)
	87	+	}
	88	+
	89	+	func (r *repoURL) Host() string {
	90	+	return fmt.Sprintf("%s://%s", r.scheme, r.host)
70	91		}
71	92
72	93		// String implements Repo.String.
73	94		func (r *repoURL) String() string {
74		-	return fmt.Sprintf("%s-%s_%s", r.hostname, r.owner, r.projectID)
	95	+	return fmt.Sprintf("%s-%s_%s", r.host, r.owner, r.project)
75	96		}
76	97
77	98		func (r *repoURL) Org() clients.Repo {
78	99		return &repoURL{
79		-	hostname: r.hostname,
80		-	owner: r.owner,
81		-	projectID: gitlabOrgProj,
	100	+	host: r.host,
	101	+	owner: r.owner,
	102	+	project: gitlabOrgProj,
82	103		}
83	104		}
84	105
85	106		// IsValid implements Repo.IsValid.
86	107		func (r *repoURL) IsValid() error {
87		-	hostMatched, err := regexp.MatchString("gitlab.*com", r.hostname)
	108	+	if strings.Contains(r.host, "gitlab.") {
	109	+	return nil
	110	+	}
	111	+
	112	+	client, err := gitlab.NewClient("", gitlab.WithBaseURL(fmt.Sprintf("%s://%s", r.scheme, r.host)))
88	113		if err != nil {
89		-	return fmt.Errorf("error processing regex: %w", err)
	114	+	return sce.WithMessage(err,
	115	+	fmt.Sprintf("couldn't create gitlab client for %s", r.host),
	116	+	)
90	117		}
91		-	if !hostMatched {
92		-	return sce.WithMessage(sce.ErrorInvalidURL, "non gitlab repository found")
	118	+
	119	+	_, resp, err := client.Projects.ListProjects(&gitlab.ListProjectsOptions{})
	120	+	if resp == nil \|\| resp.StatusCode != 200 {
	121	+	return sce.WithMessage(sce.ErrRepoUnreachable,
	122	+	fmt.Sprintf("couldn't reach gitlab instance at %s", r.host),
	123	+	)
93	124		}
94		-
95		-	isNotDigit := func(c rune) bool { return c < '0' \|\| c > '9' }
96		-	b := strings.IndexFunc(r.projectID, isNotDigit) == -1
97		-	if !b {
98		-	return sce.WithMessage(sce.ErrorInvalidURL, "incorrect format for projectID")
	125	+	if err != nil {
	126	+	return sce.WithMessage(err,
	127	+	fmt.Sprintf("error when connecting to gitlab instance at %s", r.host),
	128	+	)
99	129		}
100	130
101		-	if strings.TrimSpace(r.owner) == "" \|\| strings.TrimSpace(r.projectID) == "" {
	131	+	if strings.TrimSpace(r.owner) == "" \|\| strings.TrimSpace(r.project) == "" {
102	132		return sce.WithMessage(sce.ErrorInvalidURL,
103	133		fmt.Sprintf("%v. Expected the full project url", r.URI()))
104	134		}
		skipped 25 lines

■ ■ ■ ■ ■ ■

clients/gitlabrepo/repo_test.go

		skipped 18 lines
19	19		"testing"
20	20
21	21		"github.com/google/go-cmp/cmp"
	22	+	"github.com/google/go-cmp/cmp/cmpopts"
22	23		)
23	24
24	25		func TestRepoURL_IsValid(t *testing.T) {
		skipped 5 lines
30	31		wantErr bool
31	32		}{
32	33		{
33		-	name: "valid http address",
	34	+	name: "github repository",
34	35		expected: repoURL{
35		-	hostname: "gitlab.example.com",
36		-	owner: "foo",
37		-	projectID: "1234",
	36	+	scheme: "https",
	37	+	host: "https://github.com",
	38	+	owner: "ossf",
	39	+	project: "scorecard",
38	40		},
39		-	inputURL: "http://gitlab.example.com/foo/1234",
40		-	wantErr: false,
	41	+	inputURL: "https://github.com/ossf/scorecard",
	42	+	wantErr: true,
41	43		},
42	44		{
43		-	name: "valid https address",
	45	+	name: "GitHub project with 'gitlab.' in the title",
44	46		expected: repoURL{
45		-	hostname: "gitlab.example.com",
46		-	owner: "foo",
47		-	projectID: "1234",
	47	+	scheme: "http",
	48	+	host: "http://github.com",
	49	+	owner: "foo",
	50	+	project: "gitlab.test",
48	51		},
49		-	inputURL: "https://gitlab.example.com/foo/1234",
50		-	wantErr: false,
	52	+	inputURL: "http://github.com/foo/gitlab.test",
	53	+	wantErr: true,
51	54		},
52	55		{
53		-	name: "valid http address with trailing slash",
	56	+	name: "valid gitlab project",
54	57		expected: repoURL{
55		-	hostname: "gitlab.example.com",
56		-	owner: "foo",
57		-	projectID: "1234",
	58	+	host: "https://gitlab.com",
	59	+	owner: "ossf-test",
	60	+	project: "scorecard-check-binary-artifacts-e2e",
58	61		},
59		-	inputURL: "http://gitlab.example.com/foo/1234/",
	62	+	inputURL: "https://gitlab.com/ossf-test/scorecard-check-binary-artifacts-e2e",
60	63		wantErr: false,
61	64		},
62	65		{
63	66		name: "valid https address with trailing slash",
64	67		expected: repoURL{
65		-	hostname: "gitlab.example.com",
66		-	owner: "foo",
67		-	projectID: "1234",
	68	+	scheme: "https",
	69	+	host: "https://gitlab.com",
	70	+	owner: "ossf-test",
	71	+	project: "scorecard-check-binary-artifacts-e2e",
68	72		},
69		-	inputURL: "https://gitlab.example.com/foo/1234/",
	73	+	inputURL: "https://gitlab.com/ossf-test/scorecard-check-binary-artifacts-e2e/",
70	74		wantErr: false,
71	75		},
72		-	{
73		-	name: "non gitlab repository",
74		-	expected: repoURL{
75		-	hostname: "github.com",
76		-	owner: "foo",
77		-	projectID: "1234",
78		-	},
79		-	inputURL: "https://github.com/foo/1234",
80		-	wantErr: true,
81		-	},
	76	+
82	77		{
83		-	name: "GitLab project with wrong projectID",
	78	+	name: "valid hosted gitlab project",
84	79		expected: repoURL{
85		-	hostname: "gitlab.example.com",
86		-	owner: "foo",
87		-	projectID: "bar",
	80	+	scheme: "https",
	81	+	host: "https://salsa.debian.org",
	82	+	owner: "webmaster-team",
	83	+	project: "webml",
88	84		},
89		-	inputURL: "https://gitlab.example.com/foo/bar",
90		-	wantErr: true,
91		-	},
92		-	{
93		-	name: "GitHub project with 'gitlab.' in the title",
94		-	expected: repoURL{
95		-	hostname: "github.com",
96		-	owner: "foo",
97		-	projectID: "gitlab.test",
98		-	},
99		-	inputURL: "http://github.com/foo/gitlab.test",
100		-	wantErr: true,
101		-	},
102		-	{
103		-	name: "valid gitlab project without http or https",
104		-	expected: repoURL{
105		-	hostname: "gitlab.example.com",
106		-	owner: "foo",
107		-	projectID: "1234",
108		-	},
109		-	inputURL: "gitlab.example.com/foo/1234",
	85	+	inputURL: "https://salsa.debian.org/webmaster-team/webwml",
110	86		wantErr: false,
111	87		},
112	88		}
		skipped 2 lines
115	91		t.Run(tt.name, func(t *testing.T) {
116	92		t.Parallel()
117	93		r := repoURL{
118		-	hostname: tt.expected.hostname,
119		-	owner: tt.expected.owner,
120		-	projectID: tt.expected.projectID,
	94	+	host: tt.expected.host,
	95	+	owner: tt.expected.owner,
	96	+	project: tt.expected.project,
121	97		}
122	98		if err := r.parse(tt.inputURL); err != nil {
123	99		t.Errorf("repoURL.parse() error = %v", err)
		skipped 1 lines
125	101		if err := r.IsValid(); (err != nil) != tt.wantErr {
126	102		t.Errorf("repoURL.IsValid() error = %v, wantErr %v", err, tt.wantErr)
127	103		}
128		-	if !tt.wantErr && !cmp.Equal(tt.expected, r, cmp.AllowUnexported(repoURL{})) {
129		-	fmt.Println("expected: " + tt.expected.hostname + " GOT: " + r.hostname)
	104	+	if !tt.wantErr && !cmp.Equal(tt.expected, r, cmpopts.IgnoreUnexported(repoURL{})) {
	105	+	fmt.Println("expected: " + tt.expected.host + " GOT: " + r.host)
130	106		fmt.Println("expected: " + tt.expected.owner + " GOT: " + r.owner)
131		-	fmt.Println("expected: " + tt.expected.projectID + " GOT: " + r.projectID)
	107	+	fmt.Println("expected: " + tt.expected.project + " GOT: " + r.project)
132	108		t.Errorf("Got diff: %s", cmp.Diff(tt.expected, r))
133	109		}
	110	+	if !cmp.Equal(r.Host(), tt.expected.host) {
	111	+	t.Errorf("%s expected host: %s got host %s", tt.inputURL, tt.expected.host, r.Host())
	112	+	}
134	113		})
135	114		}
136	115		}
137	116
	117	+	func TestRepoURL_DetectGitlab(t *testing.T) {
	118	+	tests := []struct {
	119	+	repouri string
	120	+	expected bool
	121	+	}{
	122	+	{
	123	+	repouri: "github.com/ossf/scorecard",
	124	+	expected: false,
	125	+	},
	126	+	{
	127	+	repouri: "ossf/scorecard",
	128	+	expected: false,
	129	+	},
	130	+	{
	131	+	repouri: "https://github.com/ossf/scorecard",
	132	+	expected: false,
	133	+	},
	134	+	{
	135	+	repouri: "gitlab.com/gitlab-org/gitlab",
	136	+	expected: true,
	137	+	},
	138	+	{
	139	+	repouri: "https://salsa.debian.org/webmaster-team/webml",
	140	+	expected: true,
	141	+	},
	142	+	{
	143	+	// Invalid repo
	144	+	repouri: "https://abcdef.foo.txt/nilproj/nilrepo",
	145	+	expected: false,
	146	+	},
	147	+	}
	148	+
	149	+	for _, tt := range tests {
	150	+	g := DetectGitLab(tt.repouri)
	151	+	if g != tt.expected {
	152	+	t.Errorf("got %s isgitlab: %t expected %t", tt.repouri, g, tt.expected)
	153	+	}
	154	+	}
	155	+	}
	156	+

■ ■ ■ ■ ■ ■

clients/gitlabrepo/search.go

		skipped 44 lines
45	45		return clients.SearchResponse{}, fmt.Errorf("handler.buildQuery: %w", err)
46	46		}
47	47
48		-	blobs, _, err := handler.glClient.Search.BlobsByProject(handler.repourl.projectID, query, &gitlab.SearchOptions{})
	48	+	blobs, _, err := handler.glClient.Search.BlobsByProject(handler.repourl.project, query, &gitlab.SearchOptions{})
49	49		if err != nil {
50	50		return clients.SearchResponse{}, fmt.Errorf("Search.BlobsByProject: %w", err)
51	51		}
		skipped 8 lines
60	60		if _, err := queryBuilder.WriteString(
61	61		fmt.Sprintf("%s project:%s/%s",
62	62		strings.ReplaceAll(request.Query, "/", " "),
63		-	handler.repourl.owner, handler.repourl.projectID)); err != nil {
	63	+	handler.repourl.owner, handler.repourl.project)); err != nil {
64	64		return "", fmt.Errorf("WriteString: %w", err)
65	65		}
66	66		if request.Filename != "" {
		skipped 29 lines

■ ■ ■ ■ ■ ■

clients/gitlabrepo/searchCommits.go

		skipped 40 lines
41	41		return nil, fmt.Errorf("handler.buildQuiery: %w", err)
42	42		}
43	43
44		-	commits, _, err := handler.glClient.Search.CommitsByProject(handler.repourl.projectID, query, &gitlab.SearchOptions{})
	44	+	commits, _, err := handler.glClient.Search.CommitsByProject(handler.repourl.project, query, &gitlab.SearchOptions{})
45	45		if err != nil {
46	46		return nil, fmt.Errorf("Search.Commits: %w", err)
47	47		}
		skipped 27 lines
75	75		var queryBuilder strings.Builder
76	76		if _, err := queryBuilder.WriteString(
77	77		fmt.Sprintf("project:%s/%s author:%s",
78		-	handler.repourl.owner, handler.repourl.projectID,
	78	+	handler.repourl.owner, handler.repourl.project,
79	79		request.Author)); err != nil {
80	80		return "", fmt.Errorf("writestring: %w", err)
81	81		}
		skipped 4 lines

■ ■ ■ ■ ■ ■

clients/gitlabrepo/searchCommits_test.go

		skipped 33 lines
34	34		{
35	35		name: "Basic",
36	36		repourl: &repoURL{
37		-	owner: "testowner",
38		-	projectID: "1234",
	37	+	owner: "testowner",
	38	+	project: "1234",
39	39		},
40	40		searchReq: clients.SearchCommitsOptions{
41	41		Author: "testAuthor",
		skipped 3 lines
45	45		{
46	46		name: "EmptyQuery:",
47	47		repourl: &repoURL{
48		-	owner: "testowner",
49		-	projectID: "1234",
	48	+	owner: "testowner",
	49	+	project: "1234",
50	50		},
51	51		searchReq: clients.SearchCommitsOptions{},
52	52		hasError: true,
		skipped 26 lines

■ ■ ■ ■ ■ ■

clients/gitlabrepo/search_test.go

		skipped 33 lines
34	34		{
35	35		name: "Basic",
36	36		repourl: &repoURL{
37		-	owner: "testowner",
38		-	projectID: "1234",
	37	+	owner: "testowner",
	38	+	project: "1234",
39	39		},
40	40		searchReq: clients.SearchRequest{
41	41		Query: "testquery",
		skipped 3 lines
45	45		{
46	46		name: "EmptyQuery",
47	47		repourl: &repoURL{
48		-	owner: "testowner",
49		-	projectID: "1234",
	48	+	owner: "testowner",
	49	+	project: "1234",
50	50		},
51	51		searchReq: clients.SearchRequest{},
52	52		hasError: true,
		skipped 2 lines
55	55		{
56	56		name: "WithFilename",
57	57		repourl: &repoURL{
58		-	owner: "testowner",
59		-	projectID: "1234",
	58	+	owner: "testowner",
	59	+	project: "1234",
60	60		},
61	61		searchReq: clients.SearchRequest{
62	62		Query: "testquery",
		skipped 4 lines
67	67		{
68	68		name: "WithPath",
69	69		repourl: &repoURL{
70		-	owner: "testowner",
71		-	projectID: "1234",
	70	+	owner: "testowner",
	71	+	project: "1234",
72	72		},
73	73		searchReq: clients.SearchRequest{
74	74		Query: "testquery",
		skipped 4 lines
79	79		{
80	80		name: "WithFilenameAndPath",
81	81		repourl: &repoURL{
82		-	owner: "testowner",
83		-	projectID: "1234",
	82	+	owner: "testowner",
	83	+	project: "1234",
84	84		},
85	85		searchReq: clients.SearchRequest{
86	86		Query: "testquery",
		skipped 5 lines
92	92		{
93	93		name: "WithFilenameAndPathWithSeperator",
94	94		repourl: &repoURL{
95		-	owner: "testowner",
96		-	projectID: "1234",
	95	+	owner: "testowner",
	96	+	project: "1234",
97	97		},
98	98		searchReq: clients.SearchRequest{
99	99		Query: "testquery/query",
		skipped 29 lines

■ ■ ■ ■ ■ ■

clients/gitlabrepo/statuses.go

		skipped 33 lines
34	34		// for gitlab this only works if ref is SHA.
35	35		func (handler *statusesHandler) listStatuses(ref string) ([]clients.Status, error) {
36	36		commitStatuses, _, err := handler.glClient.Commits.GetCommitStatuses(
37		-	handler.repourl.projectID, ref, &gitlab.GetCommitStatusesOptions{})
	37	+	handler.repourl.project, ref, &gitlab.GetCommitStatusesOptions{})
38	38		if err != nil {
39	39		return nil, fmt.Errorf("error getting commit statuses: %w", err)
40	40		}
		skipped 16 lines

■ ■ ■ ■ ■ ■

clients/gitlabrepo/webhook.go

		skipped 39 lines
40	40		func (handler *webhookHandler) setup() error {
41	41		handler.once.Do(func() {
42	42		projectHooks, _, err := handler.glClient.Projects.ListProjectHooks(
43		-	handler.repourl.projectID, &gitlab.ListProjectHooksOptions{})
	43	+	handler.repourl.project, &gitlab.ListProjectHooksOptions{})
44	44		if err != nil {
45	45		handler.errSetup = fmt.Errorf("request for project hooks failed with %w", err)
46	46		return
		skipped 24 lines

■ ■ ■ ■ ■ ■

clients/gitlabrepo/workflows.go

		skipped 34 lines
35	35		func (handler *workflowsHandler) listSuccessfulWorkflowRuns(filename string) ([]clients.WorkflowRun, error) {
36	36		var buildStates []gitlab.BuildStateValue
37	37		buildStates = append(buildStates, gitlab.Success)
38		-	jobs, _, err := handler.glClient.Jobs.ListProjectJobs(handler.repourl.projectID,
	38	+	jobs, _, err := handler.glClient.Jobs.ListProjectJobs(handler.repourl.project,
39	39		&gitlab.ListJobsOptions{Scope: &buildStates})
40	40		if err != nil {
41	41		return nil, fmt.Errorf("error getting project jobs: %w", err)
		skipped 22 lines

■ ■ ■ ■ ■ ■

clients/localdir/repo.go

		skipped 36 lines
37	37		return fmt.Sprintf("file://%s", r.path)
38	38		}
39	39
	40	+	func (r *repoLocal) Host() string {
	41	+	return ""
	42	+	}
	43	+
40	44		// String implements Repo.String.
41	45		func (r *repoLocal) String() string {
42	46		return r.URI()
		skipped 43 lines

■ ■ ■ ■ ■ ■

clients/mockclients/repo.go

		skipped 128 lines
129	129		return ret0
130	130		}
131	131
	132	+	// URI mocks base method.
	133	+	func (m *MockRepo) Host() string {
	134	+	m.ctrl.T.Helper()
	135	+	ret := m.ctrl.Call(m, "Host")
	136	+	ret0, _ := ret[0].(string)
	137	+	return ret0
	138	+	}
	139	+
132	140		// URI indicates an expected call of URI.
133	141		func (mr MockRepoMockRecorder) URI() gomock.Call {
134	142		mr.mock.ctrl.T.Helper()
		skipped 3 lines

■ ■ ■ ■ ■ ■

clients/repo.go

		skipped 16 lines
17	17		// Repo interface uniquely identifies a repo.
18	18		type Repo interface {
19	19		URI() string
	20	+	Host() string
20	21		String() string
21	22		Org() Repo
22	23		IsValid() error
		skipped 4 lines

■ ■ ■ ■ ■ ■

e2e/attestor_policy_test.go

		skipped 14 lines
15	15		package e2e
16	16
17	17		import (
18		-	"fmt"
19	18		"os"
20	19		"strings"
21	20
		skipped 163 lines
185	184		}
186	185
187	186		for _, tc := range tt {
188		-	fmt.Printf("attestor_policy_test.go: %s\n", tc.name)
189	187		f, err := os.CreateTemp("/tmp", strings.ReplaceAll(tc.name, " ", "-"))
190	188		Expect(err).Should(BeNil())
191	189		defer os.Remove(f.Name())
		skipped 16 lines