Added GitLab OAuth Detector (#4729)

* added gitlab oauth detector

* addressed comments; embed detectors.DefaultMultiPartCredentialProvider and use available ctx in tests

* made comments more meaningful and break loop when secret is verified against one client id

* switch from detectorspb to detector_typepb

* populate secretparts

* add secretparts to results in gitlaboauth2 detector

---------

Co-authored-by: Charlie Gunyon <camgunz@users.noreply.github.com>
This commit is contained in:
Shahzad Haider
2026-05-13 18:06:54 +05:00
committed by GitHub
parent 0c381f12b3
commit ef9a56c33b
6 changed files with 503 additions and 6 deletions
+166
View File
@@ -0,0 +1,166 @@
package gitlaboauth2
import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"strings"
regexp "github.com/wasilibs/go-re2"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb"
)
type Scanner struct {
client *http.Client
detectors.EndpointSetter
detectors.DefaultMultiPartCredentialProvider
}
func (Scanner) CloudEndpoint() string { return "https://gitlab.com" }
var (
// Ensure the Scanner satisfies the interfaces at compile time.
_ detectors.Detector = (*Scanner)(nil)
_ detectors.EndpointCustomizer = (*Scanner)(nil)
_ detectors.MultiPartCredentialProvider = (*Scanner)(nil)
defaultClient = common.SaneHttpClient()
clientIdPat = regexp.MustCompile(
detectors.PrefixRegex([]string{"application_id", "client_id", "app_id", "id"}) + `\b([0-9a-f]{64})\b`)
clientSecretPat = regexp.MustCompile(`\b(gloas-[0-9a-f]{64})\b`)
)
func (s Scanner) getClient() *http.Client {
if s.client != nil {
return s.client
}
return defaultClient
}
// Keywords are used for efficiently pre-filtering chunks.
func (s Scanner) Keywords() []string {
return []string{"gloas-"}
}
// FromData will find and optionally verify GitLab OAuth secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
uniqueIdMatches := make(map[string]struct{})
for _, match := range clientIdPat.FindAllStringSubmatch(dataStr, -1) {
uniqueIdMatches[match[1]] = struct{}{}
}
uniqueSecretMatches := make(map[string]struct{})
for _, match := range clientSecretPat.FindAllStringSubmatch(dataStr, -1) {
uniqueSecretMatches[match[1]] = struct{}{}
}
for clientId := range uniqueIdMatches {
secretLoop:
for clientSecret := range uniqueSecretMatches {
for _, endpoint := range s.Endpoints() {
s1 := detectors.Result{
DetectorType: s.Type(),
Raw: []byte(clientSecret),
RawV2: []byte(clientId + clientSecret + endpoint),
SecretParts: map[string]string{
"client_id": clientId,
"client_secret": clientSecret,
"endpoint": endpoint,
},
}
if verify {
isVerified, verificationErr := verifyMatch(
ctx, s.getClient(), endpoint, clientId, clientSecret,
)
s1.Verified = isVerified
s1.SetVerificationError(verificationErr)
if s1.Verified {
// A client_id is bound to a single secret; skip remaining
// endpoints and secrets for this client_id.
results = append(results, s1)
break secretLoop
}
}
results = append(results, s1)
}
}
}
return
}
func verifyMatch(ctx context.Context, client *http.Client, endpoint string, clientId string, clientSecret string) (bool, error) {
url := endpoint + "/oauth/token"
payload := strings.NewReader("grant_type=client_credentials&client_id=" + clientId +
"&client_secret=" + clientSecret)
req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, payload)
if err != nil {
return false, err
}
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
res, err := client.Do(req)
if err != nil {
return false, err
}
defer func() {
_, _ = io.Copy(io.Discard, res.Body)
_ = res.Body.Close()
}()
// We use grant_type=client_credentials which GitLab doesn't support for OAuth apps,
// so even valid credentials never return 200. Instead, GitLab validates credentials
// before checking grant type:
//
// - 400 with "invalid_scope": credentials are valid (grant type rejected after auth passed)
// - 401 with "invalid_client": credentials are invalid
//
// Any other status (e.g. 422 for apps with `api` scope) is treated as inconclusive
// since we cannot determine credential validity, and falls through to the default case.
switch res.StatusCode {
case http.StatusBadRequest:
bodyBytes, err := io.ReadAll(res.Body)
if err != nil {
return false, err
}
var errResp struct {
Error string `json:"error"`
}
if err := json.Unmarshal(bodyBytes, &errResp); err != nil {
return false, err
}
if errResp.Error == "invalid_scope" {
return true, nil
}
return false, fmt.Errorf("unexpected error in response: %s", errResp.Error)
case http.StatusUnauthorized:
return false, nil
default:
return false, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
}
}
func (s Scanner) Type() detector_typepb.DetectorType {
return detector_typepb.DetectorType_GitLabOauth2
}
func (s Scanner) Description() string {
return "GitLab is a web-based DevOps lifecycle tool that provides a Git repository manager providing wiki, issue-tracking, and CI/CD pipeline features. GitLab OAuth application credentials can be used to access GitLab APIs on behalf of users."
}
@@ -0,0 +1,185 @@
//go:build detectors
// +build detectors
package gitlaboauth2
import (
"context"
"fmt"
"testing"
"time"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb"
)
func TestGitlabOauth_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors6")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
clientId := testSecrets.MustGetField("GITLABOAUTH_CLIENT_ID")
clientSecret := testSecrets.MustGetField("GITLABOAUTH_CLIENT_SECRET")
inactiveClientId := testSecrets.MustGetField("GITLABOAUTH_CLIENT_ID_INACTIVE")
inactiveClientSecret := testSecrets.MustGetField("GITLABOAUTH_CLIENT_SECRET_INACTIVE")
type args struct {
ctx context.Context
data []byte
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
wantVerificationErr bool
}{
{
name: "found, verified",
s: Scanner{},
args: args{
ctx: ctx,
data: []byte(fmt.Sprintf(`
gitlab:
client_id: %s
client_secret: %s
`, clientId, clientSecret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detector_typepb.DetectorType_GitLabOauth2,
Verified: true,
},
},
wantErr: false,
wantVerificationErr: false,
},
{
name: "found, unverified",
s: Scanner{},
args: args{
ctx: ctx,
data: []byte(fmt.Sprintf(`
gitlab:
client_id: %s
client_secret: %s
`, inactiveClientId, inactiveClientSecret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detector_typepb.DetectorType_GitLabOauth2,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: false,
},
{
name: "not found",
s: Scanner{},
args: args{
ctx: ctx,
data: []byte("You cannot find the secret within"),
verify: true,
},
want: nil,
wantErr: false,
wantVerificationErr: false,
},
{
name: "found, would be verified if not for timeout",
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
args: args{
ctx: ctx,
data: []byte(fmt.Sprintf(`
gitlab:
client_id: %s
client_secret: %s
`, clientId, clientSecret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detector_typepb.DetectorType_GitLabOauth2,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: true,
},
{
name: "found, verified but unexpected api surface",
s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
args: args{
ctx: ctx,
data: []byte(fmt.Sprintf(`
gitlab:
client_id: %s
client_secret: %s
`, clientId, clientSecret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detector_typepb.DetectorType_GitLabOauth2,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
tt.s.SetCloudEndpoint(tt.s.CloudEndpoint())
tt.s.UseCloudEndpoint(true)
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("GitlabOauth.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
}
for i := range got {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
}
}
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{},
"Raw", "RawV2", "verificationError", "ExtraData", "primarySecret", "SecretParts")
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
t.Errorf("GitlabOauth.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
}
func BenchmarkFromData(benchmark *testing.B) {
ctx := context.Background()
s := Scanner{}
s.SetCloudEndpoint(s.CloudEndpoint())
s.UseCloudEndpoint(true)
for name, data := range detectors.MustGetBenchmarkData() {
benchmark.Run(name, func(b *testing.B) {
b.ResetTimer()
for n := 0; n < b.N; n++ {
_, err := s.FromData(ctx, false, data)
if err != nil {
b.Fatal(err)
}
}
})
}
}
@@ -0,0 +1,138 @@
package gitlaboauth2
import (
"context"
"testing"
"github.com/google/go-cmp/cmp"
"github.com/stretchr/testify/require"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
func TestGitlabOauth_Pattern(t *testing.T) {
d := Scanner{}
d.SetCloudEndpoint("https://gitlab.com")
d.UseCloudEndpoint(true)
ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
name string
input string
want []string
}{
{
name: "valid pattern - client_id and client_secret",
input: `
gitlab:
client_id: 04fcf956cb6c5f4b106f3a4eca76eaf70e8c5d07a976ecf3baff9ac778a0098a
client_secret: gloas-8406980541370e5bd4f04c5da232c2cdabe7fa3959eb1757eeef7299e2458216
`,
want: []string{"04fcf956cb6c5f4b106f3a4eca76eaf70e8c5d07a976ecf3baff9ac778a0098agloas-8406980541370e5bd4f04c5da232c2cdabe7fa3959eb1757eeef7299e2458216https://gitlab.com"},
},
{
name: "valid pattern - application_id prefix",
input: `
GITLAB_APPLICATION_ID=99f0b46fc9241b7fa4b7d567044fab74a5f00ac0f12244ccfed1ea67d4a975df
GITLAB_SECRET=gloas-35fa9094e834aafb153bc17f1a31f48071af915c2ccf2f890b6714b954896321
`,
want: []string{"99f0b46fc9241b7fa4b7d567044fab74a5f00ac0f12244ccfed1ea67d4a975dfgloas-35fa9094e834aafb153bc17f1a31f48071af915c2ccf2f890b6714b954896321https://gitlab.com"},
},
{
name: "valid pattern - JSON format",
input: `
{
"app_id": "763c4e64f4c40dd070010617639cc11e37bbaf1a798503dd96ee5e6852754862",
"secret": "gloas-1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef"
}
`,
want: []string{"763c4e64f4c40dd070010617639cc11e37bbaf1a798503dd96ee5e6852754862gloas-1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdefhttps://gitlab.com"},
},
{
name: "multiple matches",
input: `
# Production
client_id: 04fcf956cb6c5f4b106f3a4eca76eaf70e8c5d07a976ecf3baff9ac778a0098a
client_secret: gloas-8406980541370e5bd4f04c5da232c2cdabe7fa3959eb1757eeef7299e2458216
# Staging
client_id: 99f0b46fc9241b7fa4b7d567044fab74a5f00ac0f12244ccfed1ea67d4a975df
client_secret: gloas-35fa9094e834aafb153bc17f1a31f48071af915c2ccf2f890b6714b954896321
`,
want: []string{
"04fcf956cb6c5f4b106f3a4eca76eaf70e8c5d07a976ecf3baff9ac778a0098agloas-8406980541370e5bd4f04c5da232c2cdabe7fa3959eb1757eeef7299e2458216https://gitlab.com",
"04fcf956cb6c5f4b106f3a4eca76eaf70e8c5d07a976ecf3baff9ac778a0098agloas-35fa9094e834aafb153bc17f1a31f48071af915c2ccf2f890b6714b954896321https://gitlab.com",
"99f0b46fc9241b7fa4b7d567044fab74a5f00ac0f12244ccfed1ea67d4a975dfgloas-8406980541370e5bd4f04c5da232c2cdabe7fa3959eb1757eeef7299e2458216https://gitlab.com",
"99f0b46fc9241b7fa4b7d567044fab74a5f00ac0f12244ccfed1ea67d4a975dfgloas-35fa9094e834aafb153bc17f1a31f48071af915c2ccf2f890b6714b954896321https://gitlab.com",
},
},
{
name: "invalid pattern - wrong secret prefix",
input: `
client_id: 04fcf956cb6c5f4b106f3a4eca76eaf70e8c5d07a976ecf3baff9ac778a0098a
client_secret: glpat-8406980541370e5bd4f04c5da232c2cdabe7fa3959eb1757eeef7299e2458216
`,
want: []string{},
},
{
name: "invalid pattern - secret too short",
input: `
client_id: 04fcf956cb6c5f4b106f3a4eca76eaf70e8c5d07a976ecf3baff9ac778a0098a
client_secret: gloas-8406980541370e5bd4f04c5da232c2cd
`,
want: []string{},
},
{
name: "invalid pattern - client_id too short",
input: `
client_id: 04fcf956cb6c5f4b106f3a4eca76eaf7
client_secret: gloas-8406980541370e5bd4f04c5da232c2cdabe7fa3959eb1757eeef7299e2458216
`,
want: []string{},
},
{
name: "invalid pattern - no client_id context prefix",
input: `
04fcf956cb6c5f4b106f3a4eca76eaf70e8c5d07a976ecf3baff9ac778a0098a
gloas-8406980541370e5bd4f04c5da232c2cdabe7fa3959eb1757eeef7299e2458216
`,
want: []string{},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
if len(test.want) > 0 && len(matchedDetectors) == 0 {
t.Errorf("keywords were not matched: %v", d.Keywords())
return
}
results, err := d.FromData(context.Background(), false, []byte(test.input))
require.NoError(t, err)
if len(results) != len(test.want) {
t.Errorf("mismatch in result count: expected %d, got %d", len(test.want), len(results))
return
}
actual := make(map[string]struct{}, len(results))
for _, r := range results {
if len(r.RawV2) > 0 {
actual[string(r.RawV2)] = struct{}{}
} else {
actual[string(r.Raw)] = struct{}{}
}
}
expected := make(map[string]struct{}, len(test.want))
for _, v := range test.want {
expected[v] = struct{}{}
}
if diff := cmp.Diff(expected, actual); diff != "" {
t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
}
})
}
}
+2
View File
@@ -335,6 +335,7 @@ import (
gitlabv1 "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/gitlab/v1"
gitlabv2 "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/gitlab/v2"
gitlabv3 "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/gitlab/v3"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/gitlaboauth2"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/gitter"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/glassnode"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/gocanvas"
@@ -1215,6 +1216,7 @@ func buildDetectorList() []detectors.Detector {
&gitlabv1.Scanner{},
&gitlabv2.Scanner{},
&gitlabv3.Scanner{},
&gitlaboauth2.Scanner{},
&gitter.Scanner{},
&glassnode.Scanner{},
&gocanvas.Scanner{},
+11 -6
View File
@@ -1103,6 +1103,7 @@ const (
DetectorType_ConfluenceDataCenter DetectorType = 1047
DetectorType_Cloudinary DetectorType = 1048
DetectorType_Pinecone DetectorType = 1049
DetectorType_GitLabOauth2 DetectorType = 1050
)
// Enum value maps for DetectorType.
@@ -2154,6 +2155,7 @@ var (
1047: "ConfluenceDataCenter",
1048: "Cloudinary",
1049: "Pinecone",
1050: "GitLabOauth2",
}
DetectorType_value = map[string]int32{
"Alibaba": 0,
@@ -3202,6 +3204,7 @@ var (
"ConfluenceDataCenter": 1047,
"Cloudinary": 1048,
"Pinecone": 1049,
"GitLabOauth2": 1050,
}
)
@@ -3237,7 +3240,7 @@ var File_detector_type_proto protoreflect.FileDescriptor
var file_detector_type_proto_rawDesc = []byte{
0x0a, 0x13, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x2e,
0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0d, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x5f,
0x74, 0x79, 0x70, 0x65, 0x2a, 0xac, 0x88, 0x01, 0x0a, 0x0c, 0x44, 0x65, 0x74, 0x65, 0x63, 0x74,
0x74, 0x79, 0x70, 0x65, 0x2a, 0xbf, 0x88, 0x01, 0x0a, 0x0c, 0x44, 0x65, 0x74, 0x65, 0x63, 0x74,
0x6f, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x41, 0x6c, 0x69, 0x62, 0x61, 0x62,
0x61, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x41, 0x4d, 0x51, 0x50, 0x10, 0x01, 0x12, 0x07, 0x0a,
0x03, 0x41, 0x57, 0x53, 0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x10,
@@ -4328,11 +4331,13 @@ var file_detector_type_proto_rawDesc = []byte{
0x66, 0x6c, 0x75, 0x65, 0x6e, 0x63, 0x65, 0x44, 0x61, 0x74, 0x61, 0x43, 0x65, 0x6e, 0x74, 0x65,
0x72, 0x10, 0x97, 0x08, 0x12, 0x0f, 0x0a, 0x0a, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x69, 0x6e, 0x61,
0x72, 0x79, 0x10, 0x98, 0x08, 0x12, 0x0d, 0x0a, 0x08, 0x50, 0x69, 0x6e, 0x65, 0x63, 0x6f, 0x6e,
0x65, 0x10, 0x99, 0x08, 0x42, 0x41, 0x5a, 0x3f, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63,
0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69,
0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33,
0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72,
0x5f, 0x74, 0x79, 0x70, 0x65, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
0x65, 0x10, 0x99, 0x08, 0x12, 0x11, 0x0a, 0x0c, 0x47, 0x69, 0x74, 0x4c, 0x61, 0x62, 0x4f, 0x61,
0x75, 0x74, 0x68, 0x32, 0x10, 0x9a, 0x08, 0x42, 0x41, 0x5a, 0x3f, 0x67, 0x69, 0x74, 0x68, 0x75,
0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63,
0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67,
0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64, 0x65, 0x74, 0x65, 0x63,
0x74, 0x6f, 0x72, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74,
0x6f, 0x33,
}
var (
+1
View File
@@ -1051,4 +1051,5 @@ enum DetectorType {
ConfluenceDataCenter = 1047;
Cloudinary = 1048;
Pinecone = 1049;
GitLabOauth2 = 1050;
}