package github_experimental import ( "fmt" "strings" "github.com/go-logr/logr" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/anypb" "github.com/trufflesecurity/trufflehog/v3/pkg/context" "github.com/trufflesecurity/trufflehog/v3/pkg/giturl" "github.com/trufflesecurity/trufflehog/v3/pkg/log" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb" "github.com/trufflesecurity/trufflehog/v3/pkg/sanitizer" "github.com/trufflesecurity/trufflehog/v3/pkg/sources" "github.com/trufflesecurity/trufflehog/v3/pkg/sources/git" githubsource "github.com/trufflesecurity/trufflehog/v3/pkg/sources/github" ) const ( SourceType = sourcespb.SourceType_SOURCE_TYPE_GITHUB_EXPERIMENTAL cloudV3Endpoint = "https://api.github.com" ) type Source struct { name string sourceID sources.SourceID jobID sources.JobID verify bool repoInfoCache repoInfoCache useCustomContentWriter bool git *git.Git scanOptions *git.ScanOptions log logr.Logger conn *sourcespb.GitHubExperimental connector githubsource.Connector sources.Progress sources.CommonSourceUnitUnmarshaller } // WithCustomContentWriter sets the useCustomContentWriter flag on the source. func (s *Source) WithCustomContentWriter() { s.useCustomContentWriter = true } func (s *Source) WithScanOptions(scanOptions *git.ScanOptions) { s.scanOptions = scanOptions } // Ensure the Source satisfies the interfaces at compile time var _ sources.Source = (*Source)(nil) var _ sources.SourceUnitUnmarshaller = (*Source)(nil) // Type returns the type of source. // It is used for matching source types in configuration and job input. func (s *Source) Type() sourcespb.SourceType { return SourceType } func (s *Source) SourceID() sources.SourceID { return s.sourceID } func (s *Source) JobID() sources.JobID { return s.jobID } // Init returns an initialized GitHubExperimental source. func (s *Source) Init(aCtx context.Context, name string, jobID sources.JobID, sourceID sources.SourceID, verify bool, connection *anypb.Any, concurrency int) error { err := git.CmdCheck() if err != nil { return err } s.log = aCtx.Logger() s.name = name s.sourceID = sourceID s.jobID = jobID s.verify = verify var conn sourcespb.GitHubExperimental err = anypb.UnmarshalTo(connection, &conn, proto.UnmarshalOptions{}) if err != nil { return fmt.Errorf("error unmarshalling connection: %w", err) } s.conn = &conn s.conn.Repository, err = s.normalizeRepo(s.conn.Repository) if err != nil { return fmt.Errorf("error normalizing repo: %w", err) } // Get the token from the connection token := s.conn.GetToken() if token == "" { return fmt.Errorf("token is required for GitHub Experimental source") } // Redact token from logs for security log.RedactGlobally(token) // Create authenticated connector using the TokenConnector pattern connector, err := githubsource.NewTokenConnector( aCtx, cloudV3Endpoint, // API endpoint token, // GitHub token "", // clonePath (empty for default) true, // authInUrl func(ctx context.Context, err error) bool { // Simple rate limit handler - can be enhanced later return false }, ) if err != nil { return fmt.Errorf("could not create GitHub connector: %w", err) } s.connector = connector s.repoInfoCache = newRepoInfoCache() cfg := &git.Config{ SourceName: s.name, JobID: s.jobID, SourceID: s.sourceID, SourceType: s.Type(), Verify: s.verify, SkipBinaries: false, SkipArchives: false, Concurrency: concurrency, SourceMetadataFunc: func(info git.SourceMetadataInfo) *source_metadatapb.MetaData { return &source_metadatapb.MetaData{ Data: &source_metadatapb.MetaData_Github{ Github: &source_metadatapb.Github{ Commit: sanitizer.UTF8(info.Commit), File: sanitizer.UTF8(info.File), Email: sanitizer.UTF8(info.Email), Repository: sanitizer.UTF8(info.Repository), Link: giturl.GenerateLink(info.Repository, info.Commit, info.File, info.Line), Timestamp: sanitizer.UTF8(info.Timestamp), Line: info.Line, Visibility: s.visibilityOf(aCtx, info.Repository), }, }, } }, UseCustomContentWriter: s.useCustomContentWriter, } s.git = git.NewGit(cfg) return nil } func (s *Source) visibilityOf(ctx context.Context, repoURL string) source_metadatapb.Visibility { // It isn't possible to get the visibility of a wiki. // We must use the visibility of the corresponding repository. if strings.HasSuffix(repoURL, ".wiki.git") { repoURL = strings.TrimSuffix(repoURL, ".wiki.git") + ".git" } repoInfo, ok := s.repoInfoCache.get(repoURL) if !ok { // This should never happen. err := fmt.Errorf("no repoInfo for URL: %s", repoURL) ctx.Logger().Error(err, "failed to get repository visibility") return source_metadatapb.Visibility_unknown } return repoInfo.visibility } // Chunks emits chunks of bytes over a channel. func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, targets ...sources.ChunkingTarget) error { if s.conn.ObjectDiscovery { err := s.EnumerateAndScanAllObjects(ctx, chunksChan) return err } return nil } func getRepoURLParts(repoURLString string) (string, []string, error) { // Support ssh and https URLs. repoURL, err := git.GitURLParse(repoURLString) if err != nil { return "", nil, err } // Remove the user information. // e.g., `git@github.com` -> `github.com` if repoURL.User != nil { repoURL.User = nil } urlString := repoURL.String() trimmedURL := strings.TrimPrefix(urlString, repoURL.Scheme+"://") trimmedURL = strings.TrimSuffix(trimmedURL, ".git") urlParts := strings.Split(trimmedURL, "/") // Validate switch len(urlParts) { case 2: // gist.github.com/ if !strings.EqualFold(urlParts[0], "gist.github.com") { err = fmt.Errorf("failed to parse repository or gist URL (%s): 2 path segments are only expected if the host is 'gist.github.com' ('gist.github.com', '')", urlString) } case 3: // github.com//repo> // gist.github.com// // github.company.org//repo> // github.company.org/gist/ case 4: // github.company.org/gist/ if !strings.EqualFold(urlParts[1], "gist") || (strings.EqualFold(urlParts[0], "github.com") && strings.EqualFold(urlParts[1], "gist")) { err = fmt.Errorf("failed to parse repository or gist URL (%s): 4 path segments are only expected if the host isn't 'github.com' and the path starts with 'gist' ('github.example.com', 'gist', '', '')", urlString) } default: err = fmt.Errorf("invalid repository or gist URL (%s): length of URL segments should be between 2 and 4, not %d (%v)", urlString, len(urlParts), urlParts) } if err != nil { return "", nil, err } return urlString, urlParts, nil }