package docker import ( "archive/tar" "errors" "fmt" "io" "net" "net/http" "strings" "time" "github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/daemon" "github.com/google/go-containerregistry/pkg/v1/remote" "github.com/google/go-containerregistry/pkg/v1/tarball" gzip "github.com/klauspost/pgzip" "golang.org/x/sync/errgroup" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/anypb" "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/common/glob" "github.com/trufflesecurity/trufflehog/v3/pkg/context" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb" "github.com/trufflesecurity/trufflehog/v3/pkg/sources" ) const SourceType = sourcespb.SourceType_SOURCE_TYPE_DOCKER type Source struct { name string sourceId sources.SourceID jobId sources.JobID verify bool concurrency int conn sourcespb.Docker globFilter *glob.Filter // Filter for excluding files based on glob patterns sources.Progress sources.CommonSourceUnitUnmarshaller } // Ensure the Source satisfies the interfaces at compile time. var _ sources.Source = (*Source)(nil) var _ sources.SourceUnitUnmarshaller = (*Source)(nil) // Type returns the type of source. // It is used for matching source types in configuration and job input. func (s *Source) Type() sourcespb.SourceType { return SourceType } func (s *Source) SourceID() sources.SourceID { return s.sourceId } func (s *Source) JobID() sources.JobID { return s.jobId } // Init initializes the source. func (s *Source) Init(ctx context.Context, name string, jobId sources.JobID, sourceId sources.SourceID, verify bool, connection *anypb.Any, concurrency int) error { s.name = name s.sourceId = sourceId s.jobId = jobId s.verify = verify s.concurrency = concurrency jobIDStr := fmt.Sprint(s.jobId) // Reset metrics for this source at initialization time. dockerImagesScanned.WithLabelValues(s.name, jobIDStr).Set(0) dockerLayersScanned.WithLabelValues(s.name, jobIDStr).Set(0) dockerLayersEnumerated.WithLabelValues(s.name, jobIDStr).Set(0) dockerHistoryEntriesEnumerated.WithLabelValues(s.name, jobIDStr).Set(0) dockerImagesEnumerated.WithLabelValues(s.name, jobIDStr).Set(0) dockerHistoryEntriesScanned.WithLabelValues(s.name, jobIDStr).Set(0) if err := anypb.UnmarshalTo(connection, &s.conn, proto.UnmarshalOptions{}); err != nil { return fmt.Errorf("error unmarshalling connection: %w", err) } // Extract exclude paths from connection and compile glob patterns if paths := s.conn.GetExcludePaths(); len(paths) > 0 { var err error s.globFilter, err = glob.NewGlobFilter(glob.WithExcludeGlobs(paths...)) if err != nil { return fmt.Errorf("error creating glob filter for exclude paths: %w", err) } } return nil } // imageInfo holds information about a Docker image being processed type imageInfo struct { image v1.Image // The image object from go-containerregistry base string // Base name of the image (without tag/digest) tag string // Tag or digest of the image } // historyEntryInfo represents a single entry from the image's build history type historyEntryInfo struct { index int // Position in the history array entry v1.History // The history entry containing build commands layerDigest string // SHA256 digest of the associated layer (empty for empty layers) base string // Base name of the image tag string // Tag or digest of the image } // layerInfo contains identifying information for a Docker image layer type layerInfo struct { digest v1.Hash // SHA256 digest uniquely identifying the layer base string // Base name of the image tag string // Tag or digest of the image } // Chunks emits data over a channel that is decoded and scanned for secrets. func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error { ctx = context.WithValues(ctx, "source_type", s.Type(), "source_name", s.name) jobIDStr := fmt.Sprint(s.jobId) workers := new(errgroup.Group) workers.SetLimit(s.concurrency) // if namespace is set and no images are specified, fetch all images in that namespace. registryNamespace := s.conn.GetNamespace() if registryNamespace != "" && len(s.conn.Images) == 0 { start := time.Now() namespaceImages, err := GetNamespaceImages(ctx, registryNamespace, s.conn.GetRegistryToken()) if err != nil { return fmt.Errorf("failed to list namespace: %s images: %w", registryNamespace, err) } dockerListImagesAPIDuration.WithLabelValues(s.name).Observe(time.Since(start).Seconds()) s.conn.Images = append(s.conn.Images, namespaceImages...) } for _, image := range s.conn.GetImages() { if common.IsDone(ctx) { return nil } imgInfo, err := s.processImage(ctx, image) if err != nil { ctx.Logger().Error(err, "error processing image", "image", image) continue } imageCtx := context.WithValues(ctx, "image", imgInfo.base, "tag", imgInfo.tag) imageCtx.Logger().V(2).Info("scanning image history") layers, err := imgInfo.image.Layers() if err != nil { imageCtx.Logger().Error(err, "error getting image layers") continue } dockerLayersEnumerated.WithLabelValues(s.name, jobIDStr).Add(float64(len(layers))) // Get history entries and associate them with layers historyEntries, err := getHistoryEntries(imageCtx, imgInfo, layers) if err != nil { imageCtx.Logger().Error(err, "error getting image history entries") continue } dockerHistoryEntriesEnumerated.WithLabelValues(s.name, jobIDStr).Add(float64(len(historyEntries))) // Scan each history entry for secrets in build commands for _, historyEntry := range historyEntries { if err := s.processHistoryEntry(imageCtx, historyEntry, chunksChan); err != nil { imageCtx.Logger().Error(err, "error processing history entry") continue } dockerHistoryEntriesScanned.WithLabelValues(s.name, jobIDStr).Inc() } imageCtx.Logger().V(2).Info("scanning image layers") // Process each layer concurrently for _, layer := range layers { workers.Go(func() error { if err := s.processLayer(imageCtx, layer, imgInfo, chunksChan); err != nil { imageCtx.Logger().Error(err, "error processing layer") return nil } dockerLayersScanned.WithLabelValues(s.name, jobIDStr).Inc() return nil }) } if err := workers.Wait(); err != nil { imageCtx.Logger().Error(err, "error processing layers") continue } dockerImagesScanned.WithLabelValues(s.name, jobIDStr).Inc() } return nil } // processImage processes an individual image and prepares it for further processing. // It handles three image source types: remote registry, local daemon, and tarball file. func (s *Source) processImage(ctx context.Context, image string) (imageInfo, error) { ctx.Logger().V(5).Info("Processing individual Image") var ( imgInfo imageInfo imageName name.Reference err error ) remoteOpts, err := s.remoteOpts() if err != nil { return imgInfo, err } const filePrefix = "file://" const dockerPrefix = "docker://" // Handle tarball file images if image, ok := strings.CutPrefix(image, filePrefix); ok { imgInfo.base = image imgInfo.image, err = tarball.ImageFromPath(image, nil) if err != nil { return imgInfo, err } // Handle local Docker daemon images } else if image, ok := strings.CutPrefix(image, dockerPrefix); ok { imgInfo, imageName, err = s.extractImageNameTagDigest(image) if err != nil { return imgInfo, err } imgInfo.image, err = daemon.Image(imageName) if err != nil { return imgInfo, err } // Handle remote registry images (default) } else { imgInfo, imageName, err = s.extractImageNameTagDigest(image) if err != nil { return imgInfo, err } imgInfo.image, err = remote.Image(imageName, remoteOpts...) if err != nil { return imgInfo, err } } ctx.Logger().WithValues("image", imgInfo.base, "tag", imgInfo.tag).V(2).Info("scanning image") return imgInfo, nil } // extractImageNameTagDigest parses the provided Docker image string and returns a name.Reference // representing either the image's tag or digest, and any error encountered during parsing. func (*Source) extractImageNameTagDigest(image string) (imageInfo, name.Reference, error) { var ( hasDigest bool imgInfo imageInfo imgName name.Reference err error ) imgInfo.base, imgInfo.tag, hasDigest = baseAndTagFromImage(image) // Parse as digest reference (e.g., nginx@sha256:abc123...) if hasDigest { imgName, err = name.NewDigest(image) // Parse as tag reference (e.g., nginx:latest) } else { imgName, err = name.NewTag(image) } if err != nil { return imgInfo, imgName, err } return imgInfo, imgName, nil } // getHistoryEntries collates an image's configuration history together with the // corresponding layer digests for any non-empty layers. func getHistoryEntries(ctx context.Context, imgInfo imageInfo, layers []v1.Layer) ([]historyEntryInfo, error) { ctx.Logger().V(5).Info("Getting history entries") config, err := imgInfo.image.ConfigFile() if err != nil { return nil, err } history := config.History entries := make([]historyEntryInfo, len(history)) layerIndex := 0 for historyIndex, entry := range history { e := historyEntryInfo{ base: imgInfo.base, tag: imgInfo.tag, entry: entry, index: historyIndex, } // Associate with a layer if possible. Some history entries don't create layers (e.g., ENV, LABEL) // Failing to associate won't affect scanning, just reduces traceability if !entry.EmptyLayer { if layerIndex < len(layers) { digest, err := layers[layerIndex].Digest() if err == nil { e.layerDigest = digest.String() } else { ctx.Logger().Error(err, "cannot associate layer with history entry: layer digest failed", "layerIndex", layerIndex, "historyIndex", historyIndex) } } else { ctx.Logger().V(2).Info("cannot associate layer with history entry: no correlated layer exists at this index", "layerIndex", layerIndex, "historyIndex", historyIndex) } layerIndex++ } entries[historyIndex] = e } return entries, nil } // processHistoryEntry processes a history entry from the image configuration metadata. // It scans the CreatedBy field which contains the command used to create that layer. func (s *Source) processHistoryEntry(ctx context.Context, historyInfo historyEntryInfo, chunksChan chan *sources.Chunk) error { ctx.Logger().V(5).Info("Processing history entries") // Create a descriptive identifier for this history entry // There's no file name here, so we use a synthetic path entryPath := fmt.Sprintf("image-metadata:history:%d:created-by", historyInfo.index) chunk := &sources.Chunk{ SourceType: s.Type(), SourceName: s.name, SourceID: s.SourceID(), SourceMetadata: &source_metadatapb.MetaData{ Data: &source_metadatapb.MetaData_Docker{ Docker: &source_metadatapb.Docker{ File: entryPath, Image: historyInfo.base, Tag: historyInfo.tag, Layer: historyInfo.layerDigest, }, }, }, SourceVerify: s.verify, Data: []byte(historyInfo.entry.CreatedBy), } ctx.Logger().V(2).Info("scanning image history entry", "index", historyInfo.index, "layer", historyInfo.layerDigest) return common.CancellableWrite(ctx, chunksChan, chunk) } // processLayer processes an individual layer of an image. // It decompresses the layer and extracts all files for scanning. func (s *Source) processLayer(ctx context.Context, layer v1.Layer, imgInfo imageInfo, chunksChan chan *sources.Chunk) error { ctx.Logger().V(5).Info("Processing layer") layerInfo := layerInfo{ base: imgInfo.base, tag: imgInfo.tag, } var err error layerInfo.digest, err = layer.Digest() if err != nil { return err } ctx.Logger().WithValues("layer", layerInfo.digest.String()).V(2).Info("scanning layer") rc, err := layer.Compressed() if err != nil { return err } defer rc.Close() // Configure parallel gzip decompression for better performance const ( defaultBlockSize = 1 << 24 // 16MB blocks defaultBlocks = 8 // Process 8 blocks in parallel ) gzipReader, err := gzip.NewReaderN(rc, defaultBlockSize, defaultBlocks) if err != nil { return err } defer gzipReader.Close() // Layers are tar archives, so we read them as tar files tarReader := tar.NewReader(gzipReader) for { header, err := tarReader.Next() if errors.Is(err, io.EOF) { break } if err != nil { return err } info := chunkProcessingInfo{size: header.Size, name: header.Name, reader: tarReader, layer: layerInfo} if err := s.processChunk(ctx, info, chunksChan); err != nil { return err } } return nil } // chunkProcessingInfo holds information needed to process a single file from a layer type chunkProcessingInfo struct { size int64 // Size of the file in bytes name string // Full path of the file within the layer reader io.Reader // Reader for the file contents layer layerInfo // Information about the layer this file belongs to } // processChunk processes an individual chunk of a layer. // It applies file size limits and exclusion patterns before scanning. func (s *Source) processChunk(ctx context.Context, info chunkProcessingInfo, chunksChan chan *sources.Chunk) error { // Skip files that are too large to avoid memory issues const filesizeLimitBytes int64 = 50 * 1024 * 1024 // 50MB if info.size > filesizeLimitBytes { ctx.Logger().V(2).Info("skipping file: size exceeds max allowed", "file", info.name, "size", info.size, "limit", filesizeLimitBytes) return nil } // Check if the file matches any exclude patterns filePath := "/" + info.name if s.isExcluded(ctx, filePath) { return nil } // Read the file in chunks to handle large files efficiently chunkReader := sources.NewChunkReader(sources.WithFileSize(int(info.size))) chunkResChan := chunkReader(ctx, info.reader) for data := range chunkResChan { if err := data.Error(); err != nil { ctx.Logger().Error(err, "error reading chunk.") continue } chunk := &sources.Chunk{ SourceType: s.Type(), SourceName: s.name, SourceID: s.SourceID(), SourceMetadata: &source_metadatapb.MetaData{ Data: &source_metadatapb.MetaData_Docker{ Docker: &source_metadatapb.Docker{ File: "/" + info.name, Image: info.layer.base, Tag: info.layer.tag, Layer: info.layer.digest.String(), }, }, }, SourceVerify: s.verify, } chunk.Data = data.Bytes() if err := common.CancellableWrite(ctx, chunksChan, chunk); err != nil { return err } } return nil } // isExcluded checks if a given filePath should be excluded based on the configured glob patterns. func (s *Source) isExcluded(ctx context.Context, filePath string) bool { if s.globFilter == nil { return false // No filter configured, so nothing is excluded } // ShouldInclude returns true if the file should be scanned (not excluded) // If it returns false, the file matches an exclude pattern isIncluded := s.globFilter.ShouldInclude(filePath) if !isIncluded { ctx.Logger().V(2).Info("skipping file: matches an exclude pattern", "file", filePath, "configured_exclude_paths", s.conn.GetExcludePaths()) } return !isIncluded } // remoteOpts configures the options for fetching images from remote registries. // It sets up HTTP transport and authentication based on the connection configuration. func (s *Source) remoteOpts() ([]remote.Option, error) { // Configure HTTP transport with reasonable timeouts and connection pooling defaultTransport := &http.Transport{ Proxy: http.ProxyFromEnvironment, DialContext: (&net.Dialer{ Timeout: 30 * time.Second, KeepAlive: 30 * time.Second, }).DialContext, ForceAttemptHTTP2: true, MaxIdleConns: s.concurrency * 4, IdleConnTimeout: 90 * time.Second, TLSHandshakeTimeout: 10 * time.Second, ExpectContinueTimeout: 1 * time.Second, MaxIdleConnsPerHost: s.concurrency * 2, } var opts []remote.Option opts = append(opts, remote.WithTransport(common.NewInstrumentedTransport(common.NewCustomTransport(defaultTransport)))) // Configure authentication based on credential type switch s.conn.GetCredential().(type) { case *sourcespb.Docker_Unauthenticated: return nil, nil case *sourcespb.Docker_BasicAuth: opts = append(opts, remote.WithAuth(&authn.Basic{ Username: s.conn.GetBasicAuth().GetUsername(), Password: s.conn.GetBasicAuth().GetPassword(), })) case *sourcespb.Docker_BearerToken: opts = append(opts, remote.WithAuth(&authn.Bearer{ Token: s.conn.GetBearerToken(), })) case *sourcespb.Docker_DockerKeychain: // Use credentials from ~/.docker/config.json opts = append(opts, remote.WithAuthFromKeychain(authn.DefaultKeychain)) default: return nil, fmt.Errorf("unknown credential type: %T", s.conn.Credential) } return opts, nil } func GetNamespaceImages(ctx context.Context, namespace, registryToken string) ([]string, error) { ctx.Logger().V(5).Info("Getting namespace images") registry := MakeRegistryFromNamespace(namespace) // attach the registry authentication token, if one is available. if registryToken != "" { registry.WithRegistryToken(registryToken) } ctx.Logger().Info(fmt.Sprintf("using registry: %s", registry.Name())) namespaceImages, err := registry.ListImages(ctx, namespace) if err != nil { return nil, fmt.Errorf("failed to list namespace images: %w", err) } ctx.Logger().Info(fmt.Sprintf("namespace: %s has %d images", namespace, len(namespaceImages))) return namespaceImages, nil } // baseAndTagFromImage extracts the base name and tag/digest from an image reference string. // It handles both digest-based references (image@sha256:...) and tag-based references (image:tag). func baseAndTagFromImage(image string) (base, tag string, hasDigest bool) { if base, tag, hasDigest = extractDigest(image); hasDigest { return base, tag, true } base, tag = extractTagOrUseDefault(image) return base, tag, false } // extractDigest tries to split the image string on the digest delimiter (@). // If successful, it means the image has a digest reference. func extractDigest(image string) (base, tag string, hasDigest bool) { const digestDelim = "@" if parts := strings.SplitN(image, digestDelim, 2); len(parts) > 1 { return parts[0], parts[1], true } return "", "", false } // extractTagOrUseDefault extracts the tag from the image string. // If no tag is found, it defaults to "latest". func extractTagOrUseDefault(image string) (base, tag string) { const ( tagDelim = ":" regRepoDelim = "/" ) parts := strings.Split(image, tagDelim) // Check if the last part is a tag (not a hostname with port) // We use a weak validation: if it contains "/" it's likely part of a registry hostname if len(parts) > 1 && !strings.Contains(parts[len(parts)-1], regRepoDelim) { return strings.Join(parts[:len(parts)-1], tagDelim), parts[len(parts)-1] } return image, "latest" }