From 0a5948d882789c69d743194ed2634f6f722b9e0e Mon Sep 17 00:00:00 2001 From: Devashish Date: Wed, 18 Sep 2024 14:25:16 -0400 Subject: [PATCH 01/24] packer: add hcp-sbom provisioner The hcp-sbom provisioner is a provisioner that acts essentially like a download-only file provisioner, which also verifies the file downloaded is a SPDX/CycloneDX JSON-encoded SBOM file, and sets up its upload to HCP Packer later on. --- command/execute.go | 2 + go.mod | 5 +- go.sum | 18 ++ hcl2template/types.packer_config.go | 6 + packer/build.go | 20 ++ packer/core.go | 7 + packer/provisioner.go | 101 ++++++++ provisioner/hcp-sbom/provisioner.go | 239 ++++++++++++++++++ provisioner/hcp-sbom/provisioner.hcl2spec.go | 49 ++++ provisioner/hcp-sbom/validate.go | 99 ++++++++ provisioner/hcp-sbom/version/version.go | 16 ++ .../hcp-sbom/Config-not-required.mdx | 13 + .../provisioner/hcp-sbom/Config-required.mdx | 7 + 13 files changed, 581 insertions(+), 1 deletion(-) create mode 100644 provisioner/hcp-sbom/provisioner.go create mode 100644 provisioner/hcp-sbom/provisioner.hcl2spec.go create mode 100644 provisioner/hcp-sbom/validate.go create mode 100644 provisioner/hcp-sbom/version/version.go create mode 100644 website/content/partials/provisioner/hcp-sbom/Config-not-required.mdx create mode 100644 website/content/partials/provisioner/hcp-sbom/Config-required.mdx diff --git a/command/execute.go b/command/execute.go index 7ad74f314d4..1e303858d61 100644 --- a/command/execute.go +++ b/command/execute.go @@ -28,6 +28,7 @@ import ( shelllocalpostprocessor "github.com/hashicorp/packer/post-processor/shell-local" breakpointprovisioner "github.com/hashicorp/packer/provisioner/breakpoint" fileprovisioner "github.com/hashicorp/packer/provisioner/file" + hcpsbomprovisioner "github.com/hashicorp/packer/provisioner/hcp-sbom" powershellprovisioner "github.com/hashicorp/packer/provisioner/powershell" shellprovisioner "github.com/hashicorp/packer/provisioner/shell" shelllocalprovisioner "github.com/hashicorp/packer/provisioner/shell-local" @@ -48,6 +49,7 @@ var Builders = map[string]packersdk.Builder{ var Provisioners = map[string]packersdk.Provisioner{ "breakpoint": new(breakpointprovisioner.Provisioner), "file": new(fileprovisioner.Provisioner), + "hcp-sbom": new(hcpsbomprovisioner.Provisioner), "powershell": new(powershellprovisioner.Provisioner), "shell": new(shellprovisioner.Provisioner), "shell-local": new(shelllocalprovisioner.Provisioner), diff --git a/go.mod b/go.mod index b45259084fb..f90f2eec757 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,7 @@ require ( github.com/hashicorp/hcp-sdk-go v0.112.0 github.com/hashicorp/packer-plugin-sdk v0.5.4 github.com/jehiah/go-strftime v0.0.0-20171201141054-1d33003b3869 - github.com/klauspost/compress v1.13.6 // indirect + github.com/klauspost/compress v1.13.6 github.com/klauspost/pgzip v1.2.5 github.com/masterzen/winrm v0.0.0-20210623064412-3b76017826b0 github.com/mattn/go-runewidth v0.0.13 // indirect @@ -57,10 +57,12 @@ require ( ) require ( + github.com/CycloneDX/cyclonedx-go v0.9.1 github.com/go-openapi/strfmt v0.21.10 github.com/oklog/ulid v1.3.1 github.com/pierrec/lz4/v4 v4.1.18 github.com/shirou/gopsutil/v3 v3.23.4 + github.com/spdx/tools-golang v0.5.5 ) require ( @@ -77,6 +79,7 @@ require ( github.com/Microsoft/go-winio v0.6.1 // indirect github.com/ProtonMail/go-crypto v1.1.3 // indirect github.com/agext/levenshtein v1.2.3 // indirect + github.com/anchore/go-struct-converter v0.0.0-20221118182256-c68fdcfa2092 // indirect github.com/apparentlymart/go-cidr v1.0.1 // indirect github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect diff --git a/go.sum b/go.sum index 3678a621bc7..04e2675ab4b 100644 --- a/go.sum +++ b/go.sum @@ -20,6 +20,8 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym github.com/ChrisTrenkamp/goxpath v0.0.0-20170922090931-c385f95c6022/go.mod h1:nuWgzSkT5PnyOd+272uUmV0dnAnAn42Mk7PiQC5VzN4= github.com/ChrisTrenkamp/goxpath v0.0.0-20210404020558-97928f7e12b6 h1:w0E0fgc1YafGEh5cROhlROMWXiNoZqApk2PDN0M1+Ns= github.com/ChrisTrenkamp/goxpath v0.0.0-20210404020558-97928f7e12b6/go.mod h1:nuWgzSkT5PnyOd+272uUmV0dnAnAn42Mk7PiQC5VzN4= +github.com/CycloneDX/cyclonedx-go v0.9.1 h1:yffaWOZsv77oTJa/SdVZYdgAgFioCeycBUKkqS2qzQM= +github.com/CycloneDX/cyclonedx-go v0.9.1/go.mod h1:NE/EWvzELOFlG6+ljX/QeMlVt9VKcTwu8u0ccsACEsw= github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= @@ -38,6 +40,8 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/anchore/go-struct-converter v0.0.0-20221118182256-c68fdcfa2092 h1:aM1rlcoLz8y5B2r4tTLMiVTrMtpfY0O8EScKJxaSaEc= +github.com/anchore/go-struct-converter v0.0.0-20221118182256-c68fdcfa2092/go.mod h1:rYqSE9HbjzpHTI74vwPvae4ZVYZd1lue2ta6xHPdblA= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= github.com/antchfx/xmlquery v1.3.5 h1:I7TuBRqsnfFuL11ruavGm911Awx9IqSdiU6W/ztSmVw= @@ -77,6 +81,8 @@ github.com/biogo/hts v1.4.3 h1:vir2yUTiRkPvtp6ZTpzh9lWTKQJZXJKZ563rpAQAsRM= github.com/biogo/hts v1.4.3/go.mod h1:eW40HJ1l2ExK9C+yvvoRSftInqWsf3ue+zAEjzCGWjA= github.com/bmatcuk/doublestar v1.1.5 h1:2bNwBOmhyFEFcoB3tGvTD5xanq+4kyOZlB8wFYbMjkk= github.com/bmatcuk/doublestar v1.1.5/go.mod h1:wiQtGV+rzVYxB7WIlirSN++5HPtPlXEo9MEoZQC/PmE= +github.com/bradleyjkemp/cupaloy/v2 v2.8.0 h1:any4BmKE+jGIaMpnU8YgH/I2LPiLBufr6oMMlVBbn9M= +github.com/bradleyjkemp/cupaloy/v2 v2.8.0/go.mod h1:bm7JXdkRd4BHJk9HpwqAI8BoAY1lps46Enkdqw6aRX0= github.com/cenkalti/backoff/v3 v3.2.2 h1:cfUAAO3yvKMYKPrvhDuHSwQnhZNk/RMHKdZqKTxfm6M= github.com/cenkalti/backoff/v3 v3.2.2/go.mod h1:cIeZDE3IrqwwJl6VUwCN6trj1oXrTS4rc0ij+ULvLYs= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -488,6 +494,9 @@ github.com/skeema/knownhosts v1.3.0 h1:AM+y0rI04VksttfwjkSTNQorvGqmwATnvnAHpSgc0 github.com/skeema/knownhosts v1.3.0/go.mod h1:sPINvnADmT/qYH1kfv+ePMmOBTH6Tbl7b5LvTDjFK7M= github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 h1:JIAuq3EEf9cgbU6AtGPK4CTG3Zf6CKMNqf0MHTggAUA= github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966/go.mod h1:sUM3LWHvSMaG192sy56D9F7CNvL7jUJVXoqM1QKLnog= +github.com/spdx/gordf v0.0.0-20201111095634-7098f93598fb/go.mod h1:uKWaldnbMnjsSAXRurWqqrdyZen1R7kxl8TkmWk2OyM= +github.com/spdx/tools-golang v0.5.5 h1:61c0KLfAcNqAjlg6UNMdkwpMernhw3zVRwDZ2x9XOmk= +github.com/spdx/tools-golang v0.5.5/go.mod h1:MVIsXx8ZZzaRWNQpUDhC4Dud34edUYJYecciXgrw5vE= github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -507,8 +516,12 @@ github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1F github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/terminalstatic/go-xsd-validate v0.1.5 h1:RqpJnf6HGE2CB/lZB1A8BYguk8uRtcvYAPLCF15qguo= +github.com/terminalstatic/go-xsd-validate v0.1.5/go.mod h1:18lsvYFofBflqCrvo1umpABZ99+GneNTw2kEEc8UPJw= github.com/tklauser/go-sysconf v0.3.11 h1:89WgdJhk5SNwJfu+GKyYveZ4IaJ7xAkecBo+KdJV0CM= github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7AmcWpGR8lSZfqI= github.com/tklauser/numcpus v0.6.0 h1:kebhY2Qt+3U6RNK7UqpYNA+tJ23IBEGKkB7JQBfDYms= @@ -529,6 +542,10 @@ github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3k github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= +github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= +github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= @@ -735,3 +752,4 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/hcl2template/types.packer_config.go b/hcl2template/types.packer_config.go index 626ebdb7bb2..f0037601434 100644 --- a/hcl2template/types.packer_config.go +++ b/hcl2template/types.packer_config.go @@ -575,6 +575,12 @@ func (cfg *PackerConfig) getCoreBuildProvisioner(source SourceUseBlock, pb *Prov } } + if pb.PType == "hcp-sbom" { + provisioner = &packer.SBOMInternalProvisioner{ + Provisioner: provisioner, + } + } + return packer.CoreBuildProvisioner{ PType: pb.PType, PName: pb.PName, diff --git a/packer/build.go b/packer/build.go index 8b62ec53799..eade2625dd6 100644 --- a/packer/build.go +++ b/packer/build.go @@ -50,11 +50,19 @@ type CoreBuild struct { onError string l sync.Mutex prepareCalled bool + + SBOMs []SBOM +} + +type SBOM struct { + Format string + CompressedData []byte } type BuildMetadata struct { PackerVersion string Plugins map[string]PluginDetails + SBOMs []SBOM } func (b *CoreBuild) getPluginsMetadata() map[string]PluginDetails { @@ -88,6 +96,7 @@ func (b *CoreBuild) GetMetadata() BuildMetadata { metadata := BuildMetadata{ PackerVersion: version.FormattedVersion(), Plugins: b.getPluginsMetadata(), + SBOMs: b.SBOMs, } return metadata } @@ -300,6 +309,17 @@ func (b *CoreBuild) Run(ctx context.Context, originalUi packersdk.Ui) ([]packers return nil, err } + for _, p := range b.Provisioners { + sbomInternalProvisioner, ok := p.Provisioner.(*SBOMInternalProvisioner) + if ok { + sbom := SBOM{ + Format: sbomInternalProvisioner.SBOMFormat, + CompressedData: sbomInternalProvisioner.CompressedData, + } + b.SBOMs = append(b.SBOMs, sbom) + } + } + // If there was no result, don't worry about running post-processors // because there is nothing they can do, just return. if builderArtifact == nil { diff --git a/packer/core.go b/packer/core.go index 6bff2df060a..f6724cda9ef 100644 --- a/packer/core.go +++ b/packer/core.go @@ -296,6 +296,13 @@ func (c *Core) generateCoreBuildProvisioner(rawP *template.Provisioner, rawName Provisioner: provisioner, } } + + if rawP.Type == "hcp-sbom" { + provisioner = &SBOMInternalProvisioner{ + Provisioner: provisioner, + } + } + cbp = CoreBuildProvisioner{ PType: rawP.Type, Provisioner: provisioner, diff --git a/packer/provisioner.go b/packer/provisioner.go index 81dce0ecfc0..24d670950a1 100644 --- a/packer/provisioner.go +++ b/packer/provisioner.go @@ -7,6 +7,12 @@ import ( "context" "fmt" "log" + "os" + + hcpSbomProvisioner "github.com/hashicorp/packer/provisioner/hcp-sbom" + + "github.com/klauspost/compress/zstd" + "time" "github.com/hashicorp/hcl/v2/hcldec" @@ -234,3 +240,98 @@ func (p *DebuggedProvisioner) Provision(ctx context.Context, ui packersdk.Ui, co return p.Provisioner.Provision(ctx, ui, comm, generatedData) } + +// SBOMInternalProvisioner is a wrapper provisioner for the `hcp-sbom` provisioner +// that sets the path for SBOM file download and, after the successful execution of +// the `hcp-sbom` provisioner, compresses the SBOM and prepares the data for API +// integration. +type SBOMInternalProvisioner struct { + Provisioner packersdk.Provisioner + CompressedData []byte + SBOMFormat string +} + +func (p *SBOMInternalProvisioner) ConfigSpec() hcldec.ObjectSpec { return p.ConfigSpec() } +func (p *SBOMInternalProvisioner) FlatConfig() interface{} { return p.FlatConfig() } +func (p *SBOMInternalProvisioner) Prepare(raws ...interface{}) error { + return p.Provisioner.Prepare(raws...) +} + +func (p *SBOMInternalProvisioner) Provision( + ctx context.Context, ui packersdk.Ui, comm packersdk.Communicator, + generatedData map[string]interface{}, +) error { + cwd, err := os.Getwd() + if err != nil { + return fmt.Errorf("failed to get current working directory for Packer SBOM: %s", err) + } + + tmpFile, err := os.CreateTemp(cwd, "packer-sbom-*.json") + if err != nil { + return fmt.Errorf("failed to create internal temporary file for Packer SBOM: %s", err) + } + + tmpFileName := tmpFile.Name() + if err = tmpFile.Close(); err != nil { + return fmt.Errorf("failed to close temporary file for Packer SBOM %s: %s", tmpFileName, err) + } + + defer func(name string) { + fileRemoveErr := os.Remove(name) + if fileRemoveErr != nil { + log.Printf("Error removing SBOM temporary file %s: %s", name, fileRemoveErr) + } + }(tmpFile.Name()) + + generatedData["dst"] = tmpFile.Name() + + err = p.Provisioner.Provision(ctx, ui, comm, generatedData) + if err != nil { + return err + } + + sbomFormat, err := p.getSBOMFormat(tmpFile.Name()) + if err != nil { + return err + } + + compressedData, err := p.compressFile(tmpFile.Name()) + if err != nil { + return err + } + p.CompressedData = compressedData + p.SBOMFormat = sbomFormat + return nil +} + +func (p *SBOMInternalProvisioner) compressFile(filePath string) ([]byte, error) { + data, err := os.ReadFile(filePath) + if err != nil { + return nil, fmt.Errorf("failed to read file %s: %w", filePath, err) + } + + encoder, err := zstd.NewWriter(nil, zstd.WithEncoderLevel(zstd.SpeedBestCompression)) + if err != nil { + return nil, fmt.Errorf("failed to create zstd encoder: %w", err) + } + + compressedData := encoder.EncodeAll(data, nil) + + log.Printf("SBOM file compressed successfully. Size: %d bytes\n", len(compressedData)) + return compressedData, nil +} + +func (p *SBOMInternalProvisioner) getSBOMFormat(filePath string) (string, error) { + file, err := os.Open(filePath) + if err != nil { + return "", fmt.Errorf("failed to open SBOM file %s: %w", filePath, err) + } + defer file.Close() + + format, err := hcpSbomProvisioner.ValidateSBOM(file) + if err != nil { + return "", fmt.Errorf("failed to detect SBOM format: %w", err) + } + + return format, nil +} diff --git a/provisioner/hcp-sbom/provisioner.go b/provisioner/hcp-sbom/provisioner.go new file mode 100644 index 00000000000..2fd9e834e98 --- /dev/null +++ b/provisioner/hcp-sbom/provisioner.go @@ -0,0 +1,239 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +//go:generate packer-sdc mapstructure-to-hcl2 -type Config +//go:generate packer-sdc struct-markdown + +package hcp_sbom + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "log" + "os" + + "path/filepath" + + "github.com/hashicorp/hcl/v2/hcldec" + "github.com/hashicorp/packer-plugin-sdk/common" + packersdk "github.com/hashicorp/packer-plugin-sdk/packer" + "github.com/hashicorp/packer-plugin-sdk/template/config" + "github.com/hashicorp/packer-plugin-sdk/template/interpolate" +) + +type Config struct { + common.PackerConfig `mapstructure:",squash"` + + // Source is a required field that specifies the path to the SBOM file that + // needs to be downloaded. + // It can be a file path or a URL. + Source string `mapstructure:"source" required:"true"` + // Destination is an optional field that specifies the path where the SBOM + // file will be downloaded to for the user. + // The 'Destination' must be a writable location. If the destination is a file, + // the SBOM will be saved or overwritten at that path. If the destination is + // a directory, a file will be created within the directory to store the SBOM. + // Any parent directories for the destination must already exist and be + // writable by the provisioning user (generally not root), otherwise, + // a "Permission Denied" error will occur. If the source path is a file, + // it is recommended that the destination path be a file as well. + Destination string `mapstructure:"destination"` + ctx interpolate.Context +} + +type Provisioner struct { + config Config +} + +func (p *Provisioner) ConfigSpec() hcldec.ObjectSpec { + return p.config.FlatMapstructure().HCL2Spec() +} + +func (p *Provisioner) Prepare(raws ...interface{}) error { + err := config.Decode(&p.config, &config.DecodeOpts{ + PluginType: "hcp-sbom", + Interpolate: true, + InterpolateContext: &p.config.ctx, + InterpolateFilter: &interpolate.RenderFilter{ + Exclude: []string{}, + }, + }, raws...) + if err != nil { + return err + } + + var errs *packersdk.MultiError + if p.config.Source == "" { + errs = packersdk.MultiErrorAppend(errs, errors.New("source must be specified")) + } + + if errs != nil && len(errs.Errors) > 0 { + return errs + } + + return nil +} + +func (p *Provisioner) Provision( + ctx context.Context, ui packersdk.Ui, comm packersdk.Communicator, + generatedData map[string]interface{}, +) error { + log.Println("Starting to provision with `hcp-sbom` provisioner") + + if generatedData == nil { + generatedData = make(map[string]interface{}) + } + p.config.ctx.Data = generatedData + + downloadErr := p.downloadAndValidateSBOM(ui, comm, generatedData) + if downloadErr != nil { + return fmt.Errorf("failed to download SBOM file: %w", downloadErr) + } + + return nil +} + +// downloadAndValidateSBOM handles downloading SBOM files for the User and Packer. +func (p *Provisioner) downloadAndValidateSBOM( + ui packersdk.Ui, comm packersdk.Communicator, generatedData map[string]interface{}, +) error { + src, err := interpolate.Render(p.config.Source, &p.config.ctx) + if err != nil { + return fmt.Errorf("error interpolating SBOM source: %s", err) + } + + var buf bytes.Buffer + if err = comm.Download(src, &buf); err != nil { + ui.Errorf("download failed for SBOM file: %s", err) + return err + } + + reader := bytes.NewReader(buf.Bytes()) + if _, err = ValidateSBOM(reader); err != nil { + ui.Errorf("validation failed for SBOM file: %s", err) + return err + } + _, err = reader.Seek(0, io.SeekStart) + if err != nil { + return err + } + + // SBOM for Packer + pkrDst, err := p.getPackerDestination(generatedData) + if err != nil { + return fmt.Errorf("failed to get Packer SBOM destination: %s", err) + } + + err = p.writeToFile(reader, pkrDst) + if err != nil { + return fmt.Errorf("failed to download Packer SBOM: %s", err) + } + _, err = reader.Seek(0, io.SeekStart) + if err != nil { + return err + } + log.Printf("Packer SBOM file successfully downloaded to: %s\n", pkrDst) + + // SBOM for User + usrDst, err := p.getUserDestination() + if err != nil { + return fmt.Errorf("failed to determine user SBOM destination: %s", err) + } + + if usrDst != "" { + err = p.writeToFile(reader, usrDst) + if err != nil { + return fmt.Errorf("failed to download User SBOM: %s", err) + } + log.Printf("User SBOM file successfully downloaded to: %s\n", usrDst) + } + return nil +} + +// getUserDestination determines and returns the destination path for the user SBOM file. +func (p *Provisioner) getUserDestination() (string, error) { + dst := p.config.Destination + if dst == "" { + log.Println("skipped downloading user SBOM file because 'Destination' is not provided") + return "", nil + } + + dst, err := interpolate.Render(dst, &p.config.ctx) + if err != nil { + return "", fmt.Errorf("error interpolating SBOM file destination for user: %s", err) + } + + // Check if the destination exists and determine its type + info, err := os.Stat(dst) + if err == nil { + if info.IsDir() { + // If the destination is a directory, create a temporary file inside it + tmpFile, err := os.CreateTemp(dst, "packer-user-sbom-*.json") + if err != nil { + return "", fmt.Errorf("failed to create temporary file in user SBOM directory %s: %s", dst, err) + } + dst = tmpFile.Name() + tmpFile.Close() + } + return dst, nil + } + + outDir := filepath.Dir(dst) + // In case the destination does not exist, we'll get the dirpath, + // and create it if it doesn't already exist + err = os.MkdirAll(outDir, 0755) + if err != nil { + return "", fmt.Errorf("failed to create destination directory for user SBOM: %s\n", err) + } + + // Check if the destination is a directory after the previous step. + // + // This happens if the path specified ends with a `/`, in which case the + // destination is a directory, and we must create a temporary file in + // this destination directory. + destStat, statErr := os.Stat(dst) + if statErr == nil && destStat.IsDir() { + tmpFile, err := os.CreateTemp(outDir, "packer-user-sbom-*.json") + if err != nil { + return "", fmt.Errorf("failed to create temporary file in user SBOM directory %s: %s", dst, err) + } + dst = tmpFile.Name() + tmpFile.Close() + } + + return dst, nil +} + +// getPackerDestination retrieves the destination path for the Packer SBOM file. +func (p *Provisioner) getPackerDestination(generatedData map[string]interface{}) (string, error) { + dst, ok := generatedData["dst"].(string) // This has been set by HCPSBOMInternalProvisioner.Provision + if !ok || dst == "" { + return "", fmt.Errorf("destination path for Packer SBOM file is not valid") + } + + // Ensure the destination directory exists + if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil { + return "", fmt.Errorf("failed to create destination directory for Packer SBOM: %w", err) + } + + return dst, nil +} + +func (p *Provisioner) writeToFile(buf *bytes.Reader, dst string) error { + // Open the destination file + f, err := os.OpenFile(dst, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644) + if err != nil { + return fmt.Errorf("failed to open destination file for SBOM: %s", err) + } + defer f.Close() + + // Write the buffer content to the destination file + if _, err = buf.WriteTo(f); err != nil { + return err + } + + return nil +} diff --git a/provisioner/hcp-sbom/provisioner.hcl2spec.go b/provisioner/hcp-sbom/provisioner.hcl2spec.go new file mode 100644 index 00000000000..e6ee5d22753 --- /dev/null +++ b/provisioner/hcp-sbom/provisioner.hcl2spec.go @@ -0,0 +1,49 @@ +// Code generated by "packer-sdc mapstructure-to-hcl2"; DO NOT EDIT. + +package hcp_sbom + +import ( + "github.com/hashicorp/hcl/v2/hcldec" + "github.com/zclconf/go-cty/cty" +) + +// FlatConfig is an auto-generated flat version of Config. +// Where the contents of a field with a `mapstructure:,squash` tag are bubbled up. +type FlatConfig struct { + PackerBuildName *string `mapstructure:"packer_build_name" cty:"packer_build_name" hcl:"packer_build_name"` + PackerBuilderType *string `mapstructure:"packer_builder_type" cty:"packer_builder_type" hcl:"packer_builder_type"` + PackerCoreVersion *string `mapstructure:"packer_core_version" cty:"packer_core_version" hcl:"packer_core_version"` + PackerDebug *bool `mapstructure:"packer_debug" cty:"packer_debug" hcl:"packer_debug"` + PackerForce *bool `mapstructure:"packer_force" cty:"packer_force" hcl:"packer_force"` + PackerOnError *string `mapstructure:"packer_on_error" cty:"packer_on_error" hcl:"packer_on_error"` + PackerUserVars map[string]string `mapstructure:"packer_user_variables" cty:"packer_user_variables" hcl:"packer_user_variables"` + PackerSensitiveVars []string `mapstructure:"packer_sensitive_variables" cty:"packer_sensitive_variables" hcl:"packer_sensitive_variables"` + Source *string `mapstructure:"source" required:"true" cty:"source" hcl:"source"` + Destination *string `mapstructure:"destination" cty:"destination" hcl:"destination"` +} + +// FlatMapstructure returns a new FlatConfig. +// FlatConfig is an auto-generated flat version of Config. +// Where the contents a fields with a `mapstructure:,squash` tag are bubbled up. +func (*Config) FlatMapstructure() interface{ HCL2Spec() map[string]hcldec.Spec } { + return new(FlatConfig) +} + +// HCL2Spec returns the hcl spec of a Config. +// This spec is used by HCL to read the fields of Config. +// The decoded values from this spec will then be applied to a FlatConfig. +func (*FlatConfig) HCL2Spec() map[string]hcldec.Spec { + s := map[string]hcldec.Spec{ + "packer_build_name": &hcldec.AttrSpec{Name: "packer_build_name", Type: cty.String, Required: false}, + "packer_builder_type": &hcldec.AttrSpec{Name: "packer_builder_type", Type: cty.String, Required: false}, + "packer_core_version": &hcldec.AttrSpec{Name: "packer_core_version", Type: cty.String, Required: false}, + "packer_debug": &hcldec.AttrSpec{Name: "packer_debug", Type: cty.Bool, Required: false}, + "packer_force": &hcldec.AttrSpec{Name: "packer_force", Type: cty.Bool, Required: false}, + "packer_on_error": &hcldec.AttrSpec{Name: "packer_on_error", Type: cty.String, Required: false}, + "packer_user_variables": &hcldec.AttrSpec{Name: "packer_user_variables", Type: cty.Map(cty.String), Required: false}, + "packer_sensitive_variables": &hcldec.AttrSpec{Name: "packer_sensitive_variables", Type: cty.List(cty.String), Required: false}, + "source": &hcldec.AttrSpec{Name: "source", Type: cty.String, Required: false}, + "destination": &hcldec.AttrSpec{Name: "destination", Type: cty.String, Required: false}, + } + return s +} diff --git a/provisioner/hcp-sbom/validate.go b/provisioner/hcp-sbom/validate.go new file mode 100644 index 00000000000..cad502cb934 --- /dev/null +++ b/provisioner/hcp-sbom/validate.go @@ -0,0 +1,99 @@ +package hcp_sbom + +import ( + "bytes" + "fmt" + "strings" + + "github.com/CycloneDX/cyclonedx-go" + spdxjson "github.com/spdx/tools-golang/json" + + "io" +) + +// ValidationError represents an error encountered while validating an SBOM. +type ValidationError struct { + Err error +} + +func (e *ValidationError) Error() string { + return e.Err.Error() +} + +func (e *ValidationError) Unwrap() error { + return e.Err +} + +// ValidateCycloneDX is a validation for CycloneDX in JSON format. +func ValidateCycloneDX(content io.Reader) error { + decoder := cyclonedx.NewBOMDecoder(content, cyclonedx.BOMFileFormatJSON) + bom := new(cyclonedx.BOM) + if err := decoder.Decode(bom); err != nil { + return fmt.Errorf("error parsing CycloneDX SBOM: %w", err) + } + + if !strings.EqualFold(bom.BOMFormat, "CycloneDX") { + return &ValidationError{ + Err: fmt.Errorf("invalid bomFormat: %q, expected CycloneDX", bom.BOMFormat), + } + } + if bom.SpecVersion.String() == "" { + return &ValidationError{ + Err: fmt.Errorf("specVersion is required"), + } + } + + return nil +} + +// ValidateSPDX is a validation for SPDX in JSON format. +func ValidateSPDX(content io.Reader) error { + doc, err := spdxjson.Read(content) + if err != nil { + return fmt.Errorf("error parsing SPDX JSON file: %w", err) + } + + if doc.SPDXVersion == "" { + return &ValidationError{ + Err: fmt.Errorf("missing SPDXVersion"), + } + } + + return nil +} + +// ValidateSBOM validates the SBOM file and returns the format of the SBOM. +func ValidateSBOM(content io.Reader) (string, error) { + var buf bytes.Buffer + if _, err := io.Copy(&buf, content); err != nil { + return "", fmt.Errorf("failed to copy content: %s", err) + } + + reader := bytes.NewReader(buf.Bytes()) + + // Try validating as SPDX + spdxErr := ValidateSPDX(reader) + if spdxErr == nil { + return "spdx", nil + } + + if vErr, ok := spdxErr.(*ValidationError); ok { + return "", vErr + } + + // Reset the reader's position + if _, err := reader.Seek(0, io.SeekStart); err != nil { + return "", fmt.Errorf("failed to reset reader: %s", err) + } + + cycloneDxErr := ValidateCycloneDX(reader) + if cycloneDxErr == nil { + return "cyclonedx", nil + } + + if vErr, ok := cycloneDxErr.(*ValidationError); ok { + return "", vErr + } + + return "", fmt.Errorf("error validating SBOM file: invalid SBOM format") +} diff --git a/provisioner/hcp-sbom/version/version.go b/provisioner/hcp-sbom/version/version.go new file mode 100644 index 00000000000..772d6d4f444 --- /dev/null +++ b/provisioner/hcp-sbom/version/version.go @@ -0,0 +1,16 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package version + +import ( + "github.com/hashicorp/packer-plugin-sdk/version" + packerVersion "github.com/hashicorp/packer/version" +) + +var HCPSBOMPluginVersion *version.PluginVersion + +func init() { + HCPSBOMPluginVersion = version.NewPluginVersion( + packerVersion.Version, packerVersion.VersionPrerelease, packerVersion.VersionMetadata) +} diff --git a/website/content/partials/provisioner/hcp-sbom/Config-not-required.mdx b/website/content/partials/provisioner/hcp-sbom/Config-not-required.mdx new file mode 100644 index 00000000000..eb241a06c77 --- /dev/null +++ b/website/content/partials/provisioner/hcp-sbom/Config-not-required.mdx @@ -0,0 +1,13 @@ + + +- `destination` (string) - Destination is an optional field that specifies the path where the SBOM + file will be downloaded to for the user. + The 'Destination' must be a writable location. If the destination is a file, + the SBOM will be saved or overwritten at that path. If the destination is + a directory, a file will be created within the directory to store the SBOM. + Any parent directories for the destination must already exist and be + writable by the provisioning user (generally not root), otherwise, + a "Permission Denied" error will occur. If the source path is a file, + it is recommended that the destination path be a file as well. + + diff --git a/website/content/partials/provisioner/hcp-sbom/Config-required.mdx b/website/content/partials/provisioner/hcp-sbom/Config-required.mdx new file mode 100644 index 00000000000..2f227c2b0ff --- /dev/null +++ b/website/content/partials/provisioner/hcp-sbom/Config-required.mdx @@ -0,0 +1,7 @@ + + +- `source` (string) - Source is a required field that specifies the path to the SBOM file that + needs to be downloaded. + It can be a file path or a URL. + + From aa737c434719294b6619b1323ee341dc677ab565 Mon Sep 17 00:00:00 2001 From: Lucas Bajolet Date: Fri, 25 Oct 2024 11:32:06 -0400 Subject: [PATCH 02/24] fixme: add path/home to commands for docker to run, should be generalised --- packer_test/common/commands.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packer_test/common/commands.go b/packer_test/common/commands.go index 869a6b27811..52681f6c375 100644 --- a/packer_test/common/commands.go +++ b/packer_test/common/commands.go @@ -45,6 +45,8 @@ func (ts *PackerTestSuite) PackerCommand() *packerCommand { // make them as self-contained and quick as possible. // Removing telemetry here is probably for the best. "CHECKPOINT_DISABLE": "1", + "HOME": os.Getenv("HOME"), + "PATH": os.Getenv("PATH"), }, t: ts.T(), } From a5e66973beaafccfb13785e8593a45123ee916c9 Mon Sep 17 00:00:00 2001 From: Lucas Bajolet Date: Fri, 25 Oct 2024 11:32:36 -0400 Subject: [PATCH 03/24] packer_test: add intergation tests for hcp-sbom --- .../hcp-sbom/priovisioner_test.go | 139 ++++++++++++++++++ .../provisioner_tests/hcp-sbom/suite_test.go | 23 +++ .../hcp-sbom/templates/dest_is_dir.pkr.hcl | 36 +++++ .../dest_is_dir_with_trailing_slash.pkr.hcl | 36 +++++ .../dest_is_file_no_interm_dirs.pkr.hcl | 36 +++++ .../dest_is_file_with_interm_dirs.pkr.hcl | 36 +++++ .../hcp-sbom/templates/source_is_dir.pkr.hcl | 21 +++ .../templates/source_not_existing.pkr.hcl | 21 +++ 8 files changed, 348 insertions(+) create mode 100644 packer_test/provisioner_tests/hcp-sbom/priovisioner_test.go create mode 100644 packer_test/provisioner_tests/hcp-sbom/suite_test.go create mode 100644 packer_test/provisioner_tests/hcp-sbom/templates/dest_is_dir.pkr.hcl create mode 100644 packer_test/provisioner_tests/hcp-sbom/templates/dest_is_dir_with_trailing_slash.pkr.hcl create mode 100644 packer_test/provisioner_tests/hcp-sbom/templates/dest_is_file_no_interm_dirs.pkr.hcl create mode 100644 packer_test/provisioner_tests/hcp-sbom/templates/dest_is_file_with_interm_dirs.pkr.hcl create mode 100644 packer_test/provisioner_tests/hcp-sbom/templates/source_is_dir.pkr.hcl create mode 100644 packer_test/provisioner_tests/hcp-sbom/templates/source_not_existing.pkr.hcl diff --git a/packer_test/provisioner_tests/hcp-sbom/priovisioner_test.go b/packer_test/provisioner_tests/hcp-sbom/priovisioner_test.go new file mode 100644 index 00000000000..0d82c6441e7 --- /dev/null +++ b/packer_test/provisioner_tests/hcp-sbom/priovisioner_test.go @@ -0,0 +1,139 @@ +package plugin_tests + +import ( + "os" + + "github.com/hashicorp/packer/packer_test/common/check" +) + +func (ts *PackerHCPSbomTestSuite) TestSourceNotExisting() { + ts.SkipNoAcc() + + dir := ts.MakePluginDir() + defer dir.Cleanup() + + ts.PackerCommand().UsePluginDir(dir). + SetArgs("plugins", "install", "github.com/hashicorp/docker"). + Assert(check.MustSucceed()) + + ts.PackerCommand().UsePluginDir(dir). + SetArgs("build", "templates/source_not_existing.pkr.hcl"). + Assert(check.MustFail(), check.Grep("download failed for SBOM file")) +} + +// Greayed out because the communicator for the docker plugin does not return an error +// when downloading a full directory, instead it returns a 0-byte stream without an error. +// +// So the sbom provisioner fails with a validation error instead of a file not found type +// of error. +// +// func (ts *PackerHCPSbomTestSuite) TestSourceIsDir() { +// ts.SkipNoAcc() +// +// path, cleanup := ts.MakePluginDir() +// defer cleanup() +// +// ts.PackerCommand().UsePluginDir(path). +// SetArgs("plugins", "install", "github.com/hashicorp/docker"). +// Assert(check.MustSucceed()) +// +// ts.PackerCommand().UsePluginDir(path). +// SetArgs("build", "templates/source_is_dir.pkr.hcl"). +// Assert(check.MustFail(), check.Grep("download failed for SBOM file"), check.Dump(ts.T())) +// } + +// * output file - does not exist, and intermediate dirs don't exist +func (ts *PackerHCPSbomTestSuite) TestDestFile_NoIntermediateDirs() { + ts.SkipNoAcc() + + dir := ts.MakePluginDir() + defer dir.Cleanup() + + ts.PackerCommand().UsePluginDir(dir). + SetArgs("plugins", "install", "github.com/hashicorp/docker"). + Assert(check.MustSucceed()) + + ts.PackerCommand().UsePluginDir(dir). + SetArgs("build", "./templates/dest_is_file_no_interm_dirs.pkr.hcl"). + Assert(check.MustSucceed(), check.FileExists("sbom/sbom_cyclonedx.json", false)) + + os.RemoveAll("sbom") +} + +// * output file - does not exist, and intermediate dirs already exist +func (ts *PackerHCPSbomTestSuite) TestDestFile_WithIntermediateDirs() { + ts.SkipNoAcc() + + dir := ts.MakePluginDir() + defer dir.Cleanup() + + os.MkdirAll("sbom", 0755) + + ts.PackerCommand().UsePluginDir(dir). + SetArgs("plugins", "install", "github.com/hashicorp/docker"). + Assert(check.MustSucceed()) + + ts.PackerCommand().UsePluginDir(dir). + SetArgs("build", "./templates/dest_is_file_no_interm_dirs.pkr.hcl"). + Assert(check.MustSucceed(), check.FileExists("sbom/sbom_cyclonedx.json", false)) + + os.RemoveAll("sbom") +} + +// * output directory (without trailing slash) - directory exists +func (ts *PackerHCPSbomTestSuite) TestDestDir_NoTrailingSlash() { + ts.SkipNoAcc() + + dir := ts.MakePluginDir() + defer dir.Cleanup() + + os.MkdirAll("sbom", 0755) + + ts.PackerCommand().UsePluginDir(dir). + SetArgs("plugins", "install", "github.com/hashicorp/docker"). + Assert(check.MustSucceed()) + + ts.PackerCommand().UsePluginDir(dir). + SetArgs("build", "./templates/dest_is_dir.pkr.hcl"). + Assert(check.MustSucceed(), check.FileGlob("./sbom/packer-user-sbom-*.json")) + + os.RemoveAll("sbom") +} + +// * output directory (with trailing slash) - directory exists +func (ts *PackerHCPSbomTestSuite) TestDestDir_WithTrailingSlash() { + ts.SkipNoAcc() + + dir := ts.MakePluginDir() + defer dir.Cleanup() + + os.MkdirAll("sbom", 0755) + + ts.PackerCommand().UsePluginDir(dir). + SetArgs("plugins", "install", "github.com/hashicorp/docker"). + Assert(check.MustSucceed()) + + ts.PackerCommand().UsePluginDir(dir). + SetArgs("build", "./templates/dest_is_dir_with_trailing_slash.pkr.hcl"). + Assert(check.MustSucceed(), check.FileGlob("./sbom/packer-user-sbom-*.json")) + + os.RemoveAll("sbom") +} + +// * output directory (with trailing slash) - directory doesn't exist +func (ts *PackerHCPSbomTestSuite) TestDestDir_WithTrailingSlash_NoDir() { + ts.SkipNoAcc() + + dir := ts.MakePluginDir() + defer dir.Cleanup() + + ts.PackerCommand().UsePluginDir(dir). + SetArgs("plugins", "install", "github.com/hashicorp/docker"). + Assert(check.MustSucceed()) + + ts.PackerCommand().UsePluginDir(dir). + SetArgs("build", "./templates/dest_is_dir_with_trailing_slash.pkr.hcl"). + Assert(check.MustSucceed(), check.FileGlob("./sbom/packer-user-sbom-*.json")) + + os.RemoveAll("sbom") +} diff --git a/packer_test/provisioner_tests/hcp-sbom/suite_test.go b/packer_test/provisioner_tests/hcp-sbom/suite_test.go new file mode 100644 index 00000000000..a3855ebb660 --- /dev/null +++ b/packer_test/provisioner_tests/hcp-sbom/suite_test.go @@ -0,0 +1,23 @@ +package plugin_tests + +import ( + "testing" + + "github.com/hashicorp/packer/packer_test/common" + "github.com/stretchr/testify/suite" +) + +type PackerHCPSbomTestSuite struct { + *common.PackerTestSuite +} + +func Test_PackerPluginSuite(t *testing.T) { + baseSuite, cleanup := common.InitBaseSuite(t) + defer cleanup() + + ts := &PackerHCPSbomTestSuite{ + baseSuite, + } + + suite.Run(t, ts) +} diff --git a/packer_test/provisioner_tests/hcp-sbom/templates/dest_is_dir.pkr.hcl b/packer_test/provisioner_tests/hcp-sbom/templates/dest_is_dir.pkr.hcl new file mode 100644 index 00000000000..1a405a50bee --- /dev/null +++ b/packer_test/provisioner_tests/hcp-sbom/templates/dest_is_dir.pkr.hcl @@ -0,0 +1,36 @@ +packer { + required_plugins { + docker = { + version = ">= 1.0.0" + source = "github.com/hashicorp/docker" + } + } +} + +source "docker" "ubuntu" { + image = "ubuntu:20.04" + commit = true +} + +build { + sources = ["source.docker.ubuntu"] + + provisioner "shell" { + inline = [ + "apt-get update -y", + "apt-get install -y curl", + "bash -c \"$(curl -sSL https://install.mondoo.com/sh)\"" + ] + } + + provisioner "shell" { + inline = [ + "cnquery sbom --output cyclonedx-json --output-target /tmp/sbom_cyclonedx.json", + ] + } + + provisioner "hcp-sbom" { + source = "/tmp/sbom_cyclonedx.json" + destination = "./sbom" + } +} diff --git a/packer_test/provisioner_tests/hcp-sbom/templates/dest_is_dir_with_trailing_slash.pkr.hcl b/packer_test/provisioner_tests/hcp-sbom/templates/dest_is_dir_with_trailing_slash.pkr.hcl new file mode 100644 index 00000000000..9d9ca4506b1 --- /dev/null +++ b/packer_test/provisioner_tests/hcp-sbom/templates/dest_is_dir_with_trailing_slash.pkr.hcl @@ -0,0 +1,36 @@ +packer { + required_plugins { + docker = { + version = ">= 1.0.0" + source = "github.com/hashicorp/docker" + } + } +} + +source "docker" "ubuntu" { + image = "ubuntu:20.04" + commit = true +} + +build { + sources = ["source.docker.ubuntu"] + + provisioner "shell" { + inline = [ + "apt-get update -y", + "apt-get install -y curl", + "bash -c \"$(curl -sSL https://install.mondoo.com/sh)\"" + ] + } + + provisioner "shell" { + inline = [ + "cnquery sbom --output cyclonedx-json --output-target /tmp/sbom_cyclonedx.json", + ] + } + + provisioner "hcp-sbom" { + source = "/tmp/sbom_cyclonedx.json" + destination = "./sbom/" + } +} diff --git a/packer_test/provisioner_tests/hcp-sbom/templates/dest_is_file_no_interm_dirs.pkr.hcl b/packer_test/provisioner_tests/hcp-sbom/templates/dest_is_file_no_interm_dirs.pkr.hcl new file mode 100644 index 00000000000..9d4bcb2daec --- /dev/null +++ b/packer_test/provisioner_tests/hcp-sbom/templates/dest_is_file_no_interm_dirs.pkr.hcl @@ -0,0 +1,36 @@ +packer { + required_plugins { + docker = { + version = ">= 1.0.0" + source = "github.com/hashicorp/docker" + } + } +} + +source "docker" "ubuntu" { + image = "ubuntu:20.04" + commit = true +} + +build { + sources = ["source.docker.ubuntu"] + + provisioner "shell" { + inline = [ + "apt-get update -y", + "apt-get install -y curl", + "bash -c \"$(curl -sSL https://install.mondoo.com/sh)\"" + ] + } + + provisioner "shell" { + inline = [ + "cnquery sbom --output cyclonedx-json --output-target /tmp/sbom_cyclonedx.json", + ] + } + + provisioner "hcp-sbom" { + source = "/tmp/sbom_cyclonedx.json" + destination = "./sbom/sbom_cyclonedx.json" + } +} diff --git a/packer_test/provisioner_tests/hcp-sbom/templates/dest_is_file_with_interm_dirs.pkr.hcl b/packer_test/provisioner_tests/hcp-sbom/templates/dest_is_file_with_interm_dirs.pkr.hcl new file mode 100644 index 00000000000..37ccbcc3b60 --- /dev/null +++ b/packer_test/provisioner_tests/hcp-sbom/templates/dest_is_file_with_interm_dirs.pkr.hcl @@ -0,0 +1,36 @@ +packer { + required_plugins { + docker = { + version = ">= 1.0.0" + source = "github.com/hashicorp/docker" + } + } +} + +source "docker" "ubuntu" { + image = "ubuntu:20.04" + commit = true +} + +build { + sources = ["source.docker.ubuntu"] + + provisioner "shell" { + inline = [ + "apt-get update -y", + "apt-get install -y curl", + "bash -c \"$(curl -sSL https://install.mondoo.com/sh)\"" + ] + } + + provisioner "shell" { + inline = [ + "cnquery sbom --output cyclonedx-json --output-target /tmp/sbom_cyclonedx.json", + ] + } + + provisioner "hcp-sbom" { + source = "/tmp/sbom_cyclonedx.json" + destination = "./sbom/sbom_cyclonedx" + } +} diff --git a/packer_test/provisioner_tests/hcp-sbom/templates/source_is_dir.pkr.hcl b/packer_test/provisioner_tests/hcp-sbom/templates/source_is_dir.pkr.hcl new file mode 100644 index 00000000000..02522488d52 --- /dev/null +++ b/packer_test/provisioner_tests/hcp-sbom/templates/source_is_dir.pkr.hcl @@ -0,0 +1,21 @@ +packer { + required_plugins { + docker = { + version = ">= 1.0.0" + source = "github.com/hashicorp/docker" + } + } +} + +source "docker" "ubuntu" { + image = "ubuntu:20.04" + commit = true +} + +build { + sources = ["source.docker.ubuntu"] + + provisioner "hcp-sbom" { + source = "/tmp" + } +} diff --git a/packer_test/provisioner_tests/hcp-sbom/templates/source_not_existing.pkr.hcl b/packer_test/provisioner_tests/hcp-sbom/templates/source_not_existing.pkr.hcl new file mode 100644 index 00000000000..a66b9968501 --- /dev/null +++ b/packer_test/provisioner_tests/hcp-sbom/templates/source_not_existing.pkr.hcl @@ -0,0 +1,21 @@ +packer { + required_plugins { + docker = { + version = ">= 1.0.0" + source = "github.com/hashicorp/docker" + } + } +} + +source "docker" "ubuntu" { + image = "ubuntu:20.04" + commit = true +} + +build { + sources = ["source.docker.ubuntu"] + + provisioner "hcp-sbom" { + source = "/tmp/sbom_cyclonedx.json" + } +} From e6600ea257b7056fc675e600642c9e418b846cce Mon Sep 17 00:00:00 2001 From: Devashish Date: Fri, 1 Nov 2024 12:34:55 -0400 Subject: [PATCH 04/24] website: add docs for the hcp-sbom provisioner --- website/content/community-plugins.mdx | 1 + .../content/docs/provisioners/hcp-sbom.mdx | 130 ++++++++++++++++++ website/content/docs/provisioners/index.mdx | 1 + website/data/docs-nav-data.json | 4 + 4 files changed, 136 insertions(+) create mode 100644 website/content/docs/provisioners/hcp-sbom.mdx diff --git a/website/content/community-plugins.mdx b/website/content/community-plugins.mdx index fa245b73dfd..43a427c3f95 100644 --- a/website/content/community-plugins.mdx +++ b/website/content/community-plugins.mdx @@ -24,6 +24,7 @@ HashiCorp maintainers for advice on how to get started contributing. ## Provisioners - File +- HCP SBOM - InSpec - PowerShell - Shell diff --git a/website/content/docs/provisioners/hcp-sbom.mdx b/website/content/docs/provisioners/hcp-sbom.mdx new file mode 100644 index 00000000000..a90b5bf8886 --- /dev/null +++ b/website/content/docs/provisioners/hcp-sbom.mdx @@ -0,0 +1,130 @@ +--- +description: | + The `hcp-sbom` Packer provisioner downloads an SBOM file from the guest VM and + sends it to HCP Packer when the build is done. +page_title: HCP SBOM - Provisioners +--- + + + + + +# HCP SBOM Provisioner + +Type: `hcp-sbom` + +The `hcp-sbom` Packer provisioner downloads an SBOM file from the guest machine +and sends it to HCP Packer when the build is complete (only if the template is +HCP-enabled). The SBOM file is automatically removed at the end of the process. +If you want to retain a copy of the SBOM file, you can specify the +`destination` option in the provisioner. + +Currently, we support `CycloneDX` and `SPDX` SBOM formats in `JSON`. + +## Basic Example + + +In HCL2: + +```hcl +provisioner "hcp-sbom" { + source = "/tmp/sbom_cyclonedx.json" + destination = "./sbom/sbom_cyclonedx.json" +} +``` + +In JSON: + +```json +{ + "type": "hcp-sbom", + "source": "/tmp/sbom_cyclonedx.json", + "destination": "./sbom/sbom_cyclonedx.json" +} +``` + + + +## Configuration Reference + +Required Parameters: + +@include 'provisioner/hcp-sbom/Config-required.mdx' + +Optional Parameters: + +@include '/provisioner/hcp-sbom/Config-not-required.mdx' + +## Example Usage + + +In HCL2: + +```hcl +packer { + required_plugins { + docker = { + version = ">= 1.0.0" + source = "github.com/hashicorp/docker" + } + } +} + +source "docker" "ubuntu" { + image = "ubuntu:20.04" + commit = true +} + +build { + sources = ["source.docker.ubuntu"] + + hcp_packer_registry { + bucket_name = "test-bucket" + } + + + provisioner "shell" { + inline = [ + "apt-get update -y", + "apt-get install -y curl gpg", + "bash -c \"$(curl -sSL https://install.mondoo.com/sh)\"", + "cnquery sbom --output cyclonedx-json --output-target /tmp/sbom_cyclonedx.json", + ] + } + + provisioner "hcp-sbom" { + source = "/tmp/sbom_cyclonedx.json" + destination = "./sbom" + } +} +``` + +In JSON: + +```json +{ + "builders": [ + { + "type": "docker", + "image": "ubuntu:20.04", + "commit": true + } + ], + "provisioners": [ + { + "type": "shell", + "inline": [ + "apt-get update -y", + "apt-get install -y curl", + "bash -c \"$(curl -sSL https://install.mondoo.com/sh)\"", + "cnquery sbom --output cyclonedx-json --output-target /tmp/sbom_cyclonedx.json" + ] + }, + { + "type": "hcp-sbom", + "source": "/tmp/sbom_cyclonedx.json", + "destination": "./sbom" + } + ] +} +``` diff --git a/website/content/docs/provisioners/index.mdx b/website/content/docs/provisioners/index.mdx index e6144beaef4..6be0124557f 100644 --- a/website/content/docs/provisioners/index.mdx +++ b/website/content/docs/provisioners/index.mdx @@ -20,6 +20,7 @@ The following provisioners are included with Packer: - [Breakpoint](/packer/docs/provisioners/breakpoint) - pause until the user presses `Enter` to resume a build. - [File](/packer/docs/provisioners/file) - upload files to machines image during a build. +- [HCP SBOM](/packer/docs/provisioners/hcp-sbom) - download SBOM file to machines and send to HCP Packer during a build. - [Shell](/packer/docs/provisioners/shell) - run shell scripts on the machines image during a build. - [Local Shell](/packer/docs/provisioners/shell-local) - run shell scripts on the host running Packer during a build. diff --git a/website/data/docs-nav-data.json b/website/data/docs-nav-data.json index 24c55feb678..481ecfe60b0 100644 --- a/website/data/docs-nav-data.json +++ b/website/data/docs-nav-data.json @@ -784,6 +784,10 @@ "title": "File", "path": "provisioners/file" }, + { + "title": "HCP SBOM", + "path": "provisioners/hcp-sbom" + }, { "title": "PowerShell", "path": "provisioners/powershell" From 9a29c7a066033f4a1eb875f7487553af6712656f Mon Sep 17 00:00:00 2001 From: Lucas Bajolet Date: Mon, 11 Nov 2024 11:16:40 -0500 Subject: [PATCH 05/24] hcp-sbom: remove packerDestination function Since the function did very little, and the code was called once in the provisioner, we remove the function itself and move the code over to the provisioner's Provision function. Also, since the output directory is prepared and its lifecycle is managed by Packer Core, we should not try to make the missing directories here, and instead rely on Packer core's code before calling the provisioner to do so. --- provisioner/hcp-sbom/provisioner.go | 35 +++++++---------------------- 1 file changed, 8 insertions(+), 27 deletions(-) diff --git a/provisioner/hcp-sbom/provisioner.go b/provisioner/hcp-sbom/provisioner.go index 2fd9e834e98..5c1c936858f 100644 --- a/provisioner/hcp-sbom/provisioner.go +++ b/provisioner/hcp-sbom/provisioner.go @@ -100,33 +100,29 @@ func (p *Provisioner) Provision( func (p *Provisioner) downloadAndValidateSBOM( ui packersdk.Ui, comm packersdk.Communicator, generatedData map[string]interface{}, ) error { - src, err := interpolate.Render(p.config.Source, &p.config.ctx) - if err != nil { - return fmt.Errorf("error interpolating SBOM source: %s", err) + src := p.config.Source + + pkrDst := generatedData["dst"].(string) + if pkrDst == "" { + return fmt.Errorf("packer destination path missing from configs: this is an internal error, which should be reported to be fixed.") } var buf bytes.Buffer - if err = comm.Download(src, &buf); err != nil { + if err := comm.Download(src, &buf); err != nil { ui.Errorf("download failed for SBOM file: %s", err) return err } reader := bytes.NewReader(buf.Bytes()) - if _, err = ValidateSBOM(reader); err != nil { + if _, err := ValidateSBOM(reader); err != nil { ui.Errorf("validation failed for SBOM file: %s", err) return err } - _, err = reader.Seek(0, io.SeekStart) + _, err := reader.Seek(0, io.SeekStart) if err != nil { return err } - // SBOM for Packer - pkrDst, err := p.getPackerDestination(generatedData) - if err != nil { - return fmt.Errorf("failed to get Packer SBOM destination: %s", err) - } - err = p.writeToFile(reader, pkrDst) if err != nil { return fmt.Errorf("failed to download Packer SBOM: %s", err) @@ -207,21 +203,6 @@ func (p *Provisioner) getUserDestination() (string, error) { return dst, nil } -// getPackerDestination retrieves the destination path for the Packer SBOM file. -func (p *Provisioner) getPackerDestination(generatedData map[string]interface{}) (string, error) { - dst, ok := generatedData["dst"].(string) // This has been set by HCPSBOMInternalProvisioner.Provision - if !ok || dst == "" { - return "", fmt.Errorf("destination path for Packer SBOM file is not valid") - } - - // Ensure the destination directory exists - if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil { - return "", fmt.Errorf("failed to create destination directory for Packer SBOM: %w", err) - } - - return dst, nil -} - func (p *Provisioner) writeToFile(buf *bytes.Reader, dst string) error { // Open the destination file f, err := os.OpenFile(dst, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644) From b72da6af4397709c2bb4a945cb732095eb8d45af Mon Sep 17 00:00:00 2001 From: Lucas Bajolet Date: Mon, 11 Nov 2024 11:29:37 -0500 Subject: [PATCH 06/24] hcp-sbom: rely less on io.Readers Using io.Readers from a file's contents is not a bad idea inherently, but since we're forced to reset the reader periodically, this makes manipulation more complex because of the extra function call, error checking, and extra usage errors. To simplify how we're working with that data, we're passing around the raw file contents, so they're encapsulated into readers on-demand, when needed only, which is a rather costless operation as the bytes.Reader reads directly from the slice, maintaining its own offsets, therefore meaning that resetting it is not necessary, since we can as easily discard it, without risking failure. --- packer/provisioner.go | 5 ++--- provisioner/hcp-sbom/provisioner.go | 19 ++++--------------- provisioner/hcp-sbom/validate.go | 28 +++++++--------------------- 3 files changed, 13 insertions(+), 39 deletions(-) diff --git a/packer/provisioner.go b/packer/provisioner.go index 24d670950a1..f6c6cc32527 100644 --- a/packer/provisioner.go +++ b/packer/provisioner.go @@ -322,13 +322,12 @@ func (p *SBOMInternalProvisioner) compressFile(filePath string) ([]byte, error) } func (p *SBOMInternalProvisioner) getSBOMFormat(filePath string) (string, error) { - file, err := os.Open(filePath) + fileContent, err := os.ReadFile(filePath) if err != nil { return "", fmt.Errorf("failed to open SBOM file %s: %w", filePath, err) } - defer file.Close() - format, err := hcpSbomProvisioner.ValidateSBOM(file) + format, err := hcpSbomProvisioner.ValidateSBOM(fileContent) if err != nil { return "", fmt.Errorf("failed to detect SBOM format: %w", err) } diff --git a/provisioner/hcp-sbom/provisioner.go b/provisioner/hcp-sbom/provisioner.go index 5c1c936858f..4b86189f14c 100644 --- a/provisioner/hcp-sbom/provisioner.go +++ b/provisioner/hcp-sbom/provisioner.go @@ -11,7 +11,6 @@ import ( "context" "errors" "fmt" - "io" "log" "os" @@ -113,25 +112,15 @@ func (p *Provisioner) downloadAndValidateSBOM( return err } - reader := bytes.NewReader(buf.Bytes()) - if _, err := ValidateSBOM(reader); err != nil { + if _, err := ValidateSBOM(buf.Bytes()); err != nil { ui.Errorf("validation failed for SBOM file: %s", err) return err } - _, err := reader.Seek(0, io.SeekStart) - if err != nil { - return err - } - err = p.writeToFile(reader, pkrDst) + err := p.writeToFile(bytes.NewReader(buf.Bytes()), pkrDst) if err != nil { - return fmt.Errorf("failed to download Packer SBOM: %s", err) - } - _, err = reader.Seek(0, io.SeekStart) - if err != nil { - return err + return fmt.Errorf("failed to write HCP's SBOM: %s", err) } - log.Printf("Packer SBOM file successfully downloaded to: %s\n", pkrDst) // SBOM for User usrDst, err := p.getUserDestination() @@ -140,7 +129,7 @@ func (p *Provisioner) downloadAndValidateSBOM( } if usrDst != "" { - err = p.writeToFile(reader, usrDst) + err = p.writeToFile(bytes.NewReader(buf.Bytes()), usrDst) if err != nil { return fmt.Errorf("failed to download User SBOM: %s", err) } diff --git a/provisioner/hcp-sbom/validate.go b/provisioner/hcp-sbom/validate.go index cad502cb934..f1ad5edac5d 100644 --- a/provisioner/hcp-sbom/validate.go +++ b/provisioner/hcp-sbom/validate.go @@ -7,8 +7,6 @@ import ( "github.com/CycloneDX/cyclonedx-go" spdxjson "github.com/spdx/tools-golang/json" - - "io" ) // ValidationError represents an error encountered while validating an SBOM. @@ -25,8 +23,8 @@ func (e *ValidationError) Unwrap() error { } // ValidateCycloneDX is a validation for CycloneDX in JSON format. -func ValidateCycloneDX(content io.Reader) error { - decoder := cyclonedx.NewBOMDecoder(content, cyclonedx.BOMFileFormatJSON) +func ValidateCycloneDX(content []byte) error { + decoder := cyclonedx.NewBOMDecoder(bytes.NewBuffer(content), cyclonedx.BOMFileFormatJSON) bom := new(cyclonedx.BOM) if err := decoder.Decode(bom); err != nil { return fmt.Errorf("error parsing CycloneDX SBOM: %w", err) @@ -47,8 +45,8 @@ func ValidateCycloneDX(content io.Reader) error { } // ValidateSPDX is a validation for SPDX in JSON format. -func ValidateSPDX(content io.Reader) error { - doc, err := spdxjson.Read(content) +func ValidateSPDX(content []byte) error { + doc, err := spdxjson.Read(bytes.NewBuffer(content)) if err != nil { return fmt.Errorf("error parsing SPDX JSON file: %w", err) } @@ -63,16 +61,9 @@ func ValidateSPDX(content io.Reader) error { } // ValidateSBOM validates the SBOM file and returns the format of the SBOM. -func ValidateSBOM(content io.Reader) (string, error) { - var buf bytes.Buffer - if _, err := io.Copy(&buf, content); err != nil { - return "", fmt.Errorf("failed to copy content: %s", err) - } - - reader := bytes.NewReader(buf.Bytes()) - +func ValidateSBOM(content []byte) (string, error) { // Try validating as SPDX - spdxErr := ValidateSPDX(reader) + spdxErr := ValidateSPDX(content) if spdxErr == nil { return "spdx", nil } @@ -81,12 +72,7 @@ func ValidateSBOM(content io.Reader) (string, error) { return "", vErr } - // Reset the reader's position - if _, err := reader.Seek(0, io.SeekStart); err != nil { - return "", fmt.Errorf("failed to reset reader: %s", err) - } - - cycloneDxErr := ValidateCycloneDX(reader) + cycloneDxErr := ValidateCycloneDX(content) if cycloneDxErr == nil { return "cyclonedx", nil } From 7b9073e6da1f308ea14326f3304b5d36b1088574 Mon Sep 17 00:00:00 2001 From: Lucas Bajolet Date: Mon, 11 Nov 2024 13:26:45 -0500 Subject: [PATCH 07/24] hcp-sbom: use a JSON-encoded output for Packer When outputting the data from the provisioner so Packer can consume it, we are writing to a file. The only constraints on this file are that Packer manages its lifecycle, and that Packer core and the provisioner are synced-up in terms of the contents and the structure of the file. Since we are outputting the file, and its format, we can bundle the two together, and export them under a structure that both the provisioner and Packer core have access to, so we can then fill-in the blanks, and write a serialised version of this structure to disk. This is the approach taken in this commit. First we clean-up the existing code a bit, since some abstractions were a bit hasty, and did not necessarily make sense in all cases, so the code is consolidated under `Provision`, and we use that structure then to fill-in the relevant information before serialising it to JSON and writing it in the output file provided by Packer core. --- packer/provisioner.go | 43 ++++----------- provisioner/hcp-sbom/provisioner.go | 86 ++++++++++++----------------- 2 files changed, 46 insertions(+), 83 deletions(-) diff --git a/packer/provisioner.go b/packer/provisioner.go index f6c6cc32527..2fececa6d9a 100644 --- a/packer/provisioner.go +++ b/packer/provisioner.go @@ -5,6 +5,7 @@ package packer import ( "context" + "encoding/json" "fmt" "log" "os" @@ -290,47 +291,23 @@ func (p *SBOMInternalProvisioner) Provision( return err } - sbomFormat, err := p.getSBOMFormat(tmpFile.Name()) + packerSbom, err := os.Open(tmpFileName) if err != nil { - return err + return fmt.Errorf("failed to open Packer SBOM file %q: %s", tmpFileName, err) } - compressedData, err := p.compressFile(tmpFile.Name()) + provisionerOut := &hcpSbomProvisioner.PackerSBOM{} + err = json.NewDecoder(packerSbom).Decode(provisionerOut) if err != nil { - return err - } - p.CompressedData = compressedData - p.SBOMFormat = sbomFormat - return nil -} - -func (p *SBOMInternalProvisioner) compressFile(filePath string) ([]byte, error) { - data, err := os.ReadFile(filePath) - if err != nil { - return nil, fmt.Errorf("failed to read file %s: %w", filePath, err) + return fmt.Errorf("malformed packer SBOM output from file %q: %s", tmpFileName, err) } encoder, err := zstd.NewWriter(nil, zstd.WithEncoderLevel(zstd.SpeedBestCompression)) if err != nil { - return nil, fmt.Errorf("failed to create zstd encoder: %w", err) + return fmt.Errorf("failed to create zstd encoder: %s", err) } + p.CompressedData = encoder.EncodeAll(provisionerOut.RawSBOM, nil) + p.SBOMFormat = provisionerOut.Format - compressedData := encoder.EncodeAll(data, nil) - - log.Printf("SBOM file compressed successfully. Size: %d bytes\n", len(compressedData)) - return compressedData, nil -} - -func (p *SBOMInternalProvisioner) getSBOMFormat(filePath string) (string, error) { - fileContent, err := os.ReadFile(filePath) - if err != nil { - return "", fmt.Errorf("failed to open SBOM file %s: %w", filePath, err) - } - - format, err := hcpSbomProvisioner.ValidateSBOM(fileContent) - if err != nil { - return "", fmt.Errorf("failed to detect SBOM format: %w", err) - } - - return format, nil + return nil } diff --git a/provisioner/hcp-sbom/provisioner.go b/provisioner/hcp-sbom/provisioner.go index 4b86189f14c..b7814ee12a4 100644 --- a/provisioner/hcp-sbom/provisioner.go +++ b/provisioner/hcp-sbom/provisioner.go @@ -9,6 +9,7 @@ package hcp_sbom import ( "bytes" "context" + "encoding/json" "errors" "fmt" "log" @@ -76,6 +77,17 @@ func (p *Provisioner) Prepare(raws ...interface{}) error { return nil } +// PackerSBOM is the type we write to the temporary JSON dump of the SBOM to +// be consumed by Packer core +type PackerSBOM struct { + // RawSBOM is the raw data from the SBOM downloaded from the guest + RawSBOM []byte `json:"raw_sbom"` + // Format is the format detected by the provisioner + // + // Supported values: `spdx` or `cyclonedx` + Format string `json:"format"` +} + func (p *Provisioner) Provision( ctx context.Context, ui packersdk.Ui, comm packersdk.Communicator, generatedData map[string]interface{}, @@ -87,18 +99,6 @@ func (p *Provisioner) Provision( } p.config.ctx.Data = generatedData - downloadErr := p.downloadAndValidateSBOM(ui, comm, generatedData) - if downloadErr != nil { - return fmt.Errorf("failed to download SBOM file: %w", downloadErr) - } - - return nil -} - -// downloadAndValidateSBOM handles downloading SBOM files for the User and Packer. -func (p *Provisioner) downloadAndValidateSBOM( - ui packersdk.Ui, comm packersdk.Communicator, generatedData map[string]interface{}, -) error { src := p.config.Source pkrDst := generatedData["dst"].(string) @@ -112,44 +112,46 @@ func (p *Provisioner) downloadAndValidateSBOM( return err } - if _, err := ValidateSBOM(buf.Bytes()); err != nil { - ui.Errorf("validation failed for SBOM file: %s", err) - return err + format, err := ValidateSBOM(buf.Bytes()) + if err != nil { + return fmt.Errorf("validation failed for SBOM file: %s", err) + } + + outFile, err := os.Create(pkrDst) + if err != nil { + return fmt.Errorf("failed to open/create output file %q: %s", pkrDst, err) } + defer outFile.Close() - err := p.writeToFile(bytes.NewReader(buf.Bytes()), pkrDst) + err = json.NewEncoder(outFile).Encode(PackerSBOM{ + RawSBOM: buf.Bytes(), + Format: format, + Name: p.config.SbomName, + }) if err != nil { - return fmt.Errorf("failed to write HCP's SBOM: %s", err) + return fmt.Errorf("failed to write sbom file to %q: %s", pkrDst, err) + } + + if p.config.Destination == "" { + return nil } // SBOM for User usrDst, err := p.getUserDestination() if err != nil { - return fmt.Errorf("failed to determine user SBOM destination: %s", err) + return fmt.Errorf("failed to compute destination path %q: %s", p.config.Destination, err) } - - if usrDst != "" { - err = p.writeToFile(bytes.NewReader(buf.Bytes()), usrDst) - if err != nil { - return fmt.Errorf("failed to download User SBOM: %s", err) - } - log.Printf("User SBOM file successfully downloaded to: %s\n", usrDst) + err = os.WriteFile(usrDst, buf.Bytes(), 0644) + if err != nil { + return fmt.Errorf("failed to write SBOM to destination %q: %s", usrDst, err) } + return nil } // getUserDestination determines and returns the destination path for the user SBOM file. func (p *Provisioner) getUserDestination() (string, error) { dst := p.config.Destination - if dst == "" { - log.Println("skipped downloading user SBOM file because 'Destination' is not provided") - return "", nil - } - - dst, err := interpolate.Render(dst, &p.config.ctx) - if err != nil { - return "", fmt.Errorf("error interpolating SBOM file destination for user: %s", err) - } // Check if the destination exists and determine its type info, err := os.Stat(dst) @@ -191,19 +193,3 @@ func (p *Provisioner) getUserDestination() (string, error) { return dst, nil } - -func (p *Provisioner) writeToFile(buf *bytes.Reader, dst string) error { - // Open the destination file - f, err := os.OpenFile(dst, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644) - if err != nil { - return fmt.Errorf("failed to open destination file for SBOM: %s", err) - } - defer f.Close() - - // Write the buffer content to the destination file - if _, err = buf.WriteTo(f); err != nil { - return err - } - - return nil -} From eb8c78ae683d06868038b7692d52c19109f3978e Mon Sep 17 00:00:00 2001 From: Lucas Bajolet Date: Mon, 11 Nov 2024 09:48:40 -0500 Subject: [PATCH 08/24] hcp-sbom: add sbom_name attribute Since we are uploading multiple SBOMs possibly for a single build, we need to generate names for them, so users are able to differentiate between those artifacts. This commit adds an optional property `sbom_name`, that users can provide in the configuration for this provisioner, which is then used by Packer core for uploading the data to HCP Packer later on. --- packer/provisioner.go | 2 + provisioner/hcp-sbom/provisioner.go | 46 +++++++++++++++++-- provisioner/hcp-sbom/provisioner.hcl2spec.go | 2 + .../hcp-sbom/Config-not-required.mdx | 10 ++++ 4 files changed, 55 insertions(+), 5 deletions(-) diff --git a/packer/provisioner.go b/packer/provisioner.go index 2fececa6d9a..24e20b3a247 100644 --- a/packer/provisioner.go +++ b/packer/provisioner.go @@ -250,6 +250,7 @@ type SBOMInternalProvisioner struct { Provisioner packersdk.Provisioner CompressedData []byte SBOMFormat string + SBOMName string } func (p *SBOMInternalProvisioner) ConfigSpec() hcldec.ObjectSpec { return p.ConfigSpec() } @@ -308,6 +309,7 @@ func (p *SBOMInternalProvisioner) Provision( } p.CompressedData = encoder.EncodeAll(provisionerOut.RawSBOM, nil) p.SBOMFormat = provisionerOut.Format + p.SBOMName = provisionerOut.Name return nil } diff --git a/provisioner/hcp-sbom/provisioner.go b/provisioner/hcp-sbom/provisioner.go index b7814ee12a4..14e5d6fb4dc 100644 --- a/provisioner/hcp-sbom/provisioner.go +++ b/provisioner/hcp-sbom/provisioner.go @@ -14,6 +14,8 @@ import ( "fmt" "log" "os" + "regexp" + "strings" "path/filepath" @@ -41,7 +43,17 @@ type Config struct { // a "Permission Denied" error will occur. If the source path is a file, // it is recommended that the destination path be a file as well. Destination string `mapstructure:"destination"` - ctx interpolate.Context + // The name to give the SBOM when uploaded on HCP Packer + // + // By default this will be generated, but if you prefer to have a name + // of your choosing, you can enter it here. + // The name must match the following regexp: `[a-zA-Z0-9_-]{3,36}` + // + // Note: it must be unique for a single build, otherwise the build will + // fail when uploading the SBOMs to HCP Packer, and so will the Packer + // build command. + SbomName string `mapstructure:"sbom_name"` + ctx interpolate.Context } type Provisioner struct { @@ -52,6 +64,8 @@ func (p *Provisioner) ConfigSpec() hcldec.ObjectSpec { return p.config.FlatMapstructure().HCL2Spec() } +var sbomFormatRegexp = regexp.MustCompile("^[0-9A-Za-z-]{3,36}$") + func (p *Provisioner) Prepare(raws ...interface{}) error { err := config.Decode(&p.config, &config.DecodeOpts{ PluginType: "hcp-sbom", @@ -65,16 +79,34 @@ func (p *Provisioner) Prepare(raws ...interface{}) error { return err } - var errs *packersdk.MultiError + var errs error + if p.config.Source == "" { errs = packersdk.MultiErrorAppend(errs, errors.New("source must be specified")) } - if errs != nil && len(errs.Errors) > 0 { - return errs + if p.config.SbomName != "" && !sbomFormatRegexp.MatchString(p.config.SbomName) { + // Ugly but a bit of a problem with interpolation since Provisioners + // are prepared twice in HCL2. + // + // If the information used for interpolating is populated in-between the + // first call to Prepare (at the start of the build), and when the + // Provisioner is actually called, the first call will fail, as + // the value won't contain the actual interpolated value, but a + // placeholder which doesn't match the regex. + // + // Since we don't have a way to discriminate between the calls + // in the context of the provisioner, we ignore them, and later the + // HCP Packer call will fail because of the broken regex. + if strings.Contains(p.config.SbomName, "") { + log.Printf("[WARN] interpolation incomplete for `sbom_name`, will possibly retry later with data populated into context, otherwise will fail when uploading to HCP Packer.") + } else { + errs = packersdk.MultiErrorAppend(errs, fmt.Errorf("`sbom_name` %q doesn't match the expected format, it must "+ + "contain between 3 and 36 characters, all from the following set: [A-Za-z0-9_-]", p.config.SbomName)) + } } - return nil + return errs } // PackerSBOM is the type we write to the temporary JSON dump of the SBOM to @@ -86,6 +118,10 @@ type PackerSBOM struct { // // Supported values: `spdx` or `cyclonedx` Format string `json:"format"` + // Name is the name of the SBOM to be set on HCP Packer + // + // If unset, HCP Packer will generate one + Name string `json:"name,omitempty"` } func (p *Provisioner) Provision( diff --git a/provisioner/hcp-sbom/provisioner.hcl2spec.go b/provisioner/hcp-sbom/provisioner.hcl2spec.go index e6ee5d22753..4df5397c093 100644 --- a/provisioner/hcp-sbom/provisioner.hcl2spec.go +++ b/provisioner/hcp-sbom/provisioner.hcl2spec.go @@ -20,6 +20,7 @@ type FlatConfig struct { PackerSensitiveVars []string `mapstructure:"packer_sensitive_variables" cty:"packer_sensitive_variables" hcl:"packer_sensitive_variables"` Source *string `mapstructure:"source" required:"true" cty:"source" hcl:"source"` Destination *string `mapstructure:"destination" cty:"destination" hcl:"destination"` + SbomName *string `mapstructure:"sbom_name" cty:"sbom_name" hcl:"sbom_name"` } // FlatMapstructure returns a new FlatConfig. @@ -44,6 +45,7 @@ func (*FlatConfig) HCL2Spec() map[string]hcldec.Spec { "packer_sensitive_variables": &hcldec.AttrSpec{Name: "packer_sensitive_variables", Type: cty.List(cty.String), Required: false}, "source": &hcldec.AttrSpec{Name: "source", Type: cty.String, Required: false}, "destination": &hcldec.AttrSpec{Name: "destination", Type: cty.String, Required: false}, + "sbom_name": &hcldec.AttrSpec{Name: "sbom_name", Type: cty.String, Required: false}, } return s } diff --git a/website/content/partials/provisioner/hcp-sbom/Config-not-required.mdx b/website/content/partials/provisioner/hcp-sbom/Config-not-required.mdx index eb241a06c77..871e7a5adeb 100644 --- a/website/content/partials/provisioner/hcp-sbom/Config-not-required.mdx +++ b/website/content/partials/provisioner/hcp-sbom/Config-not-required.mdx @@ -10,4 +10,14 @@ a "Permission Denied" error will occur. If the source path is a file, it is recommended that the destination path be a file as well. +- `sbom_name` (string) - The name to give the SBOM when uploaded on HCP Packer + + By default this will be generated, but if you prefer to have a name + of your choosing, you can enter it here. + The name must match the following regexp: `[a-zA-Z0-9_-]{3,36}` + + Note: it must be unique for a single build, otherwise the build will + fail when uploading the SBOMs to HCP Packer, and so will the Packer + build command. + From 95861f9464a28ec11c928b9cd4703a5eb1b1cde1 Mon Sep 17 00:00:00 2001 From: Lucas Bajolet Date: Mon, 11 Nov 2024 13:46:58 -0500 Subject: [PATCH 09/24] hcp-sbom: make validation functions private Since the SBOM validation functions are now called only from the provisioner itself, they don't need to be public anymore, so we make them private to the package. --- provisioner/hcp-sbom/provisioner.go | 2 +- provisioner/hcp-sbom/validate.go | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/provisioner/hcp-sbom/provisioner.go b/provisioner/hcp-sbom/provisioner.go index 14e5d6fb4dc..cbc515c13d1 100644 --- a/provisioner/hcp-sbom/provisioner.go +++ b/provisioner/hcp-sbom/provisioner.go @@ -148,7 +148,7 @@ func (p *Provisioner) Provision( return err } - format, err := ValidateSBOM(buf.Bytes()) + format, err := validateSBOM(buf.Bytes()) if err != nil { return fmt.Errorf("validation failed for SBOM file: %s", err) } diff --git a/provisioner/hcp-sbom/validate.go b/provisioner/hcp-sbom/validate.go index f1ad5edac5d..4f17a4ac0de 100644 --- a/provisioner/hcp-sbom/validate.go +++ b/provisioner/hcp-sbom/validate.go @@ -23,7 +23,7 @@ func (e *ValidationError) Unwrap() error { } // ValidateCycloneDX is a validation for CycloneDX in JSON format. -func ValidateCycloneDX(content []byte) error { +func validateCycloneDX(content []byte) error { decoder := cyclonedx.NewBOMDecoder(bytes.NewBuffer(content), cyclonedx.BOMFileFormatJSON) bom := new(cyclonedx.BOM) if err := decoder.Decode(bom); err != nil { @@ -44,8 +44,8 @@ func ValidateCycloneDX(content []byte) error { return nil } -// ValidateSPDX is a validation for SPDX in JSON format. -func ValidateSPDX(content []byte) error { +// validateSPDX is a validation for SPDX in JSON format. +func validateSPDX(content []byte) error { doc, err := spdxjson.Read(bytes.NewBuffer(content)) if err != nil { return fmt.Errorf("error parsing SPDX JSON file: %w", err) @@ -60,10 +60,10 @@ func ValidateSPDX(content []byte) error { return nil } -// ValidateSBOM validates the SBOM file and returns the format of the SBOM. -func ValidateSBOM(content []byte) (string, error) { +// validateSBOM validates the SBOM file and returns the format of the SBOM. +func validateSBOM(content []byte) (string, error) { // Try validating as SPDX - spdxErr := ValidateSPDX(content) + spdxErr := validateSPDX(content) if spdxErr == nil { return "spdx", nil } @@ -72,7 +72,7 @@ func ValidateSBOM(content []byte) (string, error) { return "", vErr } - cycloneDxErr := ValidateCycloneDX(content) + cycloneDxErr := validateCycloneDX(content) if cycloneDxErr == nil { return "cyclonedx", nil } From 609e031bb81428ebf8b57678eca78adc083d645a Mon Sep 17 00:00:00 2001 From: Lucas Bajolet Date: Mon, 11 Nov 2024 13:47:49 -0500 Subject: [PATCH 10/24] hcp-sbom: add prepare tests for the provisioner --- provisioner/hcp-sbom/provisioner_test.go | 86 ++++++++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100644 provisioner/hcp-sbom/provisioner_test.go diff --git a/provisioner/hcp-sbom/provisioner_test.go b/provisioner/hcp-sbom/provisioner_test.go new file mode 100644 index 00000000000..aff0323e04a --- /dev/null +++ b/provisioner/hcp-sbom/provisioner_test.go @@ -0,0 +1,86 @@ +package hcp_sbom + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/hashicorp/packer-plugin-sdk/template/interpolate" +) + +func TestConfigPrepare(t *testing.T) { + tests := []struct { + name string + inputConfig map[string]interface{} + interpolateContext interpolate.Context + expectConfig *Config + expectError bool + }{ + { + "empty config, should error without a source", + map[string]interface{}{}, + interpolate.Context{}, + nil, + true, + }, + { + "config with full context for interpolation: success", + map[string]interface{}{ + "source": "{{ .Name }}", + }, + interpolate.Context{ + Data: &struct { + Name string + }{ + Name: "testInterpolate", + }, + }, + &Config{ + Source: "testInterpolate", + }, + false, + }, + { + // Note: this will look weird to reviewers, but is actually + // expected for the moment. + // Refer to the comment in `Prepare` for context as to WHY + // this cannot be considered an error. + "config with sbom name as interpolated value, without it in context, replace with a placeholder", + map[string]interface{}{ + "source": "test", + "sbom_name": "{{ .Name }}", + }, + interpolate.Context{}, + &Config{ + Source: "test", + SbomName: "", + }, + false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + prov := &Provisioner{} + prov.config.ctx = tt.interpolateContext + err := prov.Prepare(tt.inputConfig) + if err != nil && !tt.expectError { + t.Fatalf("configuration unexpectedly failed to prepare: %s", err) + } + + if err == nil && tt.expectError { + t.Fatalf("configuration succeeded to prepare, but should have failed") + } + + if err != nil { + t.Logf("config had error %q", err) + return + } + + diff := cmp.Diff(prov.config, *tt.expectConfig, cmpopts.IgnoreUnexported(Config{})) + if diff != "" { + t.Errorf("configuration returned by `Prepare` is different from what was expected: %s", diff) + } + }) + } +} From dfd71476235676f8ccd86ddc005967a54bca29f9 Mon Sep 17 00:00:00 2001 From: Jenna Goldstrich Date: Fri, 27 Sep 2024 13:51:13 -0700 Subject: [PATCH 11/24] hcp: integrate SBOM upload to HCP code Since packer now supports keeping track of SBOMs produced during a build, we add the code to integrate those changes into the internal/hcp package, so we do upload them on build completion. --- internal/hcp/registry/types.bucket.go | 39 ++++++++++++++++++++++++++ internal/hcp/registry/types.builds.go | 4 +++ internal/hcp/registry/types.version.go | 3 ++ packer/build.go | 2 ++ 4 files changed, 48 insertions(+) diff --git a/internal/hcp/registry/types.bucket.go b/internal/hcp/registry/types.bucket.go index 17f3e34c027..ab068a30866 100644 --- a/internal/hcp/registry/types.bucket.go +++ b/internal/hcp/registry/types.bucket.go @@ -12,7 +12,10 @@ import ( "sync" "time" + "github.com/hashicorp/packer/packer" + "github.com/hashicorp/go-multierror" + "github.com/hashicorp/hcp-sdk-go/clients/cloud-packer-service/stable/2023-01-01/client/packer_service" hcpPackerModels "github.com/hashicorp/hcp-sdk-go/clients/cloud-packer-service/stable/2023-01-01/models" packerSDK "github.com/hashicorp/packer-plugin-sdk/packer" packerSDKRegistry "github.com/hashicorp/packer-plugin-sdk/packer/registry/image" @@ -222,6 +225,35 @@ func (bucket *Bucket) UpdateBuildStatus( return nil } +func (bucket *Bucket) uploadSbom(ctx context.Context, buildName string, sbom packer.SBOM) error { + buildToUpdate, err := bucket.Version.Build(buildName) + if err != nil { + return err + } + + log.Println( + "[TRACE] jennajenna uploadsbom called", buildToUpdate.ID, + ) + if buildToUpdate.ID == "" { + return fmt.Errorf("the build for the component %q does not have a valid id", buildName) + } + _, err = bucket.client.Packer.PackerServiceUploadSbom( + &packer_service.PackerServiceUploadSbomParams{ + Context: ctx, + BucketName: bucket.Name, + Fingerprint: bucket.Version.Fingerprint, + BuildID: buildToUpdate.ID, + Body: &hcpPackerModels.HashicorpCloudPacker20230101UploadSbomBody{ + CompressedSbom: sbom.CompressedData, + Name: sbom.Name, + Format: sbom.Format, + }, + }, + nil, + ) + return err +} + // markBuildComplete should be called to set a build on the HCP Packer registry to DONE. // Upon a successful call markBuildComplete will publish all artifacts created by the named build, // and set the build to done. A build with no artifacts can not be set to DONE. @@ -673,6 +705,13 @@ func (bucket *Bucket) completeBuild( } } + for _, sbom := range build.CompressedSboms { + err = bucket.uploadSbom(ctx, buildName, sbom) + if err != nil { + return packerSDKArtifacts, fmt.Errorf("Failed to upload sboms %s", err) + } + } + parErr := bucket.markBuildComplete(ctx, buildName) if parErr != nil { return packerSDKArtifacts, fmt.Errorf( diff --git a/internal/hcp/registry/types.builds.go b/internal/hcp/registry/types.builds.go index dc7e132762c..0ca531c2c1c 100644 --- a/internal/hcp/registry/types.builds.go +++ b/internal/hcp/registry/types.builds.go @@ -6,6 +6,8 @@ package registry import ( "fmt" + "github.com/hashicorp/packer/packer" + hcpPackerModels "github.com/hashicorp/hcp-sdk-go/clients/cloud-packer-service/stable/2023-01-01/models" packerSDKRegistry "github.com/hashicorp/packer-plugin-sdk/packer/registry/image" ) @@ -20,6 +22,8 @@ type Build struct { Artifacts map[string]packerSDKRegistry.Image Status hcpPackerModels.HashicorpCloudPacker20230101BuildStatus Metadata hcpPackerModels.HashicorpCloudPacker20230101BuildMetadata + + CompressedSboms []packer.SBOM } // NewBuildFromCloudPackerBuild converts a HashicorpCloudPackerBuild to a local build that can be tracked and diff --git a/internal/hcp/registry/types.version.go b/internal/hcp/registry/types.version.go index 0caf6229c11..819e09e4602 100644 --- a/internal/hcp/registry/types.version.go +++ b/internal/hcp/registry/types.version.go @@ -205,5 +205,8 @@ func (version *Version) AddMetadataToBuild( buildToUpdate.Metadata.Vcs = globalMetadata.Vcs buildToUpdate.Metadata.Cicd = globalMetadata.Cicd + // TODO IMO this shouldn't be metadata + buildToUpdate.CompressedSboms = buildMetadata.SBOMs + return nil } diff --git a/packer/build.go b/packer/build.go index eade2625dd6..d23637f67a2 100644 --- a/packer/build.go +++ b/packer/build.go @@ -55,6 +55,7 @@ type CoreBuild struct { } type SBOM struct { + Name string Format string CompressedData []byte } @@ -313,6 +314,7 @@ func (b *CoreBuild) Run(ctx context.Context, originalUi packersdk.Ui) ([]packers sbomInternalProvisioner, ok := p.Provisioner.(*SBOMInternalProvisioner) if ok { sbom := SBOM{ + Name: sbomInternalProvisioner.SBOMName, Format: sbomInternalProvisioner.SBOMFormat, CompressedData: sbomInternalProvisioner.CompressedData, } From 076a43833119d206e73896adbad623d5c641cf35 Mon Sep 17 00:00:00 2001 From: Lucas Bajolet Date: Tue, 12 Nov 2024 13:36:04 -0500 Subject: [PATCH 12/24] hcp: wrap completeBuild to mark as failed on error When a build cannot be completed without errors, the build state was left as running, unless the build explicitly failed, which meant that HCP Packer would be responsible for changing the status after the heartbeats for the build stopped being sent for two 5m periods. This commit changes this behaviour, by explicitly marking the build as failed if something did not work while trying to complete a build on HCP Packer, even if the local Packer core build succeeded before that. --- internal/hcp/registry/types.bucket.go | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/internal/hcp/registry/types.bucket.go b/internal/hcp/registry/types.bucket.go index ab068a30866..b9be5fd81b1 100644 --- a/internal/hcp/registry/types.bucket.go +++ b/internal/hcp/registry/types.bucket.go @@ -642,7 +642,6 @@ func (bucket *Bucket) completeBuild( doneCh, ok := bucket.RunningBuilds[buildName] if !ok { log.Print("[ERROR] done build does not have an entry in the heartbeat table, state will be inconsistent.") - } else { log.Printf("[TRACE] signal stopping heartbeats") // Stop heartbeating @@ -662,6 +661,23 @@ func (bucket *Bucket) completeBuild( return packerSDKArtifacts, fmt.Errorf("build failed, not uploading artifacts") } + artifacts, err := bucket.doCompleteBuild(ctx, buildName, packerSDKArtifacts, buildErr) + if err != nil { + err := bucket.UpdateBuildStatus(ctx, buildName, hcpPackerModels.HashicorpCloudPacker20230101BuildStatusBUILDFAILED) + if err != nil { + log.Printf("[ERROR] failed to update build %q status to FAILED: %s", buildName, err) + } + } + + return artifacts, err +} + +func (bucket *Bucket) doCompleteBuild( + ctx context.Context, + buildName string, + packerSDKArtifacts []packerSDK.Artifact, + buildErr error, +) ([]packerSDK.Artifact, error) { for _, art := range packerSDKArtifacts { var sdkImages []packerSDKRegistry.Image decoder, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{ From c2e2c063e5ce6d2c82bfa74ffc73f0978b11cda9 Mon Sep 17 00:00:00 2001 From: Lucas Bajolet Date: Wed, 13 Nov 2024 15:36:45 -0500 Subject: [PATCH 13/24] command: exit non-zero if uploading to HCP failed In the current state, a Packer build that succeeds but fails to push its metadata to HCP for reasons other than a lack of artifact will always succeed from the perspective of a user invoking `packer build`. This can be a bit misleading, as users may expect their artifacts to appear on HCP Packer if their build succeeded on Packer Core, so this commit changes this behaviour, instead reporting HCP errors as a real error if the build failed, so packer returns a non-zero error code if this happens. --- command/build.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/command/build.go b/command/build.go index 58b548b008f..3caa0b9d842 100644 --- a/command/build.go +++ b/command/build.go @@ -315,6 +315,15 @@ Check that you are using an HCP Ready integration before trying again: artifacts.Unlock() } } + + // If the build succeeded but uploading to HCP failed, + // Packer should exit non-zero, so we re-assign the + // error to account for this case. + if hcperr != nil && err == nil { + errs.Lock() + errs.m[name] = hcperr + errs.Unlock() + } }() if cla.Debug { From 3b1f4ccfe0b0ea92f59d549d502473241381d578 Mon Sep 17 00:00:00 2001 From: Jenna Goldstrich Date: Mon, 6 Jan 2025 18:02:08 -0800 Subject: [PATCH 14/24] hcp: use enum for HCP SBOM upload Since the protos for uploading an SBOM for a build have been changed to use an enumeration instead of a plain string with the latest revisions to the HCP Packer SBOM support feature, we update how we reference those values for the SBOM format to use that enum instead. --- internal/hcp/registry/types.bucket.go | 2 +- packer/build.go | 3 ++- packer/provisioner.go | 3 ++- provisioner/hcp-sbom/provisioner.go | 5 +++-- provisioner/hcp-sbom/validate.go | 7 ++++--- 5 files changed, 12 insertions(+), 8 deletions(-) diff --git a/internal/hcp/registry/types.bucket.go b/internal/hcp/registry/types.bucket.go index b9be5fd81b1..184a46e2ad3 100644 --- a/internal/hcp/registry/types.bucket.go +++ b/internal/hcp/registry/types.bucket.go @@ -246,7 +246,7 @@ func (bucket *Bucket) uploadSbom(ctx context.Context, buildName string, sbom pac Body: &hcpPackerModels.HashicorpCloudPacker20230101UploadSbomBody{ CompressedSbom: sbom.CompressedData, Name: sbom.Name, - Format: sbom.Format, + Format: &sbom.Format, }, }, nil, diff --git a/packer/build.go b/packer/build.go index d23637f67a2..4a311461e96 100644 --- a/packer/build.go +++ b/packer/build.go @@ -9,6 +9,7 @@ import ( "log" "sync" + hcpPackerModels "github.com/hashicorp/hcp-sdk-go/clients/cloud-packer-service/stable/2023-01-01/models" "github.com/hashicorp/packer-plugin-sdk/common" packersdk "github.com/hashicorp/packer-plugin-sdk/packer" "github.com/hashicorp/packer-plugin-sdk/packerbuilderdata" @@ -56,7 +57,7 @@ type CoreBuild struct { type SBOM struct { Name string - Format string + Format hcpPackerModels.HashicorpCloudPacker20230101SbomFormat CompressedData []byte } diff --git a/packer/provisioner.go b/packer/provisioner.go index 24e20b3a247..4be4f99ddee 100644 --- a/packer/provisioner.go +++ b/packer/provisioner.go @@ -12,6 +12,7 @@ import ( hcpSbomProvisioner "github.com/hashicorp/packer/provisioner/hcp-sbom" + hcpPackerModels "github.com/hashicorp/hcp-sdk-go/clients/cloud-packer-service/stable/2023-01-01/models" "github.com/klauspost/compress/zstd" "time" @@ -249,7 +250,7 @@ func (p *DebuggedProvisioner) Provision(ctx context.Context, ui packersdk.Ui, co type SBOMInternalProvisioner struct { Provisioner packersdk.Provisioner CompressedData []byte - SBOMFormat string + SBOMFormat hcpPackerModels.HashicorpCloudPacker20230101SbomFormat SBOMName string } diff --git a/provisioner/hcp-sbom/provisioner.go b/provisioner/hcp-sbom/provisioner.go index cbc515c13d1..cf03e567067 100644 --- a/provisioner/hcp-sbom/provisioner.go +++ b/provisioner/hcp-sbom/provisioner.go @@ -20,6 +20,7 @@ import ( "path/filepath" "github.com/hashicorp/hcl/v2/hcldec" + hcpPackerModels "github.com/hashicorp/hcp-sdk-go/clients/cloud-packer-service/stable/2023-01-01/models" "github.com/hashicorp/packer-plugin-sdk/common" packersdk "github.com/hashicorp/packer-plugin-sdk/packer" "github.com/hashicorp/packer-plugin-sdk/template/config" @@ -116,8 +117,8 @@ type PackerSBOM struct { RawSBOM []byte `json:"raw_sbom"` // Format is the format detected by the provisioner // - // Supported values: `spdx` or `cyclonedx` - Format string `json:"format"` + // Supported values: `SPDX` or `CYCLONEDX` + Format hcpPackerModels.HashicorpCloudPacker20230101SbomFormat `json:"format"` // Name is the name of the SBOM to be set on HCP Packer // // If unset, HCP Packer will generate one diff --git a/provisioner/hcp-sbom/validate.go b/provisioner/hcp-sbom/validate.go index 4f17a4ac0de..7343dcb9bbb 100644 --- a/provisioner/hcp-sbom/validate.go +++ b/provisioner/hcp-sbom/validate.go @@ -6,6 +6,7 @@ import ( "strings" "github.com/CycloneDX/cyclonedx-go" + hcpPackerModels "github.com/hashicorp/hcp-sdk-go/clients/cloud-packer-service/stable/2023-01-01/models" spdxjson "github.com/spdx/tools-golang/json" ) @@ -61,11 +62,11 @@ func validateSPDX(content []byte) error { } // validateSBOM validates the SBOM file and returns the format of the SBOM. -func validateSBOM(content []byte) (string, error) { +func validateSBOM(content []byte) (hcpPackerModels.HashicorpCloudPacker20230101SbomFormat, error) { // Try validating as SPDX spdxErr := validateSPDX(content) if spdxErr == nil { - return "spdx", nil + return hcpPackerModels.HashicorpCloudPacker20230101SbomFormatSPDX, nil } if vErr, ok := spdxErr.(*ValidationError); ok { @@ -74,7 +75,7 @@ func validateSBOM(content []byte) (string, error) { cycloneDxErr := validateCycloneDX(content) if cycloneDxErr == nil { - return "cyclonedx", nil + return hcpPackerModels.HashicorpCloudPacker20230101SbomFormatCYCLONEDX, nil } if vErr, ok := cycloneDxErr.(*ValidationError); ok { From aac9048e32d0634cf1f0197af911e2261ad2939e Mon Sep 17 00:00:00 2001 From: Devashish Date: Wed, 18 Sep 2024 14:25:16 -0400 Subject: [PATCH 15/24] packer: add hcp-sbom provisioner The hcp-sbom provisioner is a provisioner that acts essentially like a download-only file provisioner, which also verifies the file downloaded is a SPDX/CycloneDX JSON-encoded SBOM file, and sets up its upload to HCP Packer later on. --- command/execute.go | 2 + go.mod | 5 +- go.sum | 18 ++ hcl2template/types.packer_config.go | 6 + packer/build.go | 20 ++ packer/core.go | 7 + packer/provisioner.go | 79 ++++++ provisioner/hcp-sbom/provisioner.go | 231 ++++++++++++++++++ provisioner/hcp-sbom/provisioner.hcl2spec.go | 51 ++++ provisioner/hcp-sbom/provisioner_test.go | 86 +++++++ provisioner/hcp-sbom/validate.go | 85 +++++++ provisioner/hcp-sbom/version/version.go | 16 ++ .../hcp-sbom/Config-not-required.mdx | 23 ++ .../provisioner/hcp-sbom/Config-required.mdx | 7 + 14 files changed, 635 insertions(+), 1 deletion(-) create mode 100644 provisioner/hcp-sbom/provisioner.go create mode 100644 provisioner/hcp-sbom/provisioner.hcl2spec.go create mode 100644 provisioner/hcp-sbom/provisioner_test.go create mode 100644 provisioner/hcp-sbom/validate.go create mode 100644 provisioner/hcp-sbom/version/version.go create mode 100644 website/content/partials/provisioner/hcp-sbom/Config-not-required.mdx create mode 100644 website/content/partials/provisioner/hcp-sbom/Config-required.mdx diff --git a/command/execute.go b/command/execute.go index 7ad74f314d4..1e303858d61 100644 --- a/command/execute.go +++ b/command/execute.go @@ -28,6 +28,7 @@ import ( shelllocalpostprocessor "github.com/hashicorp/packer/post-processor/shell-local" breakpointprovisioner "github.com/hashicorp/packer/provisioner/breakpoint" fileprovisioner "github.com/hashicorp/packer/provisioner/file" + hcpsbomprovisioner "github.com/hashicorp/packer/provisioner/hcp-sbom" powershellprovisioner "github.com/hashicorp/packer/provisioner/powershell" shellprovisioner "github.com/hashicorp/packer/provisioner/shell" shelllocalprovisioner "github.com/hashicorp/packer/provisioner/shell-local" @@ -48,6 +49,7 @@ var Builders = map[string]packersdk.Builder{ var Provisioners = map[string]packersdk.Provisioner{ "breakpoint": new(breakpointprovisioner.Provisioner), "file": new(fileprovisioner.Provisioner), + "hcp-sbom": new(hcpsbomprovisioner.Provisioner), "powershell": new(powershellprovisioner.Provisioner), "shell": new(shellprovisioner.Provisioner), "shell-local": new(shelllocalprovisioner.Provisioner), diff --git a/go.mod b/go.mod index 7ffca662add..db2d02f1f29 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,7 @@ require ( github.com/hashicorp/hcp-sdk-go v0.131.0 github.com/hashicorp/packer-plugin-sdk v0.6.0 github.com/jehiah/go-strftime v0.0.0-20171201141054-1d33003b3869 - github.com/klauspost/compress v1.13.6 // indirect + github.com/klauspost/compress v1.13.6 github.com/klauspost/pgzip v1.2.5 github.com/masterzen/winrm v0.0.0-20210623064412-3b76017826b0 github.com/mattn/go-runewidth v0.0.13 // indirect @@ -57,10 +57,12 @@ require ( ) require ( + github.com/CycloneDX/cyclonedx-go v0.9.1 github.com/go-openapi/strfmt v0.21.10 github.com/oklog/ulid v1.3.1 github.com/pierrec/lz4/v4 v4.1.18 github.com/shirou/gopsutil/v3 v3.23.4 + github.com/spdx/tools-golang v0.5.5 ) require ( @@ -77,6 +79,7 @@ require ( github.com/Microsoft/go-winio v0.6.1 // indirect github.com/ProtonMail/go-crypto v1.1.3 // indirect github.com/agext/levenshtein v1.2.3 // indirect + github.com/anchore/go-struct-converter v0.0.0-20221118182256-c68fdcfa2092 // indirect github.com/apparentlymart/go-cidr v1.0.1 // indirect github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect diff --git a/go.sum b/go.sum index abafd431c39..7c8fe8a5e2f 100644 --- a/go.sum +++ b/go.sum @@ -20,6 +20,8 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym github.com/ChrisTrenkamp/goxpath v0.0.0-20170922090931-c385f95c6022/go.mod h1:nuWgzSkT5PnyOd+272uUmV0dnAnAn42Mk7PiQC5VzN4= github.com/ChrisTrenkamp/goxpath v0.0.0-20210404020558-97928f7e12b6 h1:w0E0fgc1YafGEh5cROhlROMWXiNoZqApk2PDN0M1+Ns= github.com/ChrisTrenkamp/goxpath v0.0.0-20210404020558-97928f7e12b6/go.mod h1:nuWgzSkT5PnyOd+272uUmV0dnAnAn42Mk7PiQC5VzN4= +github.com/CycloneDX/cyclonedx-go v0.9.1 h1:yffaWOZsv77oTJa/SdVZYdgAgFioCeycBUKkqS2qzQM= +github.com/CycloneDX/cyclonedx-go v0.9.1/go.mod h1:NE/EWvzELOFlG6+ljX/QeMlVt9VKcTwu8u0ccsACEsw= github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= @@ -38,6 +40,8 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/anchore/go-struct-converter v0.0.0-20221118182256-c68fdcfa2092 h1:aM1rlcoLz8y5B2r4tTLMiVTrMtpfY0O8EScKJxaSaEc= +github.com/anchore/go-struct-converter v0.0.0-20221118182256-c68fdcfa2092/go.mod h1:rYqSE9HbjzpHTI74vwPvae4ZVYZd1lue2ta6xHPdblA= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= github.com/antchfx/xmlquery v1.3.5 h1:I7TuBRqsnfFuL11ruavGm911Awx9IqSdiU6W/ztSmVw= @@ -78,6 +82,8 @@ github.com/biogo/hts v1.4.3 h1:vir2yUTiRkPvtp6ZTpzh9lWTKQJZXJKZ563rpAQAsRM= github.com/biogo/hts v1.4.3/go.mod h1:eW40HJ1l2ExK9C+yvvoRSftInqWsf3ue+zAEjzCGWjA= github.com/bmatcuk/doublestar v1.1.5 h1:2bNwBOmhyFEFcoB3tGvTD5xanq+4kyOZlB8wFYbMjkk= github.com/bmatcuk/doublestar v1.1.5/go.mod h1:wiQtGV+rzVYxB7WIlirSN++5HPtPlXEo9MEoZQC/PmE= +github.com/bradleyjkemp/cupaloy/v2 v2.8.0 h1:any4BmKE+jGIaMpnU8YgH/I2LPiLBufr6oMMlVBbn9M= +github.com/bradleyjkemp/cupaloy/v2 v2.8.0/go.mod h1:bm7JXdkRd4BHJk9HpwqAI8BoAY1lps46Enkdqw6aRX0= github.com/cenkalti/backoff/v3 v3.2.2 h1:cfUAAO3yvKMYKPrvhDuHSwQnhZNk/RMHKdZqKTxfm6M= github.com/cenkalti/backoff/v3 v3.2.2/go.mod h1:cIeZDE3IrqwwJl6VUwCN6trj1oXrTS4rc0ij+ULvLYs= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -490,6 +496,9 @@ github.com/skeema/knownhosts v1.3.0 h1:AM+y0rI04VksttfwjkSTNQorvGqmwATnvnAHpSgc0 github.com/skeema/knownhosts v1.3.0/go.mod h1:sPINvnADmT/qYH1kfv+ePMmOBTH6Tbl7b5LvTDjFK7M= github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 h1:JIAuq3EEf9cgbU6AtGPK4CTG3Zf6CKMNqf0MHTggAUA= github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966/go.mod h1:sUM3LWHvSMaG192sy56D9F7CNvL7jUJVXoqM1QKLnog= +github.com/spdx/gordf v0.0.0-20201111095634-7098f93598fb/go.mod h1:uKWaldnbMnjsSAXRurWqqrdyZen1R7kxl8TkmWk2OyM= +github.com/spdx/tools-golang v0.5.5 h1:61c0KLfAcNqAjlg6UNMdkwpMernhw3zVRwDZ2x9XOmk= +github.com/spdx/tools-golang v0.5.5/go.mod h1:MVIsXx8ZZzaRWNQpUDhC4Dud34edUYJYecciXgrw5vE= github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -509,8 +518,12 @@ github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1F github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/terminalstatic/go-xsd-validate v0.1.5 h1:RqpJnf6HGE2CB/lZB1A8BYguk8uRtcvYAPLCF15qguo= +github.com/terminalstatic/go-xsd-validate v0.1.5/go.mod h1:18lsvYFofBflqCrvo1umpABZ99+GneNTw2kEEc8UPJw= github.com/tklauser/go-sysconf v0.3.11 h1:89WgdJhk5SNwJfu+GKyYveZ4IaJ7xAkecBo+KdJV0CM= github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7AmcWpGR8lSZfqI= github.com/tklauser/numcpus v0.6.0 h1:kebhY2Qt+3U6RNK7UqpYNA+tJ23IBEGKkB7JQBfDYms= @@ -533,6 +546,10 @@ github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3k github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= +github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= +github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= @@ -739,3 +756,4 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/hcl2template/types.packer_config.go b/hcl2template/types.packer_config.go index cb81441d819..bf0e9636c70 100644 --- a/hcl2template/types.packer_config.go +++ b/hcl2template/types.packer_config.go @@ -573,6 +573,12 @@ func (cfg *PackerConfig) getCoreBuildProvisioner(source SourceUseBlock, pb *Prov } } + if pb.PType == "hcp-sbom" { + provisioner = &packer.SBOMInternalProvisioner{ + Provisioner: provisioner, + } + } + return packer.CoreBuildProvisioner{ PType: pb.PType, PName: pb.PName, diff --git a/packer/build.go b/packer/build.go index 8b62ec53799..eade2625dd6 100644 --- a/packer/build.go +++ b/packer/build.go @@ -50,11 +50,19 @@ type CoreBuild struct { onError string l sync.Mutex prepareCalled bool + + SBOMs []SBOM +} + +type SBOM struct { + Format string + CompressedData []byte } type BuildMetadata struct { PackerVersion string Plugins map[string]PluginDetails + SBOMs []SBOM } func (b *CoreBuild) getPluginsMetadata() map[string]PluginDetails { @@ -88,6 +96,7 @@ func (b *CoreBuild) GetMetadata() BuildMetadata { metadata := BuildMetadata{ PackerVersion: version.FormattedVersion(), Plugins: b.getPluginsMetadata(), + SBOMs: b.SBOMs, } return metadata } @@ -300,6 +309,17 @@ func (b *CoreBuild) Run(ctx context.Context, originalUi packersdk.Ui) ([]packers return nil, err } + for _, p := range b.Provisioners { + sbomInternalProvisioner, ok := p.Provisioner.(*SBOMInternalProvisioner) + if ok { + sbom := SBOM{ + Format: sbomInternalProvisioner.SBOMFormat, + CompressedData: sbomInternalProvisioner.CompressedData, + } + b.SBOMs = append(b.SBOMs, sbom) + } + } + // If there was no result, don't worry about running post-processors // because there is nothing they can do, just return. if builderArtifact == nil { diff --git a/packer/core.go b/packer/core.go index 6bff2df060a..f6724cda9ef 100644 --- a/packer/core.go +++ b/packer/core.go @@ -296,6 +296,13 @@ func (c *Core) generateCoreBuildProvisioner(rawP *template.Provisioner, rawName Provisioner: provisioner, } } + + if rawP.Type == "hcp-sbom" { + provisioner = &SBOMInternalProvisioner{ + Provisioner: provisioner, + } + } + cbp = CoreBuildProvisioner{ PType: rawP.Type, Provisioner: provisioner, diff --git a/packer/provisioner.go b/packer/provisioner.go index 81dce0ecfc0..24e20b3a247 100644 --- a/packer/provisioner.go +++ b/packer/provisioner.go @@ -5,8 +5,15 @@ package packer import ( "context" + "encoding/json" "fmt" "log" + "os" + + hcpSbomProvisioner "github.com/hashicorp/packer/provisioner/hcp-sbom" + + "github.com/klauspost/compress/zstd" + "time" "github.com/hashicorp/hcl/v2/hcldec" @@ -234,3 +241,75 @@ func (p *DebuggedProvisioner) Provision(ctx context.Context, ui packersdk.Ui, co return p.Provisioner.Provision(ctx, ui, comm, generatedData) } + +// SBOMInternalProvisioner is a wrapper provisioner for the `hcp-sbom` provisioner +// that sets the path for SBOM file download and, after the successful execution of +// the `hcp-sbom` provisioner, compresses the SBOM and prepares the data for API +// integration. +type SBOMInternalProvisioner struct { + Provisioner packersdk.Provisioner + CompressedData []byte + SBOMFormat string + SBOMName string +} + +func (p *SBOMInternalProvisioner) ConfigSpec() hcldec.ObjectSpec { return p.ConfigSpec() } +func (p *SBOMInternalProvisioner) FlatConfig() interface{} { return p.FlatConfig() } +func (p *SBOMInternalProvisioner) Prepare(raws ...interface{}) error { + return p.Provisioner.Prepare(raws...) +} + +func (p *SBOMInternalProvisioner) Provision( + ctx context.Context, ui packersdk.Ui, comm packersdk.Communicator, + generatedData map[string]interface{}, +) error { + cwd, err := os.Getwd() + if err != nil { + return fmt.Errorf("failed to get current working directory for Packer SBOM: %s", err) + } + + tmpFile, err := os.CreateTemp(cwd, "packer-sbom-*.json") + if err != nil { + return fmt.Errorf("failed to create internal temporary file for Packer SBOM: %s", err) + } + + tmpFileName := tmpFile.Name() + if err = tmpFile.Close(); err != nil { + return fmt.Errorf("failed to close temporary file for Packer SBOM %s: %s", tmpFileName, err) + } + + defer func(name string) { + fileRemoveErr := os.Remove(name) + if fileRemoveErr != nil { + log.Printf("Error removing SBOM temporary file %s: %s", name, fileRemoveErr) + } + }(tmpFile.Name()) + + generatedData["dst"] = tmpFile.Name() + + err = p.Provisioner.Provision(ctx, ui, comm, generatedData) + if err != nil { + return err + } + + packerSbom, err := os.Open(tmpFileName) + if err != nil { + return fmt.Errorf("failed to open Packer SBOM file %q: %s", tmpFileName, err) + } + + provisionerOut := &hcpSbomProvisioner.PackerSBOM{} + err = json.NewDecoder(packerSbom).Decode(provisionerOut) + if err != nil { + return fmt.Errorf("malformed packer SBOM output from file %q: %s", tmpFileName, err) + } + + encoder, err := zstd.NewWriter(nil, zstd.WithEncoderLevel(zstd.SpeedBestCompression)) + if err != nil { + return fmt.Errorf("failed to create zstd encoder: %s", err) + } + p.CompressedData = encoder.EncodeAll(provisionerOut.RawSBOM, nil) + p.SBOMFormat = provisionerOut.Format + p.SBOMName = provisionerOut.Name + + return nil +} diff --git a/provisioner/hcp-sbom/provisioner.go b/provisioner/hcp-sbom/provisioner.go new file mode 100644 index 00000000000..cbc515c13d1 --- /dev/null +++ b/provisioner/hcp-sbom/provisioner.go @@ -0,0 +1,231 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +//go:generate packer-sdc mapstructure-to-hcl2 -type Config +//go:generate packer-sdc struct-markdown + +package hcp_sbom + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "log" + "os" + "regexp" + "strings" + + "path/filepath" + + "github.com/hashicorp/hcl/v2/hcldec" + "github.com/hashicorp/packer-plugin-sdk/common" + packersdk "github.com/hashicorp/packer-plugin-sdk/packer" + "github.com/hashicorp/packer-plugin-sdk/template/config" + "github.com/hashicorp/packer-plugin-sdk/template/interpolate" +) + +type Config struct { + common.PackerConfig `mapstructure:",squash"` + + // Source is a required field that specifies the path to the SBOM file that + // needs to be downloaded. + // It can be a file path or a URL. + Source string `mapstructure:"source" required:"true"` + // Destination is an optional field that specifies the path where the SBOM + // file will be downloaded to for the user. + // The 'Destination' must be a writable location. If the destination is a file, + // the SBOM will be saved or overwritten at that path. If the destination is + // a directory, a file will be created within the directory to store the SBOM. + // Any parent directories for the destination must already exist and be + // writable by the provisioning user (generally not root), otherwise, + // a "Permission Denied" error will occur. If the source path is a file, + // it is recommended that the destination path be a file as well. + Destination string `mapstructure:"destination"` + // The name to give the SBOM when uploaded on HCP Packer + // + // By default this will be generated, but if you prefer to have a name + // of your choosing, you can enter it here. + // The name must match the following regexp: `[a-zA-Z0-9_-]{3,36}` + // + // Note: it must be unique for a single build, otherwise the build will + // fail when uploading the SBOMs to HCP Packer, and so will the Packer + // build command. + SbomName string `mapstructure:"sbom_name"` + ctx interpolate.Context +} + +type Provisioner struct { + config Config +} + +func (p *Provisioner) ConfigSpec() hcldec.ObjectSpec { + return p.config.FlatMapstructure().HCL2Spec() +} + +var sbomFormatRegexp = regexp.MustCompile("^[0-9A-Za-z-]{3,36}$") + +func (p *Provisioner) Prepare(raws ...interface{}) error { + err := config.Decode(&p.config, &config.DecodeOpts{ + PluginType: "hcp-sbom", + Interpolate: true, + InterpolateContext: &p.config.ctx, + InterpolateFilter: &interpolate.RenderFilter{ + Exclude: []string{}, + }, + }, raws...) + if err != nil { + return err + } + + var errs error + + if p.config.Source == "" { + errs = packersdk.MultiErrorAppend(errs, errors.New("source must be specified")) + } + + if p.config.SbomName != "" && !sbomFormatRegexp.MatchString(p.config.SbomName) { + // Ugly but a bit of a problem with interpolation since Provisioners + // are prepared twice in HCL2. + // + // If the information used for interpolating is populated in-between the + // first call to Prepare (at the start of the build), and when the + // Provisioner is actually called, the first call will fail, as + // the value won't contain the actual interpolated value, but a + // placeholder which doesn't match the regex. + // + // Since we don't have a way to discriminate between the calls + // in the context of the provisioner, we ignore them, and later the + // HCP Packer call will fail because of the broken regex. + if strings.Contains(p.config.SbomName, "") { + log.Printf("[WARN] interpolation incomplete for `sbom_name`, will possibly retry later with data populated into context, otherwise will fail when uploading to HCP Packer.") + } else { + errs = packersdk.MultiErrorAppend(errs, fmt.Errorf("`sbom_name` %q doesn't match the expected format, it must "+ + "contain between 3 and 36 characters, all from the following set: [A-Za-z0-9_-]", p.config.SbomName)) + } + } + + return errs +} + +// PackerSBOM is the type we write to the temporary JSON dump of the SBOM to +// be consumed by Packer core +type PackerSBOM struct { + // RawSBOM is the raw data from the SBOM downloaded from the guest + RawSBOM []byte `json:"raw_sbom"` + // Format is the format detected by the provisioner + // + // Supported values: `spdx` or `cyclonedx` + Format string `json:"format"` + // Name is the name of the SBOM to be set on HCP Packer + // + // If unset, HCP Packer will generate one + Name string `json:"name,omitempty"` +} + +func (p *Provisioner) Provision( + ctx context.Context, ui packersdk.Ui, comm packersdk.Communicator, + generatedData map[string]interface{}, +) error { + log.Println("Starting to provision with `hcp-sbom` provisioner") + + if generatedData == nil { + generatedData = make(map[string]interface{}) + } + p.config.ctx.Data = generatedData + + src := p.config.Source + + pkrDst := generatedData["dst"].(string) + if pkrDst == "" { + return fmt.Errorf("packer destination path missing from configs: this is an internal error, which should be reported to be fixed.") + } + + var buf bytes.Buffer + if err := comm.Download(src, &buf); err != nil { + ui.Errorf("download failed for SBOM file: %s", err) + return err + } + + format, err := validateSBOM(buf.Bytes()) + if err != nil { + return fmt.Errorf("validation failed for SBOM file: %s", err) + } + + outFile, err := os.Create(pkrDst) + if err != nil { + return fmt.Errorf("failed to open/create output file %q: %s", pkrDst, err) + } + defer outFile.Close() + + err = json.NewEncoder(outFile).Encode(PackerSBOM{ + RawSBOM: buf.Bytes(), + Format: format, + Name: p.config.SbomName, + }) + if err != nil { + return fmt.Errorf("failed to write sbom file to %q: %s", pkrDst, err) + } + + if p.config.Destination == "" { + return nil + } + + // SBOM for User + usrDst, err := p.getUserDestination() + if err != nil { + return fmt.Errorf("failed to compute destination path %q: %s", p.config.Destination, err) + } + err = os.WriteFile(usrDst, buf.Bytes(), 0644) + if err != nil { + return fmt.Errorf("failed to write SBOM to destination %q: %s", usrDst, err) + } + + return nil +} + +// getUserDestination determines and returns the destination path for the user SBOM file. +func (p *Provisioner) getUserDestination() (string, error) { + dst := p.config.Destination + + // Check if the destination exists and determine its type + info, err := os.Stat(dst) + if err == nil { + if info.IsDir() { + // If the destination is a directory, create a temporary file inside it + tmpFile, err := os.CreateTemp(dst, "packer-user-sbom-*.json") + if err != nil { + return "", fmt.Errorf("failed to create temporary file in user SBOM directory %s: %s", dst, err) + } + dst = tmpFile.Name() + tmpFile.Close() + } + return dst, nil + } + + outDir := filepath.Dir(dst) + // In case the destination does not exist, we'll get the dirpath, + // and create it if it doesn't already exist + err = os.MkdirAll(outDir, 0755) + if err != nil { + return "", fmt.Errorf("failed to create destination directory for user SBOM: %s\n", err) + } + + // Check if the destination is a directory after the previous step. + // + // This happens if the path specified ends with a `/`, in which case the + // destination is a directory, and we must create a temporary file in + // this destination directory. + destStat, statErr := os.Stat(dst) + if statErr == nil && destStat.IsDir() { + tmpFile, err := os.CreateTemp(outDir, "packer-user-sbom-*.json") + if err != nil { + return "", fmt.Errorf("failed to create temporary file in user SBOM directory %s: %s", dst, err) + } + dst = tmpFile.Name() + tmpFile.Close() + } + + return dst, nil +} diff --git a/provisioner/hcp-sbom/provisioner.hcl2spec.go b/provisioner/hcp-sbom/provisioner.hcl2spec.go new file mode 100644 index 00000000000..4df5397c093 --- /dev/null +++ b/provisioner/hcp-sbom/provisioner.hcl2spec.go @@ -0,0 +1,51 @@ +// Code generated by "packer-sdc mapstructure-to-hcl2"; DO NOT EDIT. + +package hcp_sbom + +import ( + "github.com/hashicorp/hcl/v2/hcldec" + "github.com/zclconf/go-cty/cty" +) + +// FlatConfig is an auto-generated flat version of Config. +// Where the contents of a field with a `mapstructure:,squash` tag are bubbled up. +type FlatConfig struct { + PackerBuildName *string `mapstructure:"packer_build_name" cty:"packer_build_name" hcl:"packer_build_name"` + PackerBuilderType *string `mapstructure:"packer_builder_type" cty:"packer_builder_type" hcl:"packer_builder_type"` + PackerCoreVersion *string `mapstructure:"packer_core_version" cty:"packer_core_version" hcl:"packer_core_version"` + PackerDebug *bool `mapstructure:"packer_debug" cty:"packer_debug" hcl:"packer_debug"` + PackerForce *bool `mapstructure:"packer_force" cty:"packer_force" hcl:"packer_force"` + PackerOnError *string `mapstructure:"packer_on_error" cty:"packer_on_error" hcl:"packer_on_error"` + PackerUserVars map[string]string `mapstructure:"packer_user_variables" cty:"packer_user_variables" hcl:"packer_user_variables"` + PackerSensitiveVars []string `mapstructure:"packer_sensitive_variables" cty:"packer_sensitive_variables" hcl:"packer_sensitive_variables"` + Source *string `mapstructure:"source" required:"true" cty:"source" hcl:"source"` + Destination *string `mapstructure:"destination" cty:"destination" hcl:"destination"` + SbomName *string `mapstructure:"sbom_name" cty:"sbom_name" hcl:"sbom_name"` +} + +// FlatMapstructure returns a new FlatConfig. +// FlatConfig is an auto-generated flat version of Config. +// Where the contents a fields with a `mapstructure:,squash` tag are bubbled up. +func (*Config) FlatMapstructure() interface{ HCL2Spec() map[string]hcldec.Spec } { + return new(FlatConfig) +} + +// HCL2Spec returns the hcl spec of a Config. +// This spec is used by HCL to read the fields of Config. +// The decoded values from this spec will then be applied to a FlatConfig. +func (*FlatConfig) HCL2Spec() map[string]hcldec.Spec { + s := map[string]hcldec.Spec{ + "packer_build_name": &hcldec.AttrSpec{Name: "packer_build_name", Type: cty.String, Required: false}, + "packer_builder_type": &hcldec.AttrSpec{Name: "packer_builder_type", Type: cty.String, Required: false}, + "packer_core_version": &hcldec.AttrSpec{Name: "packer_core_version", Type: cty.String, Required: false}, + "packer_debug": &hcldec.AttrSpec{Name: "packer_debug", Type: cty.Bool, Required: false}, + "packer_force": &hcldec.AttrSpec{Name: "packer_force", Type: cty.Bool, Required: false}, + "packer_on_error": &hcldec.AttrSpec{Name: "packer_on_error", Type: cty.String, Required: false}, + "packer_user_variables": &hcldec.AttrSpec{Name: "packer_user_variables", Type: cty.Map(cty.String), Required: false}, + "packer_sensitive_variables": &hcldec.AttrSpec{Name: "packer_sensitive_variables", Type: cty.List(cty.String), Required: false}, + "source": &hcldec.AttrSpec{Name: "source", Type: cty.String, Required: false}, + "destination": &hcldec.AttrSpec{Name: "destination", Type: cty.String, Required: false}, + "sbom_name": &hcldec.AttrSpec{Name: "sbom_name", Type: cty.String, Required: false}, + } + return s +} diff --git a/provisioner/hcp-sbom/provisioner_test.go b/provisioner/hcp-sbom/provisioner_test.go new file mode 100644 index 00000000000..aff0323e04a --- /dev/null +++ b/provisioner/hcp-sbom/provisioner_test.go @@ -0,0 +1,86 @@ +package hcp_sbom + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/hashicorp/packer-plugin-sdk/template/interpolate" +) + +func TestConfigPrepare(t *testing.T) { + tests := []struct { + name string + inputConfig map[string]interface{} + interpolateContext interpolate.Context + expectConfig *Config + expectError bool + }{ + { + "empty config, should error without a source", + map[string]interface{}{}, + interpolate.Context{}, + nil, + true, + }, + { + "config with full context for interpolation: success", + map[string]interface{}{ + "source": "{{ .Name }}", + }, + interpolate.Context{ + Data: &struct { + Name string + }{ + Name: "testInterpolate", + }, + }, + &Config{ + Source: "testInterpolate", + }, + false, + }, + { + // Note: this will look weird to reviewers, but is actually + // expected for the moment. + // Refer to the comment in `Prepare` for context as to WHY + // this cannot be considered an error. + "config with sbom name as interpolated value, without it in context, replace with a placeholder", + map[string]interface{}{ + "source": "test", + "sbom_name": "{{ .Name }}", + }, + interpolate.Context{}, + &Config{ + Source: "test", + SbomName: "", + }, + false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + prov := &Provisioner{} + prov.config.ctx = tt.interpolateContext + err := prov.Prepare(tt.inputConfig) + if err != nil && !tt.expectError { + t.Fatalf("configuration unexpectedly failed to prepare: %s", err) + } + + if err == nil && tt.expectError { + t.Fatalf("configuration succeeded to prepare, but should have failed") + } + + if err != nil { + t.Logf("config had error %q", err) + return + } + + diff := cmp.Diff(prov.config, *tt.expectConfig, cmpopts.IgnoreUnexported(Config{})) + if diff != "" { + t.Errorf("configuration returned by `Prepare` is different from what was expected: %s", diff) + } + }) + } +} diff --git a/provisioner/hcp-sbom/validate.go b/provisioner/hcp-sbom/validate.go new file mode 100644 index 00000000000..4f17a4ac0de --- /dev/null +++ b/provisioner/hcp-sbom/validate.go @@ -0,0 +1,85 @@ +package hcp_sbom + +import ( + "bytes" + "fmt" + "strings" + + "github.com/CycloneDX/cyclonedx-go" + spdxjson "github.com/spdx/tools-golang/json" +) + +// ValidationError represents an error encountered while validating an SBOM. +type ValidationError struct { + Err error +} + +func (e *ValidationError) Error() string { + return e.Err.Error() +} + +func (e *ValidationError) Unwrap() error { + return e.Err +} + +// ValidateCycloneDX is a validation for CycloneDX in JSON format. +func validateCycloneDX(content []byte) error { + decoder := cyclonedx.NewBOMDecoder(bytes.NewBuffer(content), cyclonedx.BOMFileFormatJSON) + bom := new(cyclonedx.BOM) + if err := decoder.Decode(bom); err != nil { + return fmt.Errorf("error parsing CycloneDX SBOM: %w", err) + } + + if !strings.EqualFold(bom.BOMFormat, "CycloneDX") { + return &ValidationError{ + Err: fmt.Errorf("invalid bomFormat: %q, expected CycloneDX", bom.BOMFormat), + } + } + if bom.SpecVersion.String() == "" { + return &ValidationError{ + Err: fmt.Errorf("specVersion is required"), + } + } + + return nil +} + +// validateSPDX is a validation for SPDX in JSON format. +func validateSPDX(content []byte) error { + doc, err := spdxjson.Read(bytes.NewBuffer(content)) + if err != nil { + return fmt.Errorf("error parsing SPDX JSON file: %w", err) + } + + if doc.SPDXVersion == "" { + return &ValidationError{ + Err: fmt.Errorf("missing SPDXVersion"), + } + } + + return nil +} + +// validateSBOM validates the SBOM file and returns the format of the SBOM. +func validateSBOM(content []byte) (string, error) { + // Try validating as SPDX + spdxErr := validateSPDX(content) + if spdxErr == nil { + return "spdx", nil + } + + if vErr, ok := spdxErr.(*ValidationError); ok { + return "", vErr + } + + cycloneDxErr := validateCycloneDX(content) + if cycloneDxErr == nil { + return "cyclonedx", nil + } + + if vErr, ok := cycloneDxErr.(*ValidationError); ok { + return "", vErr + } + + return "", fmt.Errorf("error validating SBOM file: invalid SBOM format") +} diff --git a/provisioner/hcp-sbom/version/version.go b/provisioner/hcp-sbom/version/version.go new file mode 100644 index 00000000000..772d6d4f444 --- /dev/null +++ b/provisioner/hcp-sbom/version/version.go @@ -0,0 +1,16 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package version + +import ( + "github.com/hashicorp/packer-plugin-sdk/version" + packerVersion "github.com/hashicorp/packer/version" +) + +var HCPSBOMPluginVersion *version.PluginVersion + +func init() { + HCPSBOMPluginVersion = version.NewPluginVersion( + packerVersion.Version, packerVersion.VersionPrerelease, packerVersion.VersionMetadata) +} diff --git a/website/content/partials/provisioner/hcp-sbom/Config-not-required.mdx b/website/content/partials/provisioner/hcp-sbom/Config-not-required.mdx new file mode 100644 index 00000000000..871e7a5adeb --- /dev/null +++ b/website/content/partials/provisioner/hcp-sbom/Config-not-required.mdx @@ -0,0 +1,23 @@ + + +- `destination` (string) - Destination is an optional field that specifies the path where the SBOM + file will be downloaded to for the user. + The 'Destination' must be a writable location. If the destination is a file, + the SBOM will be saved or overwritten at that path. If the destination is + a directory, a file will be created within the directory to store the SBOM. + Any parent directories for the destination must already exist and be + writable by the provisioning user (generally not root), otherwise, + a "Permission Denied" error will occur. If the source path is a file, + it is recommended that the destination path be a file as well. + +- `sbom_name` (string) - The name to give the SBOM when uploaded on HCP Packer + + By default this will be generated, but if you prefer to have a name + of your choosing, you can enter it here. + The name must match the following regexp: `[a-zA-Z0-9_-]{3,36}` + + Note: it must be unique for a single build, otherwise the build will + fail when uploading the SBOMs to HCP Packer, and so will the Packer + build command. + + diff --git a/website/content/partials/provisioner/hcp-sbom/Config-required.mdx b/website/content/partials/provisioner/hcp-sbom/Config-required.mdx new file mode 100644 index 00000000000..2f227c2b0ff --- /dev/null +++ b/website/content/partials/provisioner/hcp-sbom/Config-required.mdx @@ -0,0 +1,7 @@ + + +- `source` (string) - Source is a required field that specifies the path to the SBOM file that + needs to be downloaded. + It can be a file path or a URL. + + From 9e3f82e6885c96356585bb415825861fee772b6f Mon Sep 17 00:00:00 2001 From: Brian McClain Date: Tue, 21 Jan 2025 12:07:15 -0500 Subject: [PATCH 16/24] Update hcp-sbom provisioner docs --- provisioner/hcp-sbom/provisioner.go | 34 +++++-------- .../content/docs/provisioners/hcp-sbom.mdx | 51 +++++++++++-------- website/content/docs/provisioners/index.mdx | 3 +- 3 files changed, 44 insertions(+), 44 deletions(-) diff --git a/provisioner/hcp-sbom/provisioner.go b/provisioner/hcp-sbom/provisioner.go index cf03e567067..4371390861c 100644 --- a/provisioner/hcp-sbom/provisioner.go +++ b/provisioner/hcp-sbom/provisioner.go @@ -30,29 +30,21 @@ import ( type Config struct { common.PackerConfig `mapstructure:",squash"` - // Source is a required field that specifies the path to the SBOM file that - // needs to be downloaded. - // It can be a file path or a URL. + // The file path or URL to the SBOM file in the Packer artifact. + // This file must either be in the SPDX or CycloneDX format. Source string `mapstructure:"source" required:"true"` - // Destination is an optional field that specifies the path where the SBOM - // file will be downloaded to for the user. - // The 'Destination' must be a writable location. If the destination is a file, - // the SBOM will be saved or overwritten at that path. If the destination is - // a directory, a file will be created within the directory to store the SBOM. - // Any parent directories for the destination must already exist and be - // writable by the provisioning user (generally not root), otherwise, - // a "Permission Denied" error will occur. If the source path is a file, - // it is recommended that the destination path be a file as well. + + // The path on the local machine to store a copy of the SBOM file. + // You can specify an absolute or a path relative to the working directory + // when you execute the Packer build. If the file already exists on the + // local machine, Packer overwrites the file. If the destination is a + // directory, the directory must already exist. Destination string `mapstructure:"destination"` - // The name to give the SBOM when uploaded on HCP Packer - // - // By default this will be generated, but if you prefer to have a name - // of your choosing, you can enter it here. - // The name must match the following regexp: `[a-zA-Z0-9_-]{3,36}` - // - // Note: it must be unique for a single build, otherwise the build will - // fail when uploading the SBOMs to HCP Packer, and so will the Packer - // build command. + + // The name of the SBOM file stored in HCP Packer. + // If omitted, HCP Packer uses the build fingerprint as the file name. + // This value must be between three and 36 characters from the following set: `[A-Za-z0-9_-]`. + // You must specify a unique name for each build in an artifact version. SbomName string `mapstructure:"sbom_name"` ctx interpolate.Context } diff --git a/website/content/docs/provisioners/hcp-sbom.mdx b/website/content/docs/provisioners/hcp-sbom.mdx index a90b5bf8886..f0bd15cca39 100644 --- a/website/content/docs/provisioners/hcp-sbom.mdx +++ b/website/content/docs/provisioners/hcp-sbom.mdx @@ -1,7 +1,6 @@ --- description: | - The `hcp-sbom` Packer provisioner downloads an SBOM file from the guest VM and - sends it to HCP Packer when the build is done. + The hcp-sbom Packer provisioner uploads a CycloneDX or SPDX JSON-formatted software bill of materials record to HCP Packer. page_title: HCP SBOM - Provisioners --- @@ -13,52 +12,54 @@ page_title: HCP SBOM - Provisioners Type: `hcp-sbom` -The `hcp-sbom` Packer provisioner downloads an SBOM file from the guest machine -and sends it to HCP Packer when the build is complete (only if the template is -HCP-enabled). The SBOM file is automatically removed at the end of the process. -If you want to retain a copy of the SBOM file, you can specify the -`destination` option in the provisioner. +The `hcp-sbom` provisioner uploads software bill of materials (SBOM) files from artifacts built by Packer to HCP Packer. You must format SBOM files you want to upload as JSON and follow either the [SPDX](https://spdx.github.io/spdx-spec/latest) or [CycloneDX](https://cyclonedx.org/) specification. HCP Packer ties these SBOM files to the version of the artifact that Packer builds. -Currently, we support `CycloneDX` and `SPDX` SBOM formats in `JSON`. +## Example -## Basic Example +The following example uploads an SBOM from the local `/tmp` directory and stores a copy at `./sbom/sbom_cyclonedx.json` on the local machine. - -In HCL2: + + ```hcl provisioner "hcp-sbom" { - source = "/tmp/sbom_cyclonedx.json" + source = "/tmp/sbom_cyclonedx.json" destination = "./sbom/sbom_cyclonedx.json" + sbom_name = "sbom-cyclonedx" } ``` -In JSON: + + ```json { "type": "hcp-sbom", "source": "/tmp/sbom_cyclonedx.json", - "destination": "./sbom/sbom_cyclonedx.json" + "destination": "./sbom/sbom_cyclonedx.json", + "sbom_name": "sbom-cyclonedx" } ``` + + +## Configuration reference -## Configuration Reference +You can specify the following configuration options. -Required Parameters: +Required parameters: @include 'provisioner/hcp-sbom/Config-required.mdx' -Optional Parameters: +Optional parameters: @include '/provisioner/hcp-sbom/Config-not-required.mdx' -## Example Usage - +## Example usage -In HCL2: + + ```hcl packer { @@ -95,11 +96,13 @@ build { provisioner "hcp-sbom" { source = "/tmp/sbom_cyclonedx.json" destination = "./sbom" + sbom_name = "sbom-cyclonedx" } } ``` -In JSON: + + ```json { @@ -123,8 +126,12 @@ In JSON: { "type": "hcp-sbom", "source": "/tmp/sbom_cyclonedx.json", - "destination": "./sbom" + "destination": "./sbom", + "sbom_name": "sbom-cyclonedx" } ] } ``` + + + \ No newline at end of file diff --git a/website/content/docs/provisioners/index.mdx b/website/content/docs/provisioners/index.mdx index 6be0124557f..da2603e80ac 100644 --- a/website/content/docs/provisioners/index.mdx +++ b/website/content/docs/provisioners/index.mdx @@ -20,7 +20,8 @@ The following provisioners are included with Packer: - [Breakpoint](/packer/docs/provisioners/breakpoint) - pause until the user presses `Enter` to resume a build. - [File](/packer/docs/provisioners/file) - upload files to machines image during a build. -- [HCP SBOM](/packer/docs/provisioners/hcp-sbom) - download SBOM file to machines and send to HCP Packer during a build. +- [HCP SBOM](/packer/docs/provisioners/hcp-sbom) - upload an SBOM and associate it with an artifact + version in the HCP Packer registry. - [Shell](/packer/docs/provisioners/shell) - run shell scripts on the machines image during a build. - [Local Shell](/packer/docs/provisioners/shell-local) - run shell scripts on the host running Packer during a build. From 0b416544767d8cb0a9614018987b6ece78d88764 Mon Sep 17 00:00:00 2001 From: Lucas Bajolet Date: Fri, 25 Oct 2024 11:32:06 -0400 Subject: [PATCH 17/24] packer_test: add integration tests for hcp-sbom --- .../hcp-sbom/provisioner_test.go | 151 ++++++++++++++++++ .../provisioner_tests/hcp-sbom/suite_test.go | 23 +++ .../hcp-sbom/templates/dest_is_dir.pkr.hcl | 36 +++++ .../dest_is_dir_with_trailing_slash.pkr.hcl | 36 +++++ .../dest_is_file_no_interm_dirs.pkr.hcl | 36 +++++ .../dest_is_file_with_interm_dirs.pkr.hcl | 36 +++++ .../hcp-sbom/templates/source_is_dir.pkr.hcl | 21 +++ .../templates/source_not_existing.pkr.hcl | 21 +++ 8 files changed, 360 insertions(+) create mode 100644 packer_test/provisioner_tests/hcp-sbom/provisioner_test.go create mode 100644 packer_test/provisioner_tests/hcp-sbom/suite_test.go create mode 100644 packer_test/provisioner_tests/hcp-sbom/templates/dest_is_dir.pkr.hcl create mode 100644 packer_test/provisioner_tests/hcp-sbom/templates/dest_is_dir_with_trailing_slash.pkr.hcl create mode 100644 packer_test/provisioner_tests/hcp-sbom/templates/dest_is_file_no_interm_dirs.pkr.hcl create mode 100644 packer_test/provisioner_tests/hcp-sbom/templates/dest_is_file_with_interm_dirs.pkr.hcl create mode 100644 packer_test/provisioner_tests/hcp-sbom/templates/source_is_dir.pkr.hcl create mode 100644 packer_test/provisioner_tests/hcp-sbom/templates/source_not_existing.pkr.hcl diff --git a/packer_test/provisioner_tests/hcp-sbom/provisioner_test.go b/packer_test/provisioner_tests/hcp-sbom/provisioner_test.go new file mode 100644 index 00000000000..81f5a8e06ea --- /dev/null +++ b/packer_test/provisioner_tests/hcp-sbom/provisioner_test.go @@ -0,0 +1,151 @@ +package plugin_tests + +import ( + "os" + + "github.com/hashicorp/packer/packer_test/common/check" +) + +func (ts *PackerHCPSbomTestSuite) TestSourceNotExisting() { + ts.SkipNoAcc() + + dir := ts.MakePluginDir() + defer dir.Cleanup() + + ts.PackerCommand().UsePluginDir(dir). + SetArgs("plugins", "install", "github.com/hashicorp/docker"). + Assert(check.MustSucceed()) + + ts.PackerCommand().UsePluginDir(dir). + AddEnv("HOME", os.Getenv("HOME")). + AddEnv("PATH", os.Getenv("PATH")). + SetArgs("build", "templates/source_not_existing.pkr.hcl"). + Assert(check.MustFail(), check.Grep("download failed for SBOM file")) +} + +// Greayed out because the communicator for the docker plugin does not return an error +// when downloading a full directory, instead it returns a 0-byte stream without an error. +// +// So the sbom provisioner fails with a validation error instead of a file not found type +// of error. +// +// func (ts *PackerHCPSbomTestSuite) TestSourceIsDir() { +// ts.SkipNoAcc() +// +// path, cleanup := ts.MakePluginDir() +// defer cleanup() +// +// ts.PackerCommand().UsePluginDir(path). +// SetArgs("plugins", "install", "github.com/hashicorp/docker"). +// Assert(check.MustSucceed()) +// +// ts.PackerCommand().UsePluginDir(path). +// SetArgs("build", "templates/source_is_dir.pkr.hcl"). +// Assert(check.MustFail(), check.Grep("download failed for SBOM file"), check.Dump(ts.T())) +// } + +// * output file - does not exist, and intermediate dirs don't exist +func (ts *PackerHCPSbomTestSuite) TestDestFile_NoIntermediateDirs() { + ts.SkipNoAcc() + + dir := ts.MakePluginDir() + defer dir.Cleanup() + + ts.PackerCommand().UsePluginDir(dir). + SetArgs("plugins", "install", "github.com/hashicorp/docker"). + Assert(check.MustSucceed()) + + ts.PackerCommand().UsePluginDir(dir). + AddEnv("HOME", os.Getenv("HOME")). + AddEnv("PATH", os.Getenv("PATH")). + SetArgs("build", "./templates/dest_is_file_no_interm_dirs.pkr.hcl"). + Assert(check.MustSucceed(), check.FileExists("sbom/sbom_cyclonedx.json", false)) + + os.RemoveAll("sbom") +} + +// * output file - does not exist, and intermediate dirs already exist +func (ts *PackerHCPSbomTestSuite) TestDestFile_WithIntermediateDirs() { + ts.SkipNoAcc() + + dir := ts.MakePluginDir() + defer dir.Cleanup() + + os.MkdirAll("sbom", 0755) + + ts.PackerCommand().UsePluginDir(dir). + SetArgs("plugins", "install", "github.com/hashicorp/docker"). + Assert(check.MustSucceed()) + + ts.PackerCommand().UsePluginDir(dir). + AddEnv("HOME", os.Getenv("HOME")). + AddEnv("PATH", os.Getenv("PATH")). + SetArgs("build", "./templates/dest_is_file_no_interm_dirs.pkr.hcl"). + Assert(check.MustSucceed(), check.FileExists("sbom/sbom_cyclonedx.json", false)) + + os.RemoveAll("sbom") +} + +// * output directory (without trailing slash) - directory exists +func (ts *PackerHCPSbomTestSuite) TestDestDir_NoTrailingSlash() { + ts.SkipNoAcc() + + dir := ts.MakePluginDir() + defer dir.Cleanup() + + os.MkdirAll("sbom", 0755) + + ts.PackerCommand().UsePluginDir(dir). + SetArgs("plugins", "install", "github.com/hashicorp/docker"). + Assert(check.MustSucceed()) + + ts.PackerCommand().UsePluginDir(dir). + AddEnv("HOME", os.Getenv("HOME")). + AddEnv("PATH", os.Getenv("PATH")). + SetArgs("build", "./templates/dest_is_dir.pkr.hcl"). + Assert(check.MustSucceed(), check.FileGlob("./sbom/packer-user-sbom-*.json")) + + os.RemoveAll("sbom") +} + +// * output directory (with trailing slash) - directory exists +func (ts *PackerHCPSbomTestSuite) TestDestDir_WithTrailingSlash() { + ts.SkipNoAcc() + + dir := ts.MakePluginDir() + defer dir.Cleanup() + + os.MkdirAll("sbom", 0755) + + ts.PackerCommand().UsePluginDir(dir). + SetArgs("plugins", "install", "github.com/hashicorp/docker"). + Assert(check.MustSucceed()) + + ts.PackerCommand().UsePluginDir(dir). + AddEnv("HOME", os.Getenv("HOME")). + AddEnv("PATH", os.Getenv("PATH")). + SetArgs("build", "./templates/dest_is_dir_with_trailing_slash.pkr.hcl"). + Assert(check.MustSucceed(), check.FileGlob("./sbom/packer-user-sbom-*.json")) + + os.RemoveAll("sbom") +} + +// * output directory (with trailing slash) - directory doesn't exist +func (ts *PackerHCPSbomTestSuite) TestDestDir_WithTrailingSlash_NoDir() { + ts.SkipNoAcc() + + dir := ts.MakePluginDir() + defer dir.Cleanup() + + ts.PackerCommand().UsePluginDir(dir). + SetArgs("plugins", "install", "github.com/hashicorp/docker"). + Assert(check.MustSucceed()) + + ts.PackerCommand().UsePluginDir(dir). + AddEnv("HOME", os.Getenv("HOME")). + AddEnv("PATH", os.Getenv("PATH")). + SetArgs("build", "./templates/dest_is_dir_with_trailing_slash.pkr.hcl"). + Assert(check.MustSucceed(), check.FileGlob("./sbom/packer-user-sbom-*.json")) + + os.RemoveAll("sbom") +} diff --git a/packer_test/provisioner_tests/hcp-sbom/suite_test.go b/packer_test/provisioner_tests/hcp-sbom/suite_test.go new file mode 100644 index 00000000000..a3855ebb660 --- /dev/null +++ b/packer_test/provisioner_tests/hcp-sbom/suite_test.go @@ -0,0 +1,23 @@ +package plugin_tests + +import ( + "testing" + + "github.com/hashicorp/packer/packer_test/common" + "github.com/stretchr/testify/suite" +) + +type PackerHCPSbomTestSuite struct { + *common.PackerTestSuite +} + +func Test_PackerPluginSuite(t *testing.T) { + baseSuite, cleanup := common.InitBaseSuite(t) + defer cleanup() + + ts := &PackerHCPSbomTestSuite{ + baseSuite, + } + + suite.Run(t, ts) +} diff --git a/packer_test/provisioner_tests/hcp-sbom/templates/dest_is_dir.pkr.hcl b/packer_test/provisioner_tests/hcp-sbom/templates/dest_is_dir.pkr.hcl new file mode 100644 index 00000000000..1a405a50bee --- /dev/null +++ b/packer_test/provisioner_tests/hcp-sbom/templates/dest_is_dir.pkr.hcl @@ -0,0 +1,36 @@ +packer { + required_plugins { + docker = { + version = ">= 1.0.0" + source = "github.com/hashicorp/docker" + } + } +} + +source "docker" "ubuntu" { + image = "ubuntu:20.04" + commit = true +} + +build { + sources = ["source.docker.ubuntu"] + + provisioner "shell" { + inline = [ + "apt-get update -y", + "apt-get install -y curl", + "bash -c \"$(curl -sSL https://install.mondoo.com/sh)\"" + ] + } + + provisioner "shell" { + inline = [ + "cnquery sbom --output cyclonedx-json --output-target /tmp/sbom_cyclonedx.json", + ] + } + + provisioner "hcp-sbom" { + source = "/tmp/sbom_cyclonedx.json" + destination = "./sbom" + } +} diff --git a/packer_test/provisioner_tests/hcp-sbom/templates/dest_is_dir_with_trailing_slash.pkr.hcl b/packer_test/provisioner_tests/hcp-sbom/templates/dest_is_dir_with_trailing_slash.pkr.hcl new file mode 100644 index 00000000000..9d9ca4506b1 --- /dev/null +++ b/packer_test/provisioner_tests/hcp-sbom/templates/dest_is_dir_with_trailing_slash.pkr.hcl @@ -0,0 +1,36 @@ +packer { + required_plugins { + docker = { + version = ">= 1.0.0" + source = "github.com/hashicorp/docker" + } + } +} + +source "docker" "ubuntu" { + image = "ubuntu:20.04" + commit = true +} + +build { + sources = ["source.docker.ubuntu"] + + provisioner "shell" { + inline = [ + "apt-get update -y", + "apt-get install -y curl", + "bash -c \"$(curl -sSL https://install.mondoo.com/sh)\"" + ] + } + + provisioner "shell" { + inline = [ + "cnquery sbom --output cyclonedx-json --output-target /tmp/sbom_cyclonedx.json", + ] + } + + provisioner "hcp-sbom" { + source = "/tmp/sbom_cyclonedx.json" + destination = "./sbom/" + } +} diff --git a/packer_test/provisioner_tests/hcp-sbom/templates/dest_is_file_no_interm_dirs.pkr.hcl b/packer_test/provisioner_tests/hcp-sbom/templates/dest_is_file_no_interm_dirs.pkr.hcl new file mode 100644 index 00000000000..9d4bcb2daec --- /dev/null +++ b/packer_test/provisioner_tests/hcp-sbom/templates/dest_is_file_no_interm_dirs.pkr.hcl @@ -0,0 +1,36 @@ +packer { + required_plugins { + docker = { + version = ">= 1.0.0" + source = "github.com/hashicorp/docker" + } + } +} + +source "docker" "ubuntu" { + image = "ubuntu:20.04" + commit = true +} + +build { + sources = ["source.docker.ubuntu"] + + provisioner "shell" { + inline = [ + "apt-get update -y", + "apt-get install -y curl", + "bash -c \"$(curl -sSL https://install.mondoo.com/sh)\"" + ] + } + + provisioner "shell" { + inline = [ + "cnquery sbom --output cyclonedx-json --output-target /tmp/sbom_cyclonedx.json", + ] + } + + provisioner "hcp-sbom" { + source = "/tmp/sbom_cyclonedx.json" + destination = "./sbom/sbom_cyclonedx.json" + } +} diff --git a/packer_test/provisioner_tests/hcp-sbom/templates/dest_is_file_with_interm_dirs.pkr.hcl b/packer_test/provisioner_tests/hcp-sbom/templates/dest_is_file_with_interm_dirs.pkr.hcl new file mode 100644 index 00000000000..37ccbcc3b60 --- /dev/null +++ b/packer_test/provisioner_tests/hcp-sbom/templates/dest_is_file_with_interm_dirs.pkr.hcl @@ -0,0 +1,36 @@ +packer { + required_plugins { + docker = { + version = ">= 1.0.0" + source = "github.com/hashicorp/docker" + } + } +} + +source "docker" "ubuntu" { + image = "ubuntu:20.04" + commit = true +} + +build { + sources = ["source.docker.ubuntu"] + + provisioner "shell" { + inline = [ + "apt-get update -y", + "apt-get install -y curl", + "bash -c \"$(curl -sSL https://install.mondoo.com/sh)\"" + ] + } + + provisioner "shell" { + inline = [ + "cnquery sbom --output cyclonedx-json --output-target /tmp/sbom_cyclonedx.json", + ] + } + + provisioner "hcp-sbom" { + source = "/tmp/sbom_cyclonedx.json" + destination = "./sbom/sbom_cyclonedx" + } +} diff --git a/packer_test/provisioner_tests/hcp-sbom/templates/source_is_dir.pkr.hcl b/packer_test/provisioner_tests/hcp-sbom/templates/source_is_dir.pkr.hcl new file mode 100644 index 00000000000..02522488d52 --- /dev/null +++ b/packer_test/provisioner_tests/hcp-sbom/templates/source_is_dir.pkr.hcl @@ -0,0 +1,21 @@ +packer { + required_plugins { + docker = { + version = ">= 1.0.0" + source = "github.com/hashicorp/docker" + } + } +} + +source "docker" "ubuntu" { + image = "ubuntu:20.04" + commit = true +} + +build { + sources = ["source.docker.ubuntu"] + + provisioner "hcp-sbom" { + source = "/tmp" + } +} diff --git a/packer_test/provisioner_tests/hcp-sbom/templates/source_not_existing.pkr.hcl b/packer_test/provisioner_tests/hcp-sbom/templates/source_not_existing.pkr.hcl new file mode 100644 index 00000000000..a66b9968501 --- /dev/null +++ b/packer_test/provisioner_tests/hcp-sbom/templates/source_not_existing.pkr.hcl @@ -0,0 +1,21 @@ +packer { + required_plugins { + docker = { + version = ">= 1.0.0" + source = "github.com/hashicorp/docker" + } + } +} + +source "docker" "ubuntu" { + image = "ubuntu:20.04" + commit = true +} + +build { + sources = ["source.docker.ubuntu"] + + provisioner "hcp-sbom" { + source = "/tmp/sbom_cyclonedx.json" + } +} From 15ed9a0a66806c1d1dab9bb49bfc7006faa65b1a Mon Sep 17 00:00:00 2001 From: Jenna Goldstrich Date: Fri, 27 Sep 2024 13:51:13 -0700 Subject: [PATCH 18/24] hcp: integrate SBOM upload to HCP code Since packer now supports keeping track of SBOMs produced during a build, we add the code to integrate those changes into the internal/hcp package, so we do upload them on build completion. --- internal/hcp/registry/types.bucket.go | 39 ++++++++++++++++++++++++++ internal/hcp/registry/types.builds.go | 4 +++ internal/hcp/registry/types.version.go | 3 ++ packer/build.go | 2 ++ 4 files changed, 48 insertions(+) diff --git a/internal/hcp/registry/types.bucket.go b/internal/hcp/registry/types.bucket.go index 17f3e34c027..ab068a30866 100644 --- a/internal/hcp/registry/types.bucket.go +++ b/internal/hcp/registry/types.bucket.go @@ -12,7 +12,10 @@ import ( "sync" "time" + "github.com/hashicorp/packer/packer" + "github.com/hashicorp/go-multierror" + "github.com/hashicorp/hcp-sdk-go/clients/cloud-packer-service/stable/2023-01-01/client/packer_service" hcpPackerModels "github.com/hashicorp/hcp-sdk-go/clients/cloud-packer-service/stable/2023-01-01/models" packerSDK "github.com/hashicorp/packer-plugin-sdk/packer" packerSDKRegistry "github.com/hashicorp/packer-plugin-sdk/packer/registry/image" @@ -222,6 +225,35 @@ func (bucket *Bucket) UpdateBuildStatus( return nil } +func (bucket *Bucket) uploadSbom(ctx context.Context, buildName string, sbom packer.SBOM) error { + buildToUpdate, err := bucket.Version.Build(buildName) + if err != nil { + return err + } + + log.Println( + "[TRACE] jennajenna uploadsbom called", buildToUpdate.ID, + ) + if buildToUpdate.ID == "" { + return fmt.Errorf("the build for the component %q does not have a valid id", buildName) + } + _, err = bucket.client.Packer.PackerServiceUploadSbom( + &packer_service.PackerServiceUploadSbomParams{ + Context: ctx, + BucketName: bucket.Name, + Fingerprint: bucket.Version.Fingerprint, + BuildID: buildToUpdate.ID, + Body: &hcpPackerModels.HashicorpCloudPacker20230101UploadSbomBody{ + CompressedSbom: sbom.CompressedData, + Name: sbom.Name, + Format: sbom.Format, + }, + }, + nil, + ) + return err +} + // markBuildComplete should be called to set a build on the HCP Packer registry to DONE. // Upon a successful call markBuildComplete will publish all artifacts created by the named build, // and set the build to done. A build with no artifacts can not be set to DONE. @@ -673,6 +705,13 @@ func (bucket *Bucket) completeBuild( } } + for _, sbom := range build.CompressedSboms { + err = bucket.uploadSbom(ctx, buildName, sbom) + if err != nil { + return packerSDKArtifacts, fmt.Errorf("Failed to upload sboms %s", err) + } + } + parErr := bucket.markBuildComplete(ctx, buildName) if parErr != nil { return packerSDKArtifacts, fmt.Errorf( diff --git a/internal/hcp/registry/types.builds.go b/internal/hcp/registry/types.builds.go index dc7e132762c..0ca531c2c1c 100644 --- a/internal/hcp/registry/types.builds.go +++ b/internal/hcp/registry/types.builds.go @@ -6,6 +6,8 @@ package registry import ( "fmt" + "github.com/hashicorp/packer/packer" + hcpPackerModels "github.com/hashicorp/hcp-sdk-go/clients/cloud-packer-service/stable/2023-01-01/models" packerSDKRegistry "github.com/hashicorp/packer-plugin-sdk/packer/registry/image" ) @@ -20,6 +22,8 @@ type Build struct { Artifacts map[string]packerSDKRegistry.Image Status hcpPackerModels.HashicorpCloudPacker20230101BuildStatus Metadata hcpPackerModels.HashicorpCloudPacker20230101BuildMetadata + + CompressedSboms []packer.SBOM } // NewBuildFromCloudPackerBuild converts a HashicorpCloudPackerBuild to a local build that can be tracked and diff --git a/internal/hcp/registry/types.version.go b/internal/hcp/registry/types.version.go index 0caf6229c11..819e09e4602 100644 --- a/internal/hcp/registry/types.version.go +++ b/internal/hcp/registry/types.version.go @@ -205,5 +205,8 @@ func (version *Version) AddMetadataToBuild( buildToUpdate.Metadata.Vcs = globalMetadata.Vcs buildToUpdate.Metadata.Cicd = globalMetadata.Cicd + // TODO IMO this shouldn't be metadata + buildToUpdate.CompressedSboms = buildMetadata.SBOMs + return nil } diff --git a/packer/build.go b/packer/build.go index eade2625dd6..d23637f67a2 100644 --- a/packer/build.go +++ b/packer/build.go @@ -55,6 +55,7 @@ type CoreBuild struct { } type SBOM struct { + Name string Format string CompressedData []byte } @@ -313,6 +314,7 @@ func (b *CoreBuild) Run(ctx context.Context, originalUi packersdk.Ui) ([]packers sbomInternalProvisioner, ok := p.Provisioner.(*SBOMInternalProvisioner) if ok { sbom := SBOM{ + Name: sbomInternalProvisioner.SBOMName, Format: sbomInternalProvisioner.SBOMFormat, CompressedData: sbomInternalProvisioner.CompressedData, } From ce7c21a27af50fc48e7d0507bf9edd07322866f3 Mon Sep 17 00:00:00 2001 From: Lucas Bajolet Date: Tue, 12 Nov 2024 13:36:04 -0500 Subject: [PATCH 19/24] hcp: wrap completeBuild to mark as failed on error When a build cannot be completed without errors, the build state was left as running, unless the build explicitly failed, which meant that HCP Packer would be responsible for changing the status after the heartbeats for the build stopped being sent for two 5m periods. This commit changes this behaviour, by explicitly marking the build as failed if something did not work while trying to complete a build on HCP Packer, even if the local Packer core build succeeded before that. --- internal/hcp/registry/types.bucket.go | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/internal/hcp/registry/types.bucket.go b/internal/hcp/registry/types.bucket.go index ab068a30866..b9be5fd81b1 100644 --- a/internal/hcp/registry/types.bucket.go +++ b/internal/hcp/registry/types.bucket.go @@ -642,7 +642,6 @@ func (bucket *Bucket) completeBuild( doneCh, ok := bucket.RunningBuilds[buildName] if !ok { log.Print("[ERROR] done build does not have an entry in the heartbeat table, state will be inconsistent.") - } else { log.Printf("[TRACE] signal stopping heartbeats") // Stop heartbeating @@ -662,6 +661,23 @@ func (bucket *Bucket) completeBuild( return packerSDKArtifacts, fmt.Errorf("build failed, not uploading artifacts") } + artifacts, err := bucket.doCompleteBuild(ctx, buildName, packerSDKArtifacts, buildErr) + if err != nil { + err := bucket.UpdateBuildStatus(ctx, buildName, hcpPackerModels.HashicorpCloudPacker20230101BuildStatusBUILDFAILED) + if err != nil { + log.Printf("[ERROR] failed to update build %q status to FAILED: %s", buildName, err) + } + } + + return artifacts, err +} + +func (bucket *Bucket) doCompleteBuild( + ctx context.Context, + buildName string, + packerSDKArtifacts []packerSDK.Artifact, + buildErr error, +) ([]packerSDK.Artifact, error) { for _, art := range packerSDKArtifacts { var sdkImages []packerSDKRegistry.Image decoder, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{ From 465319a320a4f409f28d636f685b91397d3cd639 Mon Sep 17 00:00:00 2001 From: Lucas Bajolet Date: Wed, 13 Nov 2024 15:36:45 -0500 Subject: [PATCH 20/24] command: exit non-zero if uploading to HCP failed In the current state, a Packer build that succeeds but fails to push its metadata to HCP for reasons other than a lack of artifact will always succeed from the perspective of a user invoking `packer build`. This can be a bit misleading, as users may expect their artifacts to appear on HCP Packer if their build succeeded on Packer Core, so this commit changes this behaviour, instead reporting HCP errors as a real error if the build failed, so packer returns a non-zero error code if this happens. --- command/build.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/command/build.go b/command/build.go index 58b548b008f..3caa0b9d842 100644 --- a/command/build.go +++ b/command/build.go @@ -315,6 +315,15 @@ Check that you are using an HCP Ready integration before trying again: artifacts.Unlock() } } + + // If the build succeeded but uploading to HCP failed, + // Packer should exit non-zero, so we re-assign the + // error to account for this case. + if hcperr != nil && err == nil { + errs.Lock() + errs.m[name] = hcperr + errs.Unlock() + } }() if cla.Debug { From 2edc2650bef41a2c73bd64deeebe001d452a4bcf Mon Sep 17 00:00:00 2001 From: Jenna Goldstrich Date: Mon, 6 Jan 2025 18:02:08 -0800 Subject: [PATCH 21/24] hcp: use enum for HCP SBOM upload Since the protos for uploading an SBOM for a build have been changed to use an enumeration instead of a plain string with the latest revisions to the HCP Packer SBOM support feature, we update how we reference those values for the SBOM format to use that enum instead. --- internal/hcp/registry/types.bucket.go | 2 +- packer/build.go | 3 ++- packer/provisioner.go | 3 ++- provisioner/hcp-sbom/provisioner.go | 5 +++-- provisioner/hcp-sbom/validate.go | 7 ++++--- 5 files changed, 12 insertions(+), 8 deletions(-) diff --git a/internal/hcp/registry/types.bucket.go b/internal/hcp/registry/types.bucket.go index b9be5fd81b1..184a46e2ad3 100644 --- a/internal/hcp/registry/types.bucket.go +++ b/internal/hcp/registry/types.bucket.go @@ -246,7 +246,7 @@ func (bucket *Bucket) uploadSbom(ctx context.Context, buildName string, sbom pac Body: &hcpPackerModels.HashicorpCloudPacker20230101UploadSbomBody{ CompressedSbom: sbom.CompressedData, Name: sbom.Name, - Format: sbom.Format, + Format: &sbom.Format, }, }, nil, diff --git a/packer/build.go b/packer/build.go index d23637f67a2..4a311461e96 100644 --- a/packer/build.go +++ b/packer/build.go @@ -9,6 +9,7 @@ import ( "log" "sync" + hcpPackerModels "github.com/hashicorp/hcp-sdk-go/clients/cloud-packer-service/stable/2023-01-01/models" "github.com/hashicorp/packer-plugin-sdk/common" packersdk "github.com/hashicorp/packer-plugin-sdk/packer" "github.com/hashicorp/packer-plugin-sdk/packerbuilderdata" @@ -56,7 +57,7 @@ type CoreBuild struct { type SBOM struct { Name string - Format string + Format hcpPackerModels.HashicorpCloudPacker20230101SbomFormat CompressedData []byte } diff --git a/packer/provisioner.go b/packer/provisioner.go index 24e20b3a247..4be4f99ddee 100644 --- a/packer/provisioner.go +++ b/packer/provisioner.go @@ -12,6 +12,7 @@ import ( hcpSbomProvisioner "github.com/hashicorp/packer/provisioner/hcp-sbom" + hcpPackerModels "github.com/hashicorp/hcp-sdk-go/clients/cloud-packer-service/stable/2023-01-01/models" "github.com/klauspost/compress/zstd" "time" @@ -249,7 +250,7 @@ func (p *DebuggedProvisioner) Provision(ctx context.Context, ui packersdk.Ui, co type SBOMInternalProvisioner struct { Provisioner packersdk.Provisioner CompressedData []byte - SBOMFormat string + SBOMFormat hcpPackerModels.HashicorpCloudPacker20230101SbomFormat SBOMName string } diff --git a/provisioner/hcp-sbom/provisioner.go b/provisioner/hcp-sbom/provisioner.go index cbc515c13d1..cf03e567067 100644 --- a/provisioner/hcp-sbom/provisioner.go +++ b/provisioner/hcp-sbom/provisioner.go @@ -20,6 +20,7 @@ import ( "path/filepath" "github.com/hashicorp/hcl/v2/hcldec" + hcpPackerModels "github.com/hashicorp/hcp-sdk-go/clients/cloud-packer-service/stable/2023-01-01/models" "github.com/hashicorp/packer-plugin-sdk/common" packersdk "github.com/hashicorp/packer-plugin-sdk/packer" "github.com/hashicorp/packer-plugin-sdk/template/config" @@ -116,8 +117,8 @@ type PackerSBOM struct { RawSBOM []byte `json:"raw_sbom"` // Format is the format detected by the provisioner // - // Supported values: `spdx` or `cyclonedx` - Format string `json:"format"` + // Supported values: `SPDX` or `CYCLONEDX` + Format hcpPackerModels.HashicorpCloudPacker20230101SbomFormat `json:"format"` // Name is the name of the SBOM to be set on HCP Packer // // If unset, HCP Packer will generate one diff --git a/provisioner/hcp-sbom/validate.go b/provisioner/hcp-sbom/validate.go index 4f17a4ac0de..7343dcb9bbb 100644 --- a/provisioner/hcp-sbom/validate.go +++ b/provisioner/hcp-sbom/validate.go @@ -6,6 +6,7 @@ import ( "strings" "github.com/CycloneDX/cyclonedx-go" + hcpPackerModels "github.com/hashicorp/hcp-sdk-go/clients/cloud-packer-service/stable/2023-01-01/models" spdxjson "github.com/spdx/tools-golang/json" ) @@ -61,11 +62,11 @@ func validateSPDX(content []byte) error { } // validateSBOM validates the SBOM file and returns the format of the SBOM. -func validateSBOM(content []byte) (string, error) { +func validateSBOM(content []byte) (hcpPackerModels.HashicorpCloudPacker20230101SbomFormat, error) { // Try validating as SPDX spdxErr := validateSPDX(content) if spdxErr == nil { - return "spdx", nil + return hcpPackerModels.HashicorpCloudPacker20230101SbomFormatSPDX, nil } if vErr, ok := spdxErr.(*ValidationError); ok { @@ -74,7 +75,7 @@ func validateSBOM(content []byte) (string, error) { cycloneDxErr := validateCycloneDX(content) if cycloneDxErr == nil { - return "cyclonedx", nil + return hcpPackerModels.HashicorpCloudPacker20230101SbomFormatCYCLONEDX, nil } if vErr, ok := cycloneDxErr.(*ValidationError); ok { From 2213e5ebba382308f131cca4465fb5275064bb76 Mon Sep 17 00:00:00 2001 From: Devashish Date: Fri, 1 Nov 2024 12:34:55 -0400 Subject: [PATCH 22/24] website: add docs for the hcp-sbom provisioner --- website/content/community-plugins.mdx | 1 + .../content/docs/provisioners/hcp-sbom.mdx | 130 ++++++++++++++++++ website/content/docs/provisioners/index.mdx | 1 + website/data/docs-nav-data.json | 4 + 4 files changed, 136 insertions(+) create mode 100644 website/content/docs/provisioners/hcp-sbom.mdx diff --git a/website/content/community-plugins.mdx b/website/content/community-plugins.mdx index fa245b73dfd..43a427c3f95 100644 --- a/website/content/community-plugins.mdx +++ b/website/content/community-plugins.mdx @@ -24,6 +24,7 @@ HashiCorp maintainers for advice on how to get started contributing. ## Provisioners - File +- HCP SBOM - InSpec - PowerShell - Shell diff --git a/website/content/docs/provisioners/hcp-sbom.mdx b/website/content/docs/provisioners/hcp-sbom.mdx new file mode 100644 index 00000000000..a90b5bf8886 --- /dev/null +++ b/website/content/docs/provisioners/hcp-sbom.mdx @@ -0,0 +1,130 @@ +--- +description: | + The `hcp-sbom` Packer provisioner downloads an SBOM file from the guest VM and + sends it to HCP Packer when the build is done. +page_title: HCP SBOM - Provisioners +--- + + + + + +# HCP SBOM Provisioner + +Type: `hcp-sbom` + +The `hcp-sbom` Packer provisioner downloads an SBOM file from the guest machine +and sends it to HCP Packer when the build is complete (only if the template is +HCP-enabled). The SBOM file is automatically removed at the end of the process. +If you want to retain a copy of the SBOM file, you can specify the +`destination` option in the provisioner. + +Currently, we support `CycloneDX` and `SPDX` SBOM formats in `JSON`. + +## Basic Example + + +In HCL2: + +```hcl +provisioner "hcp-sbom" { + source = "/tmp/sbom_cyclonedx.json" + destination = "./sbom/sbom_cyclonedx.json" +} +``` + +In JSON: + +```json +{ + "type": "hcp-sbom", + "source": "/tmp/sbom_cyclonedx.json", + "destination": "./sbom/sbom_cyclonedx.json" +} +``` + + + +## Configuration Reference + +Required Parameters: + +@include 'provisioner/hcp-sbom/Config-required.mdx' + +Optional Parameters: + +@include '/provisioner/hcp-sbom/Config-not-required.mdx' + +## Example Usage + + +In HCL2: + +```hcl +packer { + required_plugins { + docker = { + version = ">= 1.0.0" + source = "github.com/hashicorp/docker" + } + } +} + +source "docker" "ubuntu" { + image = "ubuntu:20.04" + commit = true +} + +build { + sources = ["source.docker.ubuntu"] + + hcp_packer_registry { + bucket_name = "test-bucket" + } + + + provisioner "shell" { + inline = [ + "apt-get update -y", + "apt-get install -y curl gpg", + "bash -c \"$(curl -sSL https://install.mondoo.com/sh)\"", + "cnquery sbom --output cyclonedx-json --output-target /tmp/sbom_cyclonedx.json", + ] + } + + provisioner "hcp-sbom" { + source = "/tmp/sbom_cyclonedx.json" + destination = "./sbom" + } +} +``` + +In JSON: + +```json +{ + "builders": [ + { + "type": "docker", + "image": "ubuntu:20.04", + "commit": true + } + ], + "provisioners": [ + { + "type": "shell", + "inline": [ + "apt-get update -y", + "apt-get install -y curl", + "bash -c \"$(curl -sSL https://install.mondoo.com/sh)\"", + "cnquery sbom --output cyclonedx-json --output-target /tmp/sbom_cyclonedx.json" + ] + }, + { + "type": "hcp-sbom", + "source": "/tmp/sbom_cyclonedx.json", + "destination": "./sbom" + } + ] +} +``` diff --git a/website/content/docs/provisioners/index.mdx b/website/content/docs/provisioners/index.mdx index e6144beaef4..6be0124557f 100644 --- a/website/content/docs/provisioners/index.mdx +++ b/website/content/docs/provisioners/index.mdx @@ -20,6 +20,7 @@ The following provisioners are included with Packer: - [Breakpoint](/packer/docs/provisioners/breakpoint) - pause until the user presses `Enter` to resume a build. - [File](/packer/docs/provisioners/file) - upload files to machines image during a build. +- [HCP SBOM](/packer/docs/provisioners/hcp-sbom) - download SBOM file to machines and send to HCP Packer during a build. - [Shell](/packer/docs/provisioners/shell) - run shell scripts on the machines image during a build. - [Local Shell](/packer/docs/provisioners/shell-local) - run shell scripts on the host running Packer during a build. diff --git a/website/data/docs-nav-data.json b/website/data/docs-nav-data.json index 51b173740f0..65ed0cb2dce 100644 --- a/website/data/docs-nav-data.json +++ b/website/data/docs-nav-data.json @@ -792,6 +792,10 @@ "title": "File", "path": "provisioners/file" }, + { + "title": "HCP SBOM", + "path": "provisioners/hcp-sbom" + }, { "title": "PowerShell", "path": "provisioners/powershell" From 3cb5ad3ef1c0e3c8cf19f0e6293bc2b25b21c0bc Mon Sep 17 00:00:00 2001 From: Brian McClain Date: Tue, 21 Jan 2025 12:28:01 -0500 Subject: [PATCH 23/24] Cleanup from branch conflict --- packer_test/common/commands.go | 2 - .../hcp-sbom/priovisioner_test.go | 139 ------------------ 2 files changed, 141 deletions(-) delete mode 100644 packer_test/provisioner_tests/hcp-sbom/priovisioner_test.go diff --git a/packer_test/common/commands.go b/packer_test/common/commands.go index 52681f6c375..869a6b27811 100644 --- a/packer_test/common/commands.go +++ b/packer_test/common/commands.go @@ -45,8 +45,6 @@ func (ts *PackerTestSuite) PackerCommand() *packerCommand { // make them as self-contained and quick as possible. // Removing telemetry here is probably for the best. "CHECKPOINT_DISABLE": "1", - "HOME": os.Getenv("HOME"), - "PATH": os.Getenv("PATH"), }, t: ts.T(), } diff --git a/packer_test/provisioner_tests/hcp-sbom/priovisioner_test.go b/packer_test/provisioner_tests/hcp-sbom/priovisioner_test.go deleted file mode 100644 index 0d82c6441e7..00000000000 --- a/packer_test/provisioner_tests/hcp-sbom/priovisioner_test.go +++ /dev/null @@ -1,139 +0,0 @@ -package plugin_tests - -import ( - "os" - - "github.com/hashicorp/packer/packer_test/common/check" -) - -func (ts *PackerHCPSbomTestSuite) TestSourceNotExisting() { - ts.SkipNoAcc() - - dir := ts.MakePluginDir() - defer dir.Cleanup() - - ts.PackerCommand().UsePluginDir(dir). - SetArgs("plugins", "install", "github.com/hashicorp/docker"). - Assert(check.MustSucceed()) - - ts.PackerCommand().UsePluginDir(dir). - SetArgs("build", "templates/source_not_existing.pkr.hcl"). - Assert(check.MustFail(), check.Grep("download failed for SBOM file")) -} - -// Greayed out because the communicator for the docker plugin does not return an error -// when downloading a full directory, instead it returns a 0-byte stream without an error. -// -// So the sbom provisioner fails with a validation error instead of a file not found type -// of error. -// -// func (ts *PackerHCPSbomTestSuite) TestSourceIsDir() { -// ts.SkipNoAcc() -// -// path, cleanup := ts.MakePluginDir() -// defer cleanup() -// -// ts.PackerCommand().UsePluginDir(path). -// SetArgs("plugins", "install", "github.com/hashicorp/docker"). -// Assert(check.MustSucceed()) -// -// ts.PackerCommand().UsePluginDir(path). -// SetArgs("build", "templates/source_is_dir.pkr.hcl"). -// Assert(check.MustFail(), check.Grep("download failed for SBOM file"), check.Dump(ts.T())) -// } - -// * output file - does not exist, and intermediate dirs don't exist -func (ts *PackerHCPSbomTestSuite) TestDestFile_NoIntermediateDirs() { - ts.SkipNoAcc() - - dir := ts.MakePluginDir() - defer dir.Cleanup() - - ts.PackerCommand().UsePluginDir(dir). - SetArgs("plugins", "install", "github.com/hashicorp/docker"). - Assert(check.MustSucceed()) - - ts.PackerCommand().UsePluginDir(dir). - SetArgs("build", "./templates/dest_is_file_no_interm_dirs.pkr.hcl"). - Assert(check.MustSucceed(), check.FileExists("sbom/sbom_cyclonedx.json", false)) - - os.RemoveAll("sbom") -} - -// * output file - does not exist, and intermediate dirs already exist -func (ts *PackerHCPSbomTestSuite) TestDestFile_WithIntermediateDirs() { - ts.SkipNoAcc() - - dir := ts.MakePluginDir() - defer dir.Cleanup() - - os.MkdirAll("sbom", 0755) - - ts.PackerCommand().UsePluginDir(dir). - SetArgs("plugins", "install", "github.com/hashicorp/docker"). - Assert(check.MustSucceed()) - - ts.PackerCommand().UsePluginDir(dir). - SetArgs("build", "./templates/dest_is_file_no_interm_dirs.pkr.hcl"). - Assert(check.MustSucceed(), check.FileExists("sbom/sbom_cyclonedx.json", false)) - - os.RemoveAll("sbom") -} - -// * output directory (without trailing slash) - directory exists -func (ts *PackerHCPSbomTestSuite) TestDestDir_NoTrailingSlash() { - ts.SkipNoAcc() - - dir := ts.MakePluginDir() - defer dir.Cleanup() - - os.MkdirAll("sbom", 0755) - - ts.PackerCommand().UsePluginDir(dir). - SetArgs("plugins", "install", "github.com/hashicorp/docker"). - Assert(check.MustSucceed()) - - ts.PackerCommand().UsePluginDir(dir). - SetArgs("build", "./templates/dest_is_dir.pkr.hcl"). - Assert(check.MustSucceed(), check.FileGlob("./sbom/packer-user-sbom-*.json")) - - os.RemoveAll("sbom") -} - -// * output directory (with trailing slash) - directory exists -func (ts *PackerHCPSbomTestSuite) TestDestDir_WithTrailingSlash() { - ts.SkipNoAcc() - - dir := ts.MakePluginDir() - defer dir.Cleanup() - - os.MkdirAll("sbom", 0755) - - ts.PackerCommand().UsePluginDir(dir). - SetArgs("plugins", "install", "github.com/hashicorp/docker"). - Assert(check.MustSucceed()) - - ts.PackerCommand().UsePluginDir(dir). - SetArgs("build", "./templates/dest_is_dir_with_trailing_slash.pkr.hcl"). - Assert(check.MustSucceed(), check.FileGlob("./sbom/packer-user-sbom-*.json")) - - os.RemoveAll("sbom") -} - -// * output directory (with trailing slash) - directory doesn't exist -func (ts *PackerHCPSbomTestSuite) TestDestDir_WithTrailingSlash_NoDir() { - ts.SkipNoAcc() - - dir := ts.MakePluginDir() - defer dir.Cleanup() - - ts.PackerCommand().UsePluginDir(dir). - SetArgs("plugins", "install", "github.com/hashicorp/docker"). - Assert(check.MustSucceed()) - - ts.PackerCommand().UsePluginDir(dir). - SetArgs("build", "./templates/dest_is_dir_with_trailing_slash.pkr.hcl"). - Assert(check.MustSucceed(), check.FileGlob("./sbom/packer-user-sbom-*.json")) - - os.RemoveAll("sbom") -} From 4a0217fef2115c0ef3b4c49f1e67d77c01d10186 Mon Sep 17 00:00:00 2001 From: Brian McClain Date: Tue, 21 Jan 2025 12:41:05 -0500 Subject: [PATCH 24/24] Run make generate --- .../hcp-sbom/Config-not-required.mdx | 27 +++++++------------ .../provisioner/hcp-sbom/Config-required.mdx | 5 ++-- 2 files changed, 11 insertions(+), 21 deletions(-) diff --git a/website/content/partials/provisioner/hcp-sbom/Config-not-required.mdx b/website/content/partials/provisioner/hcp-sbom/Config-not-required.mdx index 871e7a5adeb..fbba4f3c852 100644 --- a/website/content/partials/provisioner/hcp-sbom/Config-not-required.mdx +++ b/website/content/partials/provisioner/hcp-sbom/Config-not-required.mdx @@ -1,23 +1,14 @@ -- `destination` (string) - Destination is an optional field that specifies the path where the SBOM - file will be downloaded to for the user. - The 'Destination' must be a writable location. If the destination is a file, - the SBOM will be saved or overwritten at that path. If the destination is - a directory, a file will be created within the directory to store the SBOM. - Any parent directories for the destination must already exist and be - writable by the provisioning user (generally not root), otherwise, - a "Permission Denied" error will occur. If the source path is a file, - it is recommended that the destination path be a file as well. +- `destination` (string) - The path on the local machine to store a copy of the SBOM file. + You can specify an absolute or a path relative to the working directory + when you execute the Packer build. If the file already exists on the + local machine, Packer overwrites the file. If the destination is a + directory, the directory must already exist. -- `sbom_name` (string) - The name to give the SBOM when uploaded on HCP Packer - - By default this will be generated, but if you prefer to have a name - of your choosing, you can enter it here. - The name must match the following regexp: `[a-zA-Z0-9_-]{3,36}` - - Note: it must be unique for a single build, otherwise the build will - fail when uploading the SBOMs to HCP Packer, and so will the Packer - build command. +- `sbom_name` (string) - The name of the SBOM file stored in HCP Packer. + If omitted, HCP Packer uses the build fingerprint as the file name. + This value must be between three and 36 characters from the following set: `[A-Za-z0-9_-]`. + You must specify a unique name for each build in an artifact version. diff --git a/website/content/partials/provisioner/hcp-sbom/Config-required.mdx b/website/content/partials/provisioner/hcp-sbom/Config-required.mdx index 2f227c2b0ff..4df8744eb32 100644 --- a/website/content/partials/provisioner/hcp-sbom/Config-required.mdx +++ b/website/content/partials/provisioner/hcp-sbom/Config-required.mdx @@ -1,7 +1,6 @@ -- `source` (string) - Source is a required field that specifies the path to the SBOM file that - needs to be downloaded. - It can be a file path or a URL. +- `source` (string) - The file path or URL to the SBOM file in the Packer artifact. + This file must either be in the SPDX or CycloneDX format.