Skip to content

Commit f543a87

Browse files
kacpersawkacpersaw
authored
chore: cache terraform providers for workspaces terraform tests (#20603)
Fixes flaky `TestWorkspaceTagsTerraform` and `TestWorkspaceTemplateParamsChange` tests that were failing with `connection reset by peer` errors when downloading the coder/coder provider. This applies the same caching solution which was done in #17373 1. Extracts provider caching logic into `testutil/terraform_cache.go` 2. Updates TestProvision to use the shared caching helpers 3. Updates enterprise workspace tests to use the shared caching helpers The cache is persisted at `~/.cache/coderv2-test/` and automatically cached between CI runs via existing GitHub Actions cache setup. Closes coder/internal#607
1 parent e49c917 commit f543a87

File tree

3 files changed

+196
-211
lines changed

3 files changed

+196
-211
lines changed

‎enterprise/coderd/workspaces_test.go‎

Lines changed: 9 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,6 @@ import (
77
"encoding/json"
88
"fmt"
99
"net/http"
10-
"os"
11-
"os/exec"
1210
"path/filepath"
1311
"strings"
1412
"sync/atomic"
@@ -3390,51 +3388,19 @@ func workspaceTagsTerraform(t *testing.T, tc testWorkspaceTagsTerraformCase, dyn
33903388
}
33913389
}
33923390

3393-
// downloadProviders is a test helper that creates a temporary file and writes a
3394-
// terraform CLI config file with a provider_installation stanza for coder/coder
3395-
// using dev_overrides. It also fetches the latest provider release from GitHub
3396-
// and extracts the binary to the temporary dir. It is the responsibility of the
3397-
// caller to set TF_CLI_CONFIG_FILE.
3391+
// downloadProviders is a test helper that caches Terraform providers and returns
3392+
// the path to a Terraform CLI config file that uses the cached providers.
3393+
// This uses the shared testutil caching infrastructure to avoid re-downloading
3394+
// providers on every test run. It is the responsibility of the caller to set
3395+
// TF_CLI_CONFIG_FILE.
33983396
func downloadProviders(t *testing.T, providersTf string) string {
33993397
t.Helper()
3400-
// We firstly write a Terraform CLI config file to a temporary directory:
3401-
var (
3402-
tempDir = t.TempDir()
3403-
cacheDir = filepath.Join(tempDir, ".cache")
3404-
providersTfPath = filepath.Join(tempDir, "providers.tf")
3405-
cliConfigPath = filepath.Join(tempDir, "local.tfrc")
3406-
)
34073398

3408-
// Write files to disk
3409-
require.NoError(t, os.MkdirAll(cacheDir, os.ModePerm|os.ModeDir))
3410-
require.NoError(t, os.WriteFile(providersTfPath, []byte(providersTf), os.ModePerm)) // nolint:gosec
3411-
cliConfigTemplate := `
3412-
provider_installation {
3413-
filesystem_mirror {
3414-
path = %q
3415-
include = ["*/*/*"]
3416-
}
3417-
direct {
3418-
exclude = ["*/*/*"]
3419-
}
3420-
}`
3421-
err := os.WriteFile(cliConfigPath, []byte(fmt.Sprintf(cliConfigTemplate, cacheDir)), os.ModePerm) // nolint:gosec
3422-
require.NoError(t, err, "failed to write %s", cliConfigPath)
3423-
3424-
ctx := testutil.Context(t, testutil.WaitLong)
3425-
3426-
// Run terraform providers mirror to mirror required providers to cacheDir
3427-
cmd := exec.CommandContext(ctx, "terraform", "providers", "mirror", cacheDir)
3428-
cmd.Env = os.Environ() // without this terraform may complain about path
3429-
cmd.Env = append(cmd.Env, "TF_CLI_CONFIG_FILE="+cliConfigPath)
3430-
cmd.Dir = tempDir
3431-
out, err := cmd.CombinedOutput()
3432-
if !assert.NoError(t, err) {
3433-
t.Log("failed to download providers:")
3434-
t.Log(string(out))
3435-
t.FailNow()
3436-
}
3399+
cacheRootDir := filepath.Join(testutil.PersistentCacheDir(t), "terraform_workspace_tags_test")
3400+
templateFiles := map[string]string{"providers.tf": providersTf}
3401+
testName := "TestWorkspaceTagsTerraform"
34373402

3403+
cliConfigPath := testutil.CacheTFProviders(t, cacheRootDir, testName, templateFiles)
34383404
t.Logf("Set TF_CLI_CONFIG_FILE=%s", cliConfigPath)
34393405
return cliConfigPath
34403406
}

‎provisioner/terraform/provision_test.go‎

Lines changed: 2 additions & 168 deletions
Original file line numberDiff line numberDiff line change
@@ -3,17 +3,13 @@
33
package terraform_test
44

55
import (
6-
"bytes"
76
"context"
8-
"crypto/sha256"
9-
"encoding/hex"
107
"encoding/json"
118
"errors"
129
"fmt"
1310
"net"
1411
"net/http"
1512
"os"
16-
"os/exec"
1713
"path/filepath"
1814
"sort"
1915
"strings"
@@ -94,168 +90,6 @@ func configure(ctx context.Context, t *testing.T, client proto.DRPCProvisionerCl
9490
return sess
9591
}
9692

97-
func hashTemplateFilesAndTestName(t *testing.T, testName string, templateFiles map[string]string) string {
98-
t.Helper()
99-
100-
sortedFileNames := make([]string, 0, len(templateFiles))
101-
for fileName := range templateFiles {
102-
sortedFileNames = append(sortedFileNames, fileName)
103-
}
104-
sort.Strings(sortedFileNames)
105-
106-
// Inserting a delimiter between the file name and the file content
107-
// ensures that a file named `ab` with content `cd`
108-
// will not hash to the same value as a file named `abc` with content `d`.
109-
// This can still happen if the file name or content include the delimiter,
110-
// but hopefully they won't.
111-
delimiter := []byte("🎉 🌱 🌷")
112-
113-
hasher := sha256.New()
114-
for _, fileName := range sortedFileNames {
115-
file := templateFiles[fileName]
116-
_, err := hasher.Write([]byte(fileName))
117-
require.NoError(t, err)
118-
_, err = hasher.Write(delimiter)
119-
require.NoError(t, err)
120-
_, err = hasher.Write([]byte(file))
121-
require.NoError(t, err)
122-
}
123-
_, err := hasher.Write(delimiter)
124-
require.NoError(t, err)
125-
_, err = hasher.Write([]byte(testName))
126-
require.NoError(t, err)
127-
128-
return hex.EncodeToString(hasher.Sum(nil))
129-
}
130-
131-
const (
132-
terraformConfigFileName = "terraform.rc"
133-
cacheProvidersDirName = "providers"
134-
cacheTemplateFilesDirName = "files"
135-
)
136-
137-
// Writes a Terraform CLI config file (`terraform.rc`) in `dir` to enforce using the local provider mirror.
138-
// This blocks network access for providers, forcing Terraform to use only what's cached in `dir`.
139-
// Returns the path to the generated config file.
140-
func writeCliConfig(t *testing.T, dir string) string {
141-
t.Helper()
142-
143-
cliConfigPath := filepath.Join(dir, terraformConfigFileName)
144-
require.NoError(t, os.MkdirAll(filepath.Dir(cliConfigPath), 0o700))
145-
146-
content := fmt.Sprintf(`
147-
provider_installation {
148-
filesystem_mirror {
149-
path = "%s"
150-
include = ["*/*"]
151-
}
152-
direct {
153-
exclude = ["*/*"]
154-
}
155-
}
156-
`, filepath.Join(dir, cacheProvidersDirName))
157-
require.NoError(t, os.WriteFile(cliConfigPath, []byte(content), 0o600))
158-
return cliConfigPath
159-
}
160-
161-
func runCmd(t *testing.T, dir string, args ...string) {
162-
t.Helper()
163-
164-
stdout, stderr := bytes.NewBuffer(nil), bytes.NewBuffer(nil)
165-
cmd := exec.Command(args[0], args[1:]...) //#nosec
166-
cmd.Dir = dir
167-
cmd.Stdout = stdout
168-
cmd.Stderr = stderr
169-
if err := cmd.Run(); err != nil {
170-
t.Fatalf("failed to run %s: %s\nstdout: %s\nstderr: %s", strings.Join(args, " "), err, stdout.String(), stderr.String())
171-
}
172-
}
173-
174-
// Each test gets a unique cache dir based on its name and template files.
175-
// This ensures that tests can download providers in parallel and that they
176-
// will redownload providers if the template files change.
177-
func getTestCacheDir(t *testing.T, rootDir string, testName string, templateFiles map[string]string) string {
178-
t.Helper()
179-
180-
hash := hashTemplateFilesAndTestName(t, testName, templateFiles)
181-
dir := filepath.Join(rootDir, hash[:12])
182-
return dir
183-
}
184-
185-
// Ensures Terraform providers are downloaded and cached locally in a unique directory for the test.
186-
// Uses `terraform init` then `mirror` to populate the cache if needed.
187-
// Returns the cache directory path.
188-
func downloadProviders(t *testing.T, rootDir string, testName string, templateFiles map[string]string) string {
189-
t.Helper()
190-
191-
dir := getTestCacheDir(t, rootDir, testName, templateFiles)
192-
if _, err := os.Stat(dir); err == nil {
193-
t.Logf("%s: using cached terraform providers", testName)
194-
return dir
195-
}
196-
filesDir := filepath.Join(dir, cacheTemplateFilesDirName)
197-
defer func() {
198-
// The files dir will contain a copy of terraform providers generated
199-
// by the terraform init command. We don't want to persist them since
200-
// we already have a registry mirror in the providers dir.
201-
if err := os.RemoveAll(filesDir); err != nil {
202-
t.Logf("failed to remove files dir %s: %s", filesDir, err)
203-
}
204-
if !t.Failed() {
205-
return
206-
}
207-
// If `downloadProviders` function failed, clean up the cache dir.
208-
// We don't want to leave it around because it may be incomplete or corrupted.
209-
if err := os.RemoveAll(dir); err != nil {
210-
t.Logf("failed to remove dir %s: %s", dir, err)
211-
}
212-
}()
213-
214-
require.NoError(t, os.MkdirAll(filesDir, 0o700))
215-
216-
for fileName, file := range templateFiles {
217-
filePath := filepath.Join(filesDir, fileName)
218-
require.NoError(t, os.MkdirAll(filepath.Dir(filePath), 0o700))
219-
require.NoError(t, os.WriteFile(filePath, []byte(file), 0o600))
220-
}
221-
222-
providersDir := filepath.Join(dir, cacheProvidersDirName)
223-
require.NoError(t, os.MkdirAll(providersDir, 0o700))
224-
225-
// We need to run init because if a test uses modules in its template,
226-
// the mirror command will fail without it.
227-
runCmd(t, filesDir, "terraform", "init")
228-
// Now, mirror the providers into `providersDir`. We use this explicit mirror
229-
// instead of relying only on the standard Terraform plugin cache.
230-
//
231-
// Why? Because this mirror, when used with the CLI config from `writeCliConfig`,
232-
// prevents Terraform from hitting the network registry during `plan`. This cuts
233-
// down on network calls, making CI tests less flaky.
234-
//
235-
// In contrast, the standard cache *still* contacts the registry for metadata
236-
// during `init`, even if the plugins are already cached locally - see link below.
237-
//
238-
// Ref: https://developer.hashicorp.com/terraform/cli/config/config-file#provider-plugin-cache
239-
// > When a plugin cache directory is enabled, the terraform init command will
240-
// > still use the configured or implied installation methods to obtain metadata
241-
// > about which plugins are available
242-
runCmd(t, filesDir, "terraform", "providers", "mirror", providersDir)
243-
244-
return dir
245-
}
246-
247-
// Caches providers locally and generates a Terraform CLI config to use *only* that cache.
248-
// This setup prevents network access for providers during `terraform init`, improving reliability
249-
// in subsequent test runs.
250-
// Returns the path to the generated CLI config file.
251-
func cacheProviders(t *testing.T, rootDir string, testName string, templateFiles map[string]string) string {
252-
t.Helper()
253-
254-
providersParentDir := downloadProviders(t, rootDir, testName, templateFiles)
255-
cliConfigPath := writeCliConfig(t, providersParentDir)
256-
return cliConfigPath
257-
}
258-
25993
func readProvisionLog(t *testing.T, response proto.DRPCProvisioner_SessionClient) string {
26094
var logBuf strings.Builder
26195
for {
@@ -1177,7 +1011,7 @@ func TestProvision(t *testing.T) {
11771011
cacheRootDir := filepath.Join(testutil.PersistentCacheDir(t), "terraform_provision_test")
11781012
expectedCacheDirs := make(map[string]bool)
11791013
for _, testCase := range testCases {
1180-
cacheDir := getTestCacheDir(t, cacheRootDir, testCase.Name, testCase.Files)
1014+
cacheDir := testutil.GetTestTFCacheDir(t, cacheRootDir, testCase.Name, testCase.Files)
11811015
expectedCacheDirs[cacheDir] = true
11821016
}
11831017
currentCacheDirs, err := filepath.Glob(filepath.Join(cacheRootDir, "*"))
@@ -1199,7 +1033,7 @@ func TestProvision(t *testing.T) {
11991033

12001034
cliConfigPath := ""
12011035
if !testCase.SkipCacheProviders {
1202-
cliConfigPath = cacheProviders(
1036+
cliConfigPath = testutil.CacheTFProviders(
12031037
t,
12041038
cacheRootDir,
12051039
testCase.Name,

0 commit comments

Comments
 (0)