Skip to content

Commit 158243d

Browse files
pawbanapawbana
authored
fix: add cache for terraform installer files (#20776)
Replaces not working mocks by simple proxy that caches terraform files using test cache https://github.com/coder/coder/blob/16b8e6072fd84f45404e3f84bb2b6fea2424b090/testutil/cache.go#L13 Fixes: coder/internal#1126
1 parent eb64473 commit 158243d

File tree

3 files changed

+77
-154
lines changed

3 files changed

+77
-154
lines changed

‎provisioner/terraform/install.go‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ var (
3434
// operation.
3535
//
3636
//nolint:revive // verbose is a control flag that controls the verbosity of the log output.
37-
func Install(ctx context.Context, log slog.Logger, verbose bool, dir string, wantVersion *version.Version, baseUrl string, verifyChecksums bool) (string, error) {
37+
func Install(ctx context.Context, log slog.Logger, verbose bool, dir string, wantVersion *version.Version, baseUrl string) (string, error) {
3838
err := os.MkdirAll(dir, 0o750)
3939
if err != nil {
4040
return "", err
@@ -63,10 +63,9 @@ func Install(ctx context.Context, log slog.Logger, verbose bool, dir string, wan
6363
}
6464

6565
installer := &releases.ExactVersion{
66-
InstallDir: dir,
67-
Product: product.Terraform,
68-
Version: TerraformVersion,
69-
SkipChecksumVerification: !verifyChecksums,
66+
InstallDir: dir,
67+
Product: product.Terraform,
68+
Version: TerraformVersion,
7069
}
7170
installer.SetLogger(slog.Stdlib(ctx, log, slog.LevelDebug))
7271
if baseUrl != "" {

‎provisioner/terraform/install_test.go‎

Lines changed: 72 additions & 148 deletions
Original file line numberDiff line numberDiff line change
@@ -6,12 +6,12 @@
66
package terraform_test
77

88
import (
9-
"archive/zip"
109
"context"
11-
"encoding/json"
12-
"fmt"
10+
"errors"
11+
"io"
1312
"net"
1413
"net/http"
14+
"net/url"
1515
"os"
1616
"path/filepath"
1717
"strings"
@@ -28,172 +28,93 @@ import (
2828
)
2929

3030
const (
31-
// simple script that mocks `./terraform version -json`
32-
terraformExecutableTemplate = `#!/bin/bash
33-
cat <<EOF
34-
{
35-
"terraform_version": "${ver}",
36-
"platform": "linux_amd64",
37-
"provider_selections": {},
38-
"terraform_outdated": true
39-
}
40-
EOF
41-
`
31+
cacheSubDir = "terraform_install_test"
32+
terraformURL = "https://releases.hashicorp.com"
4233
)
4334

4435
var (
4536
version1 = terraform.TerraformVersion
4637
version2 = version.Must(version.NewVersion("1.2.0"))
4738
)
4839

49-
type productBuild struct {
50-
Name string `json:"name"`
51-
Version string `json:"version"`
52-
OS string `json:"os"`
53-
Arch string `json:"arch"`
54-
Filename string `json:"filename"`
55-
URL string `json:"url"`
56-
}
57-
58-
type productVersion struct {
59-
Name string `json:"name"`
60-
Version *version.Version `json:"version"`
61-
Builds []productBuild `json:"builds"`
62-
}
63-
64-
type product struct {
65-
Name string `json:"name"`
66-
Versions map[string]productVersion `json:"versions"`
40+
type terraformProxy struct {
41+
t *testing.T
42+
cacheRoot string
43+
listener net.Listener
44+
srv *http.Server
45+
fsHandler http.Handler
46+
httpClient *http.Client
47+
mutex *sync.Mutex
6748
}
6849

69-
func zipFilename(v *version.Version) string {
70-
return fmt.Sprintf("terraform_%s_linux_amd64.zip", v)
71-
}
72-
73-
// returns `/${version}/index.json` in struct format
74-
func versionedJSON(v *version.Version) productVersion {
75-
return productVersion{
76-
Name: "terraform",
77-
Version: v,
78-
Builds: []productBuild{
79-
{
80-
Arch: "amd64",
81-
Filename: zipFilename(v),
82-
Name: "terraform",
83-
OS: "linux",
84-
URL: fmt.Sprintf("/terraform/%s/%s", v, zipFilename(v)),
85-
Version: v.String(),
86-
},
87-
},
50+
// Simple cached proxy for terraform files.
51+
// Serves files from persistent cache or forwards requests to releases.hashicorp.com
52+
// Modifies downloaded index.json files so they point to proxy.
53+
func persistentlyCachedProxy(t *testing.T) *terraformProxy {
54+
cacheRoot := filepath.Join(testutil.PersistentCacheDir(t), cacheSubDir)
55+
proxy := terraformProxy{
56+
t: t,
57+
mutex: &sync.Mutex{},
58+
cacheRoot: cacheRoot,
59+
fsHandler: http.FileServer(http.Dir(cacheRoot)),
60+
httpClient: &http.Client{},
8861
}
89-
}
9062

91-
// returns `/index.json` in struct format
92-
func mainJSON(versions ...*version.Version) product {
93-
vj := map[string]productVersion{}
94-
for _, v := range versions {
95-
vj[v.String()] = versionedJSON(v)
96-
}
97-
mj := product{
98-
Name: "terraform",
99-
Versions: vj,
63+
listener, err := net.Listen("tcp", "127.0.0.1:0")
64+
if err != nil {
65+
t.Fatalf("failed to create listener")
10066
}
101-
return mj
102-
}
67+
proxy.listener = listener
10368

104-
func exeContent(v *version.Version) []byte {
105-
return []byte(strings.ReplaceAll(terraformExecutableTemplate, "${ver}", v.String()))
106-
}
69+
m := http.NewServeMux()
70+
m.HandleFunc("GET /", proxy.handleGet)
10771

108-
func mustMarshal(t *testing.T, obj any) []byte {
109-
b, err := json.Marshal(obj)
110-
require.NoError(t, err)
111-
return b
72+
proxy.srv = &http.Server{
73+
WriteTimeout: 30 * time.Second,
74+
ReadTimeout: 30 * time.Second,
75+
Handler: m,
76+
}
77+
return &proxy
11278
}
11379

114-
// Mock files are based on https://releases.hashicorp.com/terraform
115-
// mock directory structure:
116-
//
117-
// ${tmpDir}/index.json
118-
// ${tmpDir}/${version}/index.json
119-
// ${tmpDir}/${version}/terraform_${version}_linux_amd64.zip
120-
// -> zip contains 'terraform' binary and sometimes 'LICENSE.txt'
121-
func createFakeTerraformInstallationFiles(t *testing.T) string {
122-
tmpDir := t.TempDir()
123-
124-
mij := mustMarshal(t, mainJSON(version1, version2))
125-
jv1 := mustMarshal(t, versionedJSON(version1))
126-
jv2 := mustMarshal(t, versionedJSON(version2))
127-
128-
// `index.json`
129-
require.NoError(t, os.WriteFile(filepath.Join(tmpDir, "index.json"), mij, 0o400))
130-
131-
// `${version1}/index.json`
132-
require.NoError(t, os.Mkdir(filepath.Join(tmpDir, version1.String()), 0o700))
133-
require.NoError(t, os.WriteFile(filepath.Join(tmpDir, version1.String(), "index.json"), jv1, 0o400))
134-
135-
// `${version2}/index.json`
136-
require.NoError(t, os.Mkdir(filepath.Join(tmpDir, version2.String()), 0o700))
137-
require.NoError(t, os.WriteFile(filepath.Join(tmpDir, version2.String(), "index.json"), jv2, 0o400))
138-
139-
// `${version1}/linux_amd64.zip`
140-
zip1, err := os.Create(filepath.Join(tmpDir, version1.String(), zipFilename(version1)))
141-
require.NoError(t, err)
142-
zip1Writer := zip.NewWriter(zip1)
80+
func uriToFilename(u url.URL) string {
81+
return strings.ReplaceAll(u.RequestURI(), "/", "_")
82+
}
14383

144-
// `${version1}/linux_amd64.zip/terraform`
145-
exe1, err := zip1Writer.Create("terraform")
146-
require.NoError(t, err)
147-
n, err := exe1.Write(exeContent(version1))
148-
require.NoError(t, err)
149-
require.NotZero(t, n)
84+
func (p *terraformProxy) handleGet(w http.ResponseWriter, r *http.Request) {
85+
p.mutex.Lock()
86+
defer p.mutex.Unlock()
15087

151-
// `${version1}/linux_amd64.zip/LICENSE.txt`
152-
lic1, err := zip1Writer.Create("LICENSE.txt")
153-
require.NoError(t, err)
154-
n, err = lic1.Write([]byte("some license"))
155-
require.NoError(t, err)
156-
require.NotZero(t, n)
157-
require.NoError(t, zip1Writer.Close())
88+
filename := uriToFilename(*r.URL)
89+
path := filepath.Join(p.cacheRoot, filename)
90+
if _, err := os.Stat(path); errors.Is(err, os.ErrNotExist) {
91+
require.NoError(p.t, os.MkdirAll(p.cacheRoot, os.ModeDir|0o700))
15892

159-
// `${version2}/linux_amd64.zip`
160-
zip2, err := os.Create(filepath.Join(tmpDir, version2.String(), zipFilename(version2)))
161-
require.NoError(t, err)
162-
zip2Writer := zip.NewWriter(zip2)
93+
// Update cache
94+
req, err := http.NewRequestWithContext(p.t.Context(), "GET", terraformURL+r.URL.Path, nil)
95+
require.NoError(p.t, err)
16396

164-
// `${version1}/linux_amd64.zip/terraform`
165-
exe2, err := zip2Writer.Create("terraform")
166-
require.NoError(t, err)
167-
n, err = exe2.Write(exeContent(version2))
168-
require.NoError(t, err)
169-
require.NotZero(t, n)
170-
require.NoError(t, zip2Writer.Close())
97+
resp, err := p.httpClient.Do(req)
98+
require.NoError(p.t, err)
99+
defer resp.Body.Close()
171100

172-
return tmpDir
173-
}
101+
body, err := io.ReadAll(resp.Body)
102+
require.NoError(p.t, err)
174103

175-
// starts http server serving fake terraform installation files
176-
func startFakeTerraformServer(t *testing.T, tmpDir string) string {
177-
listener, err := net.Listen("tcp", "127.0.0.1:0")
178-
if err != nil {
179-
t.Fatalf("failed to create listener")
104+
// update index.json so urls in it point to proxy by making them relative
105+
// "https://releases.hashicorp.com/terraform/1.13.4/terraform_1.13.4_windows_amd64.zip" -> "/terraform/1.13.4/terraform_1.13.4_windows_amd64.zip"
106+
if strings.HasSuffix(r.URL.Path, "index.json") {
107+
body = []byte(strings.ReplaceAll(string(body), terraformURL, ""))
108+
}
109+
require.NoError(p.t, os.WriteFile(path, body, 0o400))
110+
} else if err != nil {
111+
p.t.Errorf("unexpected error when trying to read file from cache: %v", err)
180112
}
181113

182-
mux := http.NewServeMux()
183-
fs := http.FileServer(http.Dir(tmpDir))
184-
mux.Handle("/terraform/", http.StripPrefix("/terraform", fs))
185-
186-
srv := http.Server{
187-
ReadHeaderTimeout: time.Second,
188-
Handler: mux,
189-
}
190-
go srv.Serve(listener)
191-
t.Cleanup(func() {
192-
if err := srv.Close(); err != nil {
193-
t.Errorf("failed to close server: %v", err)
194-
}
195-
})
196-
return "http://" + listener.Addr().String()
114+
// Serve from cache
115+
r.URL.Path = filename
116+
r.URL.RawPath = filename
117+
p.fsHandler.ServeHTTP(w, r)
197118
}
198119

199120
func TestInstall(t *testing.T) {
@@ -205,8 +126,11 @@ func TestInstall(t *testing.T) {
205126
dir := t.TempDir()
206127
log := testutil.Logger(t)
207128

208-
tmpDir := createFakeTerraformInstallationFiles(t)
209-
addr := startFakeTerraformServer(t, tmpDir)
129+
proxy := persistentlyCachedProxy(t)
130+
go proxy.srv.Serve(proxy.listener)
131+
t.Cleanup(func() {
132+
require.NoError(t, proxy.srv.Close())
133+
})
210134

211135
// Install spins off 8 installs with Version and waits for them all
212136
// to complete. The locking mechanism within Install should
@@ -219,7 +143,7 @@ func TestInstall(t *testing.T) {
219143
wg.Add(1)
220144
go func() {
221145
defer wg.Done()
222-
p, err := terraform.Install(ctx, log, false, dir, version, addr, false)
146+
p, err := terraform.Install(ctx, log, false, dir, version, "http://"+proxy.listener.Addr().String())
223147
assert.NoError(t, err)
224148
paths <- p
225149
}()

‎provisioner/terraform/serve.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -103,7 +103,7 @@ func Serve(ctx context.Context, options *ServeOptions) error {
103103
slog.F("min_version", minTerraformVersion.String()))
104104
}
105105

106-
binPath, err := Install(ctx, options.Logger, options.ExternalProvisioner, options.CachePath, TerraformVersion, "", true)
106+
binPath, err := Install(ctx, options.Logger, options.ExternalProvisioner, options.CachePath, TerraformVersion, "")
107107
if err != nil {
108108
return xerrors.Errorf("install terraform: %w", err)
109109
}

0 commit comments

Comments
 (0)