diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0b5dfab0..e7155b68 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -11,8 +11,8 @@ permissions: packages: write jobs: - artifact-builder: - name: "Prepare release artifacts" + artifact-builder-x86: + name: "Prepare release artifacts (x86)" if: github.ref_type == 'tag' runs-on: ubuntu-latest container: @@ -55,6 +55,49 @@ jobs: uses: actions/attest-build-provenance@v2 with: subject-path: '~/maddy-x86_64-linux-musl.tar.zst' + artifact-builder-arm: + name: "Prepare release artifacts (aarch64)" + if: github.ref_type == 'tag' + runs-on: ubuntu-22.04-arm + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + # Building in a Docker container is a workaround for the issue of + # JavaScript-based GitHub Actions not being supported in Alpine + # containers on the Arm64 platform. Otherwise, we could completely reuse + # artifact-builder-x86 as a matrix job by running it on an Arm runner. + - name: Build in Docker container + run: | + # Create Dockerfile for the build + cat > Dockerfile << 'EOF' + FROM alpine:edge + RUN apk add --no-cache gcc go zstd musl-dev scdoc + WORKDIR /build + COPY . . + RUN ./build.sh --builddir /package-output/ --static build && \ + ver=$(cat .version) && \ + if [ "v$ver" != "${{github.ref_name}}" ]; then echo ".version does not match the Git tag"; exit 1; fi && \ + mv /package-output/ /maddy-$ver-aarch64-linux-musl && \ + cd / && \ + tar c ./maddy-$ver-aarch64-linux-musl | zstd > /maddy-aarch64-linux-musl.tar.zst + EOF + # Build the image, create a temporary container and copy the artifact. + docker build -t maddy-builder . + container_id=$(docker create maddy-builder) + docker cp $container_id:/maddy-aarch64-linux-musl.tar.zst . + docker rm $container_id + - name: Upload binary tree + uses: actions/upload-artifact@v4 + with: + name: maddy-binary-aarch64.tar.zst + path: maddy-aarch64-linux-musl.tar.zst + if-no-files-found: error + - name: "Generate artifact attestation" + uses: actions/attest-build-provenance@v2 + with: + subject-path: 'maddy-aarch64-linux-musl.tar.zst' docker-builder: name: "Build & push Docker image" if: github.ref_type == 'tag' diff --git a/.gitignore b/.gitignore index 5bf94b90..790b848a 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,8 @@ _testmain.go *.exe~ *.test *.prof +**/.envrc +**/.DS_Store # Tests coverage *.out diff --git a/.version b/.version index 6f4eebdf..ac39a106 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -0.8.1 +0.9.0 diff --git a/dist/vim/syntax/maddy-conf.vim b/dist/vim/syntax/maddy-conf.vim index d59e7990..9e56cdd8 100644 --- a/dist/vim/syntax/maddy-conf.vim +++ b/dist/vim/syntax/maddy-conf.vim @@ -183,6 +183,7 @@ syn keyword maddyModDir \ quarantine_threshold \ read_timeout \ reject_threshold + \ reject_action \ relaxed_requiretls \ required_fields \ require_sender_match @@ -198,6 +199,7 @@ syn keyword maddyModDir \ sig_expiry \ sign_fields \ sign_subdomains + \ soft_reject_action \ softfail_action \ SOME_action \ source diff --git a/docs/index.md b/docs/index.md index 901f7230..a9b7ee68 100644 --- a/docs/index.md +++ b/docs/index.md @@ -13,10 +13,11 @@ daemon with uniform configuration and minimal maintenance cost. feature-packed implementation you may want to use Dovecot instead. maddy still can handle message delivery business. -[![builds.sr.ht status](https://builds.sr.ht/~emersion/maddy.svg)](https://builds.sr.ht/~emersion/maddy?) -[![License text](https://img.shields.io/github/license/foxcpp/maddy)](https://github.com/foxcpp/maddy/blob/master/LICENSE) -[![Issues tracker](https://img.shields.io/github/issues/foxcpp/maddy)](https://github.com/foxcpp/maddy) +[![CI status](https://img.shields.io/github/actions/workflow/status/foxcpp/maddy/cicd.yml?style=flat-square)](https://github.com/foxcpp/maddy/actions/workflows/cicd.yml) +[![Issues tracker](https://img.shields.io/github/issues/foxcpp/maddy?style=flat-square)](https://github.com/foxcpp/maddy) * [Setup tutorial](https://maddy.email/tutorials/setting-up/) +* [Documentation](https://maddy.email/) + * [IRC channel](https://webchat.oftc.net/?channels=maddy&uio=MT11bmRlZmluZWQb1) * [Mailing list](https://lists.sr.ht/~foxcpp/maddy) diff --git a/docs/reference/checks/dnsbl.md b/docs/reference/checks/dnsbl.md index a2d27362..d7bb74cf 100644 --- a/docs/reference/checks/dnsbl.md +++ b/docs/reference/checks/dnsbl.md @@ -29,6 +29,30 @@ check.dnsbl { mailfrom yes score 1 } + + # Example with per-response-code scoring (new in 0.8) + zen.spamhaus.org { + client_ipv4 yes + client_ipv6 yes + + # SBL - Spamhaus Block List (known spam sources) + response 127.0.0.2 127.0.0.3 { + score 10 + message "Listed in Spamhaus SBL. See https://check.spamhaus.org/" + } + + # XBL - Exploits Block List (compromised hosts) + response 127.0.0.4 127.0.0.5 127.0.0.6 127.0.0.7 { + score 10 + message "Listed in Spamhaus XBL. See https://check.spamhaus.org/" + } + + # PBL - Policy Block List (dynamic IPs) + response 127.0.0.10 127.0.0.11 { + score 5 + message "Listed in Spamhaus PBL. See https://check.spamhaus.org/" + } + } } ``` @@ -171,3 +195,64 @@ will be rejected. It is possible to specify a negative value to make list act like a whitelist and override results of other blocklists. + +**Note:** When using `response` blocks (see below), the score from matching response +rules is used instead of this flat score value. + +--- + +### response _ip..._ + +Defines per-response-code rules for scoring and custom messages. This is useful +for combined DNSBLs like Spamhaus ZEN that return different codes for different +listing types. + +This works for both IP-based lookups (client_ipv4, client_ipv6) and domain-based +lookups (ehlo, mailfrom). + +Each `response` block takes one or more IP addresses or CIDR ranges as arguments +and contains the following directives: + +#### score _integer_ +**Required** + +Score to add when this response code is returned. If multiple response codes +are returned by the DNSBL, and they match different rules, the scores from +all matched rules are summed together. Each rule is counted only once, even +if multiple returned IPs match networks within that rule. + +#### message _string_ +**Optional** + +Custom rejection or quarantine message to include when this response code +matches. This message is shown to the client or logged when the threshold +is reached. + +**Example:** + +``` +zen.spamhaus.org { + client_ipv4 yes + + # High severity - known spam sources + response 127.0.0.2 127.0.0.3 { + score 10 + message "Listed in Spamhaus SBL" + } + + # Lower severity - dynamic IPs + response 127.0.0.10 127.0.0.11 { + score 5 + message "Listed in Spamhaus PBL" + } +} +``` + +**Scoring behavior:** +- If DNSBL returns `127.0.0.2` only → Score: 10 (matches first rule) +- If DNSBL returns `127.0.0.11` only → Score: 5 (matches second rule) +- If DNSBL returns both `127.0.0.2` and `127.0.0.11` → Score: 15 (both rules match, scores sum) +- If DNSBL returns both `127.0.0.2` and `127.0.0.3` → Score: 10 (same rule matches, counted once) + +**Backwards compatibility:** When `response` blocks are not used, the legacy +`responses` and `score` directives work as before. diff --git a/docs/reference/checks/rspamd.md b/docs/reference/checks/rspamd.md index 90063ae9..f37f5dcc 100644 --- a/docs/reference/checks/rspamd.md +++ b/docs/reference/checks/rspamd.md @@ -14,6 +14,8 @@ check.rspamd { error_resp_action ignore add_header_action quarantine rewrite_subj_action quarantine + reject_action reject + soft_reject_action reject flags pass_all } @@ -90,6 +92,20 @@ X-Spam-Flag and X-Spam-Score are added to the header irregardless of value. --- +### reject_action _action_ +Default: `reject` + +Action to take when rspamd requests to "reject". + +--- + +### soft_reject_action _action_ +Default: `reject` + +Action to take when rspamd requests to "soft reject". + +--- + ### flags _string-list..._ Default: `pass_all` diff --git a/docs/reference/endpoints/smtp.md b/docs/reference/endpoints/smtp.md index 58d3d9b1..f5fb7b4a 100644 --- a/docs/reference/endpoints/smtp.md +++ b/docs/reference/endpoints/smtp.md @@ -26,7 +26,7 @@ smtp tcp://0.0.0.0:25 { endpoint concurrency 500 } - # Example pipeline ocnfiguration. + # Example pipeline configuration. destination example.org { deliver_to &local_mailboxes } diff --git a/docs/tutorials/setting-up.md b/docs/tutorials/setting-up.md index 2ec83519..240caa80 100644 --- a/docs/tutorials/setting-up.md +++ b/docs/tutorials/setting-up.md @@ -105,7 +105,7 @@ $(local_domains) = $(primary_domain) example.com other.example.com ## TLS certificates -One thing that can't be automagically configured is TLS certs. If you already +One thing that can't be automatically configured is TLS certs. If you already have them somewhere - use them, open /etc/maddy/maddy.conf and put the right paths in. You need to make sure maddy can read them while running as unprivileged user (maddy never runs as root, even during start-up), one way to diff --git a/framework/dns/debugflags.go b/framework/dns/debugflags.go index fde218be..937edc97 100644 --- a/framework/dns/debugflags.go +++ b/framework/dns/debugflags.go @@ -32,5 +32,12 @@ func init() { Usage: "replace the DNS resolver address", Value: "system-default", Destination: &overrideServ, + Action: func(context *cli.Context, s string) error { + if s != "" && s != "system-default" { + override(s) + } + overrideServ = s + return nil + }, }) } diff --git a/framework/dns/override.go b/framework/dns/override.go index 25f0da08..0f073afd 100644 --- a/framework/dns/override.go +++ b/framework/dns/override.go @@ -32,7 +32,7 @@ var overrideServ string // // The server argument is in form of "IP:PORT". It is expected that the server // will be available both using TCP and UDP on the same port. -func override(server string) { +func override(server string) { // nolint: unused // used in debugflags.go net.DefaultResolver.PreferGo = true net.DefaultResolver.Dial = func(ctx context.Context, network, _ string) (net.Conn, error) { dialer := net.Dialer{ diff --git a/framework/dns/resolver.go b/framework/dns/resolver.go index f1393fe6..41687311 100644 --- a/framework/dns/resolver.go +++ b/framework/dns/resolver.go @@ -53,9 +53,5 @@ func LookupAddr(ctx context.Context, r Resolver, ip net.IP) (string, error) { } func DefaultResolver() Resolver { - if overrideServ != "" && overrideServ != "system-default" { - override(overrideServ) - } - return net.DefaultResolver } diff --git a/framework/log/orderedjson.go b/framework/log/orderedjson.go index 9de32baf..387253c5 100644 --- a/framework/log/orderedjson.go +++ b/framework/log/orderedjson.go @@ -31,6 +31,11 @@ import ( // human-readable when values from multiple messages are lined up to each // other. +type module interface { + Name() string + InstanceName() string +} + func marshalOrderedJSON(output *strings.Builder, m map[string]interface{}) error { order := make([]string, 0, len(m)) for k := range m { @@ -62,6 +67,8 @@ func marshalOrderedJSON(output *strings.Builder, m map[string]interface{}) error val = casted.FormatLog() case fmt.Stringer: val = casted.String() + case module: + val = casted.Name() + "/" + casted.InstanceName() case error: val = casted.Error() } diff --git a/framework/module/lifetime.go b/framework/module/lifetime.go index 5338b01f..2ccb08d4 100644 --- a/framework/module/lifetime.go +++ b/framework/module/lifetime.go @@ -37,18 +37,27 @@ type ReloadModule interface { Reload() error } +// EarlyStopModule is a LifetimeModule that needs to do some bookkeeping +// before new server instance starts during reload. +type EarlyStopModule interface { + LifetimeModule + EarlyStop() error +} + type LifetimeTracker struct { logger *log.Logger instances []*struct { - mod LifetimeModule - started bool + mod LifetimeModule + started bool + earlyStopped bool } } func (lt *LifetimeTracker) Add(mod LifetimeModule) { lt.instances = append(lt.instances, &struct { - mod LifetimeModule - started bool + mod LifetimeModule + started bool + earlyStopped bool }{mod: mod, started: false}) } @@ -59,8 +68,13 @@ func (lt *LifetimeTracker) StartAll() error { continue } + lt.logger.DebugMsg("starting module", + "mod_name", entry.mod.Name(), "inst_name", entry.mod.InstanceName()) + if err := entry.mod.Start(); err != nil { - lt.StopAll() + if err := lt.StopAll(); err != nil { + lt.logger.Error("StopAll failed after Start fail", err) + } return fmt.Errorf("failed to start module %v: %w", entry.mod.InstanceName(), err) } @@ -94,6 +108,32 @@ func (lt *LifetimeTracker) ReloadAll() error { return nil } +func (lt *LifetimeTracker) EarlyStopAll() error { + for i := len(lt.instances) - 1; i >= 0; i-- { + entry := lt.instances[i] + + if !entry.started { + continue + } + + rsm, ok := entry.mod.(EarlyStopModule) + if !ok { + continue + } + + if err := rsm.EarlyStop(); err != nil { + lt.logger.Error("module early stop failed", err, + "mod_name", entry.mod.Name(), "inst_name", entry.mod.InstanceName()) + continue + } + lt.logger.DebugMsg("module early stopped", + "mod_name", entry.mod.Name(), "inst_name", entry.mod.InstanceName()) + + entry.earlyStopped = true + } + return nil +} + // StopAll calls Stop for all registered LifetimeModule instances. func (lt *LifetimeTracker) StopAll() error { for i := len(lt.instances) - 1; i >= 0; i-- { diff --git a/framework/resource/netresource/listen.go b/framework/resource/netresource/listen.go index 6d164fd7..23fea4a5 100644 --- a/framework/resource/netresource/listen.go +++ b/framework/resource/netresource/listen.go @@ -16,8 +16,8 @@ func CloseUnusedListeners() error { return tracker.CloseUnused() } -func CloseAllListeners() { - tracker.Close() +func CloseAllListeners() error { + return tracker.Close() } func ResetListenersUsage() { diff --git a/framework/resource/netresource/tracker.go b/framework/resource/netresource/tracker.go index a1827aa2..2099f7a7 100644 --- a/framework/resource/netresource/tracker.go +++ b/framework/resource/netresource/tracker.go @@ -67,14 +67,22 @@ func (lt *ListenerTracker) ResetUsage() { } func (lt *ListenerTracker) CloseUnused() error { - lt.tcp.CloseUnused(func(key string) bool { return true }) - lt.unix.CloseUnused(func(key string) bool { return true }) + if err := lt.tcp.CloseUnused(func(key string) bool { return true }); err != nil { + lt.logger.Error("CloseUnused for TCP failed", err) + } + if err := lt.unix.CloseUnused(func(key string) bool { return true }); err != nil { + lt.logger.Error("CloseUnused for Unix failed", err) + } return nil } func (lt *ListenerTracker) Close() error { - lt.tcp.Close() - lt.unix.Close() + if err := lt.tcp.Close(); err != nil { + lt.logger.Error("Close for TCP failed", err) + } + if err := lt.unix.Close(); err != nil { + lt.logger.Error("Close for Unix failed", err) + } return nil } diff --git a/go.mod b/go.mod index bfa5673c..242d0e6d 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,8 @@ module github.com/foxcpp/maddy -go 1.23 +go 1.23.1 + +toolchain go1.23.5 require ( blitiri.com.ar/go/spf v1.5.1 @@ -15,7 +17,7 @@ require ( github.com/emersion/go-msgauth v0.6.8 github.com/emersion/go-sasl v0.0.0-20241020182733-b788ff22d5a6 github.com/emersion/go-smtp v0.21.3 - github.com/foxcpp/go-dovecot-sasl v0.0.0-20200522223722-c4699d7a24bf + github.com/foxcpp/go-dovecot-sasl v0.0.0-20260303144336-f7632c6ec0ba github.com/foxcpp/go-imap-backend-tests v0.0.0-20220105184719-e80aa29a5e16 github.com/foxcpp/go-imap-i18nlevel v0.0.0-20200208001533-d6ec88553005 github.com/foxcpp/go-imap-mess v0.0.0-20230108134257-b7ec3a649613 @@ -34,6 +36,7 @@ require ( github.com/libdns/cloudflare v0.1.1 github.com/libdns/digitalocean v0.0.0-20230728223659-4f9064657aea github.com/libdns/gandi v1.0.3 + github.com/libdns/gcore v0.0.0-20250127070537-4a9d185c9d20 github.com/libdns/googleclouddns v1.1.0 github.com/libdns/hetzner v0.0.1 github.com/libdns/leaseweb v0.4.0 @@ -49,6 +52,7 @@ require ( github.com/minio/minio-go/v7 v7.0.84 github.com/netauth/netauth v0.6.2 github.com/prometheus/client_golang v1.20.5 + github.com/stretchr/testify v1.10.0 github.com/urfave/cli/v2 v2.27.5 go.uber.org/zap v1.27.0 golang.org/x/crypto v0.32.0 @@ -64,6 +68,7 @@ require ( cloud.google.com/go/compute/metadata v0.6.0 // indirect filippo.io/edwards25519 v1.1.0 // indirect github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect + github.com/G-Core/gcore-dns-sdk-go v0.2.9 // indirect github.com/aws/aws-sdk-go v1.44.40 // indirect github.com/aws/aws-sdk-go-v2 v1.33.0 // indirect github.com/aws/aws-sdk-go-v2/config v1.29.1 // indirect @@ -83,6 +88,7 @@ require ( github.com/caddyserver/zerossl v0.1.3 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.6 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/digitalocean/godo v1.134.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/fatih/color v1.18.0 // indirect @@ -118,6 +124,7 @@ require ( github.com/pelletier/go-toml/v2 v2.2.3 // indirect github.com/pierrec/lz4 v2.6.1+incompatible // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.62.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect diff --git a/go.sum b/go.sum index 2f99728a..81c986e0 100644 --- a/go.sum +++ b/go.sum @@ -190,6 +190,8 @@ github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzS github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/G-Core/gcore-dns-sdk-go v0.2.9 h1:LMMZIRX8y3aJJuAviNSpFmLbovZUw+6Om+8VElp1F90= +github.com/G-Core/gcore-dns-sdk-go v0.2.9/go.mod h1:35t795gOfzfVanhzkFyUXEzaBuMXwETmJldPpP28MN4= github.com/GehirnInc/crypt v0.0.0-20230320061759-8cc1b52080c5 h1:IEjq88XO4PuBDcvmjQJcQGg+w+UaafSy8G5Kcb5tBhI= github.com/GehirnInc/crypt v0.0.0-20230320061759-8cc1b52080c5/go.mod h1:exZ0C/1emQJAw5tHOaUDyY1ycttqBAPcxuzf7QbY6ec= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= @@ -304,6 +306,8 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/foxcpp/go-dovecot-sasl v0.0.0-20200522223722-c4699d7a24bf h1:rmBPY5fryjp9zLQYsUmQqqgsYq7qeVfrjtr96Tf9vD8= github.com/foxcpp/go-dovecot-sasl v0.0.0-20200522223722-c4699d7a24bf/go.mod h1:5yZUmwr851vgjyAfN7OEfnrmKOh/qLA5dbGelXYsu1E= +github.com/foxcpp/go-dovecot-sasl v0.0.0-20260303144336-f7632c6ec0ba h1:yxQhqX9RQCvECZKBtqwCZoKy/6CLaozDZeWH9Lvndy0= +github.com/foxcpp/go-dovecot-sasl v0.0.0-20260303144336-f7632c6ec0ba/go.mod h1:5yZUmwr851vgjyAfN7OEfnrmKOh/qLA5dbGelXYsu1E= github.com/foxcpp/go-imap v1.0.0-beta.1.0.20220623182312-df940c324887 h1:qUoaaHyrRpQw85ru6VQcC6JowdhrWl7lSbI1zRX1FTM= github.com/foxcpp/go-imap v1.0.0-beta.1.0.20220623182312-df940c324887/go.mod h1:Qlx1FSx2FTxjnjWpIlVNEuX+ylerZQNFE5NsmKFSejY= github.com/foxcpp/go-imap-backend-tests v0.0.0-20220105184719-e80aa29a5e16 h1:qheFPDpteiUy7Ym18R68OYenpk85UyKYGkhYTmddSBg= @@ -521,6 +525,8 @@ github.com/libdns/cloudflare v0.1.1 h1:FVPfWwP8zZCqj268LZjmkDleXlHPlFU9KC4OJ3yn0 github.com/libdns/cloudflare v0.1.1/go.mod h1:9VK91idpOjg6v7/WbjkEW49bSCxj00ALesIFDhJ8PBU= github.com/libdns/digitalocean v0.0.0-20230728223659-4f9064657aea h1:IGlMNZCUp8Ho7NYYorpP5ZJgg2mFXARs6eHs/pSqFkA= github.com/libdns/digitalocean v0.0.0-20230728223659-4f9064657aea/go.mod h1:B2TChhOTxvBflpRTHlguXWtwa1Ha5WI6JkB6aCViM+0= +github.com/libdns/gcore v0.0.0-20250127070537-4a9d185c9d20 h1:bQwFw+C9sX/zYZlV53ey0KnNkxrfWYIFpvptuAVhJ1Y= +github.com/libdns/gcore v0.0.0-20250127070537-4a9d185c9d20/go.mod h1:JGoT1mbmqQwtYQqN5F/vGc9j4TTTMKw/hDm5vXADHUI= github.com/libdns/googleclouddns v1.1.0 h1:murPR1LfTZZObLV2OLxUVmymWH25glkMFKpDjkk2m0E= github.com/libdns/googleclouddns v1.1.0/go.mod h1:3tzd056dfqKlf71V8Oy19En4WjJ3ybyuWx6P9bQSCIw= github.com/libdns/hetzner v0.0.1 h1:WsmcsOKnfpKmzwhfyqhGQEIlEeEaEUvb7ezoJgBKaqU= diff --git a/internal/auth/sasl.go b/internal/auth/sasl.go index 00d25ffa..b29959b0 100644 --- a/internal/auth/sasl.go +++ b/internal/auth/sasl.go @@ -54,6 +54,8 @@ type SASLAuth struct { AuthMap module.Table AuthNormalize authz.NormalizeFunc + ErrorMap func(err error) error + Plain []module.PlainAuth } @@ -62,7 +64,7 @@ func (s *SASLAuth) SASLMechanisms() []string { if len(s.Plain) != 0 { mechs = append(mechs, sasl.Plain) - if s.OnlyFirstID { + if s.EnableLogin { mechs = append(mechs, sasl.Login) } } @@ -105,12 +107,16 @@ func (s *SASLAuth) AuthPlain(username, password string) error { var lastErr error for _, p := range s.Plain { - username, err := s.usernameForAuth(context.TODO(), username) + mappedUsername, err := s.usernameForAuth(context.TODO(), username) if err != nil { return err } - lastErr = p.AuthPlain(username, password) + s.Log.DebugMsg("attempting authentication", + "mapped_username", mappedUsername, "original_username", username, + "module", p) + + lastErr = p.AuthPlain(mappedUsername, password) if lastErr == nil { return nil } @@ -128,7 +134,10 @@ type ContextData struct { } // CreateSASL creates the sasl.Server instance for the corresponding mechanism. -func (s *SASLAuth) CreateSASL(mech string, remoteAddr net.Addr, successCb func(identity string, data ContextData) error) sasl.Server { +func (s *SASLAuth) CreateSASL( + mech string, remoteAddr net.Addr, + successCb func(identity string, data ContextData) error, +) sasl.Server { switch mech { case sasl.Plain: return sasl.NewPlainServer(func(identity, username, password string) error { @@ -136,12 +145,18 @@ func (s *SASLAuth) CreateSASL(mech string, remoteAddr net.Addr, successCb func(i identity = username } if identity != username { + if s.ErrorMap != nil { + return s.ErrorMap(ErrInvalidAuthCred) + } return ErrInvalidAuthCred } err := s.AuthPlain(username, password) if err != nil { s.Log.Error("authentication failed", err, "username", username, "src_ip", remoteAddr) + if s.ErrorMap != nil { + return s.ErrorMap(ErrInvalidAuthCred) + } return ErrInvalidAuthCred } @@ -156,9 +171,20 @@ func (s *SASLAuth) CreateSASL(mech string, remoteAddr net.Addr, successCb func(i } return sasllogin.NewLoginServer(func(username, password string) error { - err := s.AuthPlain(username, password) + username, err := s.usernameForAuth(context.Background(), username) + if err != nil { + if s.ErrorMap != nil { + return s.ErrorMap(ErrInvalidAuthCred) + } + return err + } + + err = s.AuthPlain(username, password) if err != nil { s.Log.Error("authentication failed", err, "username", username, "src_ip", remoteAddr) + if s.ErrorMap != nil { + return s.ErrorMap(ErrInvalidAuthCred) + } return ErrInvalidAuthCred } diff --git a/internal/check/dnsbl/common.go b/internal/check/dnsbl/common.go index 7b874cb9..ae43b23a 100644 --- a/internal/check/dnsbl/common.go +++ b/internal/check/dnsbl/common.go @@ -32,9 +32,15 @@ type ListedErr struct { Identity string List string Reason string + Score int + Message string } func (le ListedErr) Fields() map[string]interface{} { + msg := "Client identity listed in the used DNSBL" + if le.Message != "" { + msg = le.Message + } return map[string]interface{}{ "check": "dnsbl", "list": le.List, @@ -42,7 +48,7 @@ func (le ListedErr) Fields() map[string]interface{} { "reason": le.Reason, "smtp_code": 554, "smtp_enchcode": exterrors.EnhancedCode{5, 7, 0}, - "smtp_msg": "Client identity listed in the used DNSBL", + "smtp_msg": msg, } } @@ -66,28 +72,85 @@ func checkDomain(ctx context.Context, resolver dns.Resolver, cfg List, domain st return nil } - // Attempt to extract explanation string. - txts, err := resolver.LookupTXT(context.Background(), query) - if err != nil || len(txts) == 0 { - // Not significant, include addresses as reason. Usually they are - // mapped to some predefined 'reasons' by BL. - return ListedErr{ - Identity: domain, - List: cfg.Zone, - Reason: strings.Join(addrs, "; "), + var score int + var customMessage string + var filteredAddrs []string + + // If ResponseRules is configured, use new behavior + if len(cfg.ResponseRules) > 0 { + // Convert string addresses to IPAddr for matching + ipAddrs := make([]net.IPAddr, 0, len(addrs)) + for _, addr := range addrs { + if ip := net.ParseIP(addr); ip != nil { + ipAddrs = append(ipAddrs, net.IPAddr{IP: ip}) + } } + + matchedScore, matchedMessages, matchedReasons, matched := matchResponseRules(ipAddrs, cfg.ResponseRules) + if !matched { + return nil + } + score = matchedScore + + // Use first matched message if available + if len(matchedMessages) > 0 { + customMessage = matchedMessages[0] + } + + filteredAddrs = matchedReasons + } else { + // Legacy behavior: accept all addresses + filteredAddrs = addrs } - // Some BLs provide multiple reasons (meta-BLs such as Spamhaus Zen) so - // don't mangle them by joining with "", instead join with "; ". + // Attempt to extract explanation string from TXT records (shared by both paths) + txts, err := resolver.LookupTXT(ctx, query) + var reason string + if err == nil && len(txts) > 0 { + reason = strings.Join(txts, "; ") + } else { + // Not significant, include addresses as reason. Usually they are + // mapped to some predefined 'reasons' by BL. + reason = strings.Join(filteredAddrs, "; ") + } return ListedErr{ Identity: domain, List: cfg.Zone, - Reason: strings.Join(txts, "; "), + Reason: reason, + Score: score, + Message: customMessage, } } +func matchResponseRules(addrs []net.IPAddr, rules []ResponseRule) (score int, messages []string, reasons []string, matched bool) { + // Track which rules have been matched to avoid counting the same rule multiple times + matchedRules := make(map[int]bool) + + for _, addr := range addrs { + for ruleIdx, rule := range rules { + // Skip if this rule has already been matched + if matchedRules[ruleIdx] { + continue + } + + for _, respNet := range rule.Networks { + if respNet.Contains(addr.IP) { + score += rule.Score + if rule.Message != "" { + messages = append(messages, rule.Message) + } + reasons = append(reasons, addr.IP.String()) + matchedRules[ruleIdx] = true + matched = true + break // Move to next rule + } + } + } + } + return +} + func checkIP(ctx context.Context, resolver dns.Resolver, cfg List, ip net.IP) error { ipv6 := true if ipv4 := ip.To4(); ipv4 != nil { @@ -113,52 +176,72 @@ func checkIP(ctx context.Context, resolver dns.Resolver, cfg List, ip net.IP) er return err } - filteredAddrs := make([]net.IPAddr, 0, len(addrs)) -addrsLoop: - for _, addr := range addrs { - // No responses whitelist configured - permit all. - if len(cfg.Responses) == 0 { - filteredAddrs = append(filteredAddrs, addr) - continue + var filteredAddrs []net.IPAddr + var score int + var customMessage string + + // If ResponseRules is configured, use new behavior + if len(cfg.ResponseRules) > 0 { + matchedScore, matchedMessages, matchedReasons, matched := matchResponseRules(addrs, cfg.ResponseRules) + if !matched { + return nil } + score = matchedScore - for _, respNet := range cfg.Responses { - if respNet.Contains(addr.IP) { + // Use first matched message if available + if len(matchedMessages) > 0 { + customMessage = matchedMessages[0] + } + + // Build filteredAddrs from matched reasons for TXT lookup fallback + for _, reason := range matchedReasons { + filteredAddrs = append(filteredAddrs, net.IPAddr{IP: net.ParseIP(reason)}) + } + } else { + // Legacy behavior: use flat Responses filter + filteredAddrs = make([]net.IPAddr, 0, len(addrs)) + addrsLoop: + for _, addr := range addrs { + // No responses whitelist configured - permit all. + if len(cfg.Responses) == 0 { filteredAddrs = append(filteredAddrs, addr) - continue addrsLoop + continue + } + + for _, respNet := range cfg.Responses { + if respNet.Contains(addr.IP) { + filteredAddrs = append(filteredAddrs, addr) + continue addrsLoop + } } } - } - if len(filteredAddrs) == 0 { - return nil + if len(filteredAddrs) == 0 { + return nil + } } - // Attempt to extract explanation string. + // Attempt to extract explanation string from TXT records (shared by both paths) txts, err := resolver.LookupTXT(ctx, query) - if err != nil || len(txts) == 0 { + var reason string + if err == nil && len(txts) > 0 { + reason = strings.Join(txts, "; ") + } else { // Not significant, include addresses as reason. Usually they are // mapped to some predefined 'reasons' by BL. - reasonParts := make([]string, 0, len(filteredAddrs)) for _, addr := range filteredAddrs { reasonParts = append(reasonParts, addr.IP.String()) } - - return ListedErr{ - Identity: ip.String(), - List: cfg.Zone, - Reason: strings.Join(reasonParts, "; "), - } + reason = strings.Join(reasonParts, "; ") } - // Some BLs provide multiple reasons (meta-BLs such as Spamhaus Zen) so - // don't mangle them by joining with "", instead join with "; ". - return ListedErr{ Identity: ip.String(), List: cfg.Zone, - Reason: strings.Join(txts, "; "), + Reason: reason, + Score: score, + Message: customMessage, } } diff --git a/internal/check/dnsbl/common_test.go b/internal/check/dnsbl/common_test.go index caa06575..8d2b24e3 100644 --- a/internal/check/dnsbl/common_test.go +++ b/internal/check/dnsbl/common_test.go @@ -236,3 +236,104 @@ func TestCheckIP(t *testing.T) { Reason: "127.0.0.1", }) } + +func TestCheckDomainWithResponseRules(t *testing.T) { + test := func(zones map[string]mockdns.Zone, cfg List, domain string, expectedErr error) { + t.Helper() + resolver := mockdns.Resolver{Zones: zones} + err := checkDomain(context.Background(), &resolver, cfg, domain) + if expectedErr == nil { + if err != nil { + t.Errorf("expected no error, got '%#v'", err) + } + } else { + if err == nil { + t.Errorf("expected err to be '%#v', got nil", expectedErr) + } else { + expectedLE, okExpected := expectedErr.(ListedErr) + actualLE, okActual := err.(ListedErr) + if !okExpected || !okActual { + t.Errorf("expected err to be '%#v', got '%#v'", expectedErr, err) + } else { + if expectedLE.Identity != actualLE.Identity || + expectedLE.List != actualLE.List || + expectedLE.Score != actualLE.Score || + expectedLE.Message != actualLE.Message { + t.Errorf("expected err to be '%#v', got '%#v'", expectedErr, err) + } + } + } + } + } + + // Test domain with single response code and custom message + test(map[string]mockdns.Zone{ + "spam.example.com.dnsbl.example.org.": { + A: []string{"127.0.0.2"}, + }, + }, List{ + Zone: "dnsbl.example.org", + ResponseRules: []ResponseRule{ + { + Networks: []net.IPNet{ + {IP: net.IPv4(127, 0, 0, 2), Mask: net.IPv4Mask(255, 255, 255, 255)}, + }, + Score: 10, + Message: "Domain listed as spam source", + }, + }, + }, "spam.example.com", ListedErr{ + Identity: "spam.example.com", + List: "dnsbl.example.org", + Score: 10, + Message: "Domain listed as spam source", + }) + + // Test domain with multiple response codes - scores should sum + test(map[string]mockdns.Zone{ + "multi.example.com.dnsbl.example.org.": { + A: []string{"127.0.0.2", "127.0.0.11"}, + }, + }, List{ + Zone: "dnsbl.example.org", + ResponseRules: []ResponseRule{ + { + Networks: []net.IPNet{ + {IP: net.IPv4(127, 0, 0, 2), Mask: net.IPv4Mask(255, 255, 255, 255)}, + }, + Score: 10, + Message: "High severity", + }, + { + Networks: []net.IPNet{ + {IP: net.IPv4(127, 0, 0, 11), Mask: net.IPv4Mask(255, 255, 255, 255)}, + }, + Score: 5, + Message: "Low severity", + }, + }, + }, "multi.example.com", ListedErr{ + Identity: "multi.example.com", + List: "dnsbl.example.org", + Score: 15, // 10 + 5 + Message: "High severity", + }) + + // Test domain with no matching response codes + test(map[string]mockdns.Zone{ + "unknown.example.com.dnsbl.example.org.": { + A: []string{"127.0.0.99"}, + }, + }, List{ + Zone: "dnsbl.example.org", + ResponseRules: []ResponseRule{ + { + Networks: []net.IPNet{ + {IP: net.IPv4(127, 0, 0, 2), Mask: net.IPv4Mask(255, 255, 255, 255)}, + }, + Score: 10, + Message: "Listed", + }, + }, + }, "unknown.example.com", nil) +} diff --git a/internal/check/dnsbl/dnsbl.go b/internal/check/dnsbl/dnsbl.go index 79738720..77b5e3ea 100644 --- a/internal/check/dnsbl/dnsbl.go +++ b/internal/check/dnsbl/dnsbl.go @@ -38,6 +38,12 @@ import ( "golang.org/x/sync/errgroup" ) +type ResponseRule struct { + Networks []net.IPNet + Score int + Message string +} + type List struct { Zone string @@ -49,6 +55,8 @@ type List struct { ScoreAdj int Responses []net.IPNet + + ResponseRules []ResponseRule } var defaultBL = List{ @@ -124,6 +132,14 @@ func (bl *DNSBL) readListCfg(node config.Node) error { cfg.Bool("mailfrom", false, defaultBL.EHLO, &listCfg.MAILFROM) cfg.Int("score", false, false, 1, &listCfg.ScoreAdj) cfg.StringList("responses", false, false, []string{"127.0.0.1/24"}, &responseNets) + cfg.Callback("response", func(_ *config.Map, node config.Node) error { + rule, err := parseResponseRule(node) + if err != nil { + return err + } + listCfg.ResponseRules = append(listCfg.ResponseRules, rule) + return nil + }) if _, err := cfg.Process(); err != nil { return err } @@ -142,6 +158,11 @@ func (bl *DNSBL) readListCfg(node config.Node) error { listCfg.Responses = append(listCfg.Responses, *ipNet) } + // Warn if both response and responses are configured + if len(listCfg.ResponseRules) > 0 && len(responseNets) > 0 { + bl.log.Msg("both 'response' blocks and 'responses' directive are specified, 'response' blocks take precedence", "list", node.Name) + } + for _, zone := range append([]string{node.Name}, node.Args...) { zoneCfg := listCfg zoneCfg.Zone = zone @@ -171,6 +192,44 @@ func (bl *DNSBL) readListCfg(node config.Node) error { return nil } +func parseResponseRule(node config.Node) (ResponseRule, error) { + var rule ResponseRule + + if len(node.Args) == 0 { + return rule, config.NodeErr(node, "response block requires at least one IP address or CIDR as argument") + } + + // Parse IP addresses/CIDRs from arguments + for _, arg := range node.Args { + // If there is no / - it is a plain IP address, append '/32' or '/128' + resp := arg + if !strings.Contains(resp, "/") { + // Check if it's IPv6 to determine the mask + if strings.Contains(resp, ":") { + resp += "/128" + } else { + resp += "/32" + } + } + + _, ipNet, err := net.ParseCIDR(resp) + if err != nil { + return rule, config.NodeErr(node, "invalid IP address or CIDR: %s: %v", arg, err) + } + rule.Networks = append(rule.Networks, *ipNet) + } + + // Parse directives within the response block + cfg := config.NewMap(nil, node) + cfg.Int("score", false, true, 0, &rule.Score) + cfg.String("message", false, false, "", &rule.Message) + if _, err := cfg.Process(); err != nil { + return rule, err + } + + return rule, nil +} + func (bl *DNSBL) testList(listCfg List) { // Check RFC 5782 Section 5 requirements. @@ -296,6 +355,7 @@ func (bl *DNSBL) checkLists(ctx context.Context, ip net.IP, ehlo, mailFrom strin score int listedOn []string reasons []string + messages []string ) for _, list := range bl.bls { @@ -311,7 +371,18 @@ func (bl *DNSBL) checkLists(ctx context.Context, ip net.IP, ehlo, mailFrom strin defer lck.Unlock() listedOn = append(listedOn, listErr.List) reasons = append(reasons, listErr.Reason) - score += list.ScoreAdj + + // Use score from ListedErr if set (new behavior), otherwise use legacy ScoreAdj + if listErr.Score != 0 { + score += listErr.Score + } else { + score += list.ScoreAdj + } + + // Collect custom messages if available + if listErr.Message != "" { + messages = append(messages, listErr.Message) + } } return nil }) @@ -332,13 +403,19 @@ func (bl *DNSBL) checkLists(ctx context.Context, ip net.IP, ehlo, mailFrom strin } } + // Use custom message if available, otherwise use default + message := "Client identity is listed in the used DNSBL" + if len(messages) > 0 { + message = strings.Join(messages, "; ") + } + if score >= bl.rejectThres { return module.CheckResult{ Reject: true, Reason: &exterrors.SMTPError{ Code: 554, EnhancedCode: exterrors.EnhancedCode{5, 7, 0}, - Message: "Client identity is listed in the used DNSBL", + Message: message, Err: err, CheckName: "dnsbl", }, @@ -350,7 +427,7 @@ func (bl *DNSBL) checkLists(ctx context.Context, ip net.IP, ehlo, mailFrom strin Reason: &exterrors.SMTPError{ Code: 554, EnhancedCode: exterrors.EnhancedCode{5, 7, 0}, - Message: "Client identity is listed in the used DNSBL", + Message: message, Err: err, CheckName: "dnsbl", }, diff --git a/internal/check/dnsbl/dnsbl_test.go b/internal/check/dnsbl/dnsbl_test.go index 1845aeb7..d4cd7238 100644 --- a/internal/check/dnsbl/dnsbl_test.go +++ b/internal/check/dnsbl/dnsbl_test.go @@ -211,3 +211,282 @@ func TestCheckLists(t *testing.T) { true, false, ) } + +func TestCheckIPWithResponseRules(t *testing.T) { + test := func(zones map[string]mockdns.Zone, cfg List, ip net.IP, expectedErr error) { + t.Helper() + resolver := mockdns.Resolver{Zones: zones} + err := checkIP(context.Background(), &resolver, cfg, ip) + if expectedErr == nil { + if err != nil { + t.Errorf("expected no error, got '%#v'", err) + } + } else { + if err == nil { + t.Errorf("expected err to be '%#v', got nil", expectedErr) + } else { + expectedLE, okExpected := expectedErr.(ListedErr) + actualLE, okActual := err.(ListedErr) + if !okExpected || !okActual { + t.Errorf("expected err to be '%#v', got '%#v'", expectedErr, err) + } else { + if expectedLE.Identity != actualLE.Identity || + expectedLE.List != actualLE.List || + expectedLE.Score != actualLE.Score || + expectedLE.Message != actualLE.Message { + t.Errorf("expected err to be '%#v', got '%#v'", expectedErr, err) + } + } + } + } + } + + // Test single response code with score and message + test(map[string]mockdns.Zone{ + "4.3.2.1.example.org.": { + A: []string{"127.0.0.2"}, + }, + }, List{ + Zone: "example.org", + ClientIPv4: true, + ResponseRules: []ResponseRule{ + { + Networks: []net.IPNet{ + {IP: net.IPv4(127, 0, 0, 2), Mask: net.IPv4Mask(255, 255, 255, 255)}, + }, + Score: 10, + Message: "Listed in SBL", + }, + }, + }, net.IPv4(1, 2, 3, 4), ListedErr{ + Identity: "1.2.3.4", + List: "example.org", + Score: 10, + Message: "Listed in SBL", + }) + + // Test multiple response codes with different scores - scores should sum + test(map[string]mockdns.Zone{ + "4.3.2.1.example.org.": { + A: []string{"127.0.0.2", "127.0.0.11"}, + }, + }, List{ + Zone: "example.org", + ClientIPv4: true, + ResponseRules: []ResponseRule{ + { + Networks: []net.IPNet{ + {IP: net.IPv4(127, 0, 0, 2), Mask: net.IPv4Mask(255, 255, 255, 255)}, + {IP: net.IPv4(127, 0, 0, 3), Mask: net.IPv4Mask(255, 255, 255, 255)}, + }, + Score: 10, + Message: "Listed in SBL", + }, + { + Networks: []net.IPNet{ + {IP: net.IPv4(127, 0, 0, 10), Mask: net.IPv4Mask(255, 255, 255, 255)}, + {IP: net.IPv4(127, 0, 0, 11), Mask: net.IPv4Mask(255, 255, 255, 255)}, + }, + Score: 5, + Message: "Listed in PBL", + }, + }, + }, net.IPv4(1, 2, 3, 4), ListedErr{ + Identity: "1.2.3.4", + List: "example.org", + Score: 15, // 10 + 5 + Message: "Listed in SBL", + }) + + // Test response code that doesn't match any rule - should return nil + test(map[string]mockdns.Zone{ + "4.3.2.1.example.org.": { + A: []string{"127.0.0.99"}, + }, + }, List{ + Zone: "example.org", + ClientIPv4: true, + ResponseRules: []ResponseRule{ + { + Networks: []net.IPNet{ + {IP: net.IPv4(127, 0, 0, 2), Mask: net.IPv4Mask(255, 255, 255, 255)}, + }, + Score: 10, + Message: "Listed in SBL", + }, + }, + }, net.IPv4(1, 2, 3, 4), nil) + + // Test low severity only - should get score 5 + test(map[string]mockdns.Zone{ + "4.3.2.1.example.org.": { + A: []string{"127.0.0.10"}, + }, + }, List{ + Zone: "example.org", + ClientIPv4: true, + ResponseRules: []ResponseRule{ + { + Networks: []net.IPNet{ + {IP: net.IPv4(127, 0, 0, 2), Mask: net.IPv4Mask(255, 255, 255, 255)}, + }, + Score: 10, + Message: "Listed in SBL", + }, + { + Networks: []net.IPNet{ + {IP: net.IPv4(127, 0, 0, 10), Mask: net.IPv4Mask(255, 255, 255, 255)}, + }, + Score: 5, + Message: "Listed in PBL", + }, + }, + }, net.IPv4(1, 2, 3, 4), ListedErr{ + Identity: "1.2.3.4", + List: "example.org", + Score: 5, + Message: "Listed in PBL", + }) + + // Test high severity - should get score 10 + test(map[string]mockdns.Zone{ + "4.3.2.1.example.org.": { + A: []string{"127.0.0.2"}, + }, + }, List{ + Zone: "example.org", + ClientIPv4: true, + ResponseRules: []ResponseRule{ + { + Networks: []net.IPNet{ + {IP: net.IPv4(127, 0, 0, 2), Mask: net.IPv4Mask(255, 255, 255, 255)}, + }, + Score: 10, + Message: "Listed in SBL", + }, + }, + }, net.IPv4(1, 2, 3, 4), ListedErr{ + Identity: "1.2.3.4", + List: "example.org", + Score: 10, + Message: "Listed in SBL", + }) +} + +func TestCheckListsWithResponseRules(t *testing.T) { + test := func(zones map[string]mockdns.Zone, bls []List, ip net.IP, ehlo, mailFrom string, reject, quarantine bool) { + mod := &DNSBL{ + bls: bls, + resolver: &mockdns.Resolver{Zones: zones}, + log: testutils.Logger(t, "dnsbl"), + quarantineThres: 5, + rejectThres: 10, + } + result := mod.checkLists(context.Background(), ip, ehlo, mailFrom) + + if result.Reject && !reject { + t.Errorf("Expected message to not be rejected") + } + if !result.Reject && reject { + t.Errorf("Expected message to be rejected") + } + if result.Quarantine && !quarantine { + t.Errorf("Expected message to not be quarantined") + } + if !result.Quarantine && quarantine { + t.Errorf("Expected message to be quarantined") + } + } + + // Test: Only low-severity code returned -> quarantine but not reject + test(map[string]mockdns.Zone{ + "4.3.2.1.zen.example.org.": { + A: []string{"127.0.0.11"}, + }, + }, []List{ + { + Zone: "zen.example.org", + ClientIPv4: true, + ResponseRules: []ResponseRule{ + { + Networks: []net.IPNet{ + {IP: net.IPv4(127, 0, 0, 2), Mask: net.IPv4Mask(255, 255, 255, 255)}, + }, + Score: 10, + Message: "Listed in SBL", + }, + { + Networks: []net.IPNet{ + {IP: net.IPv4(127, 0, 0, 10), Mask: net.IPv4Mask(255, 255, 255, 255)}, + {IP: net.IPv4(127, 0, 0, 11), Mask: net.IPv4Mask(255, 255, 255, 255)}, + }, + Score: 5, + Message: "Listed in PBL", + }, + }, + }, + }, net.IPv4(1, 2, 3, 4), "mx.example.com", "foo@example.com", false, true) + + // Test: High-severity code returned -> reject + test(map[string]mockdns.Zone{ + "4.3.2.1.zen.example.org.": { + A: []string{"127.0.0.2"}, + }, + }, []List{ + { + Zone: "zen.example.org", + ClientIPv4: true, + ResponseRules: []ResponseRule{ + { + Networks: []net.IPNet{ + {IP: net.IPv4(127, 0, 0, 2), Mask: net.IPv4Mask(255, 255, 255, 255)}, + }, + Score: 10, + Message: "Listed in SBL", + }, + }, + }, + }, net.IPv4(1, 2, 3, 4), "mx.example.com", "foo@example.com", true, false) + + // Test: Legacy configuration without response blocks -> existing behavior preserved + test(map[string]mockdns.Zone{ + "4.3.2.1.example.org.": { + A: []string{"127.0.0.1"}, + }, + }, []List{ + { + Zone: "example.org", + ClientIPv4: true, + ScoreAdj: 10, + }, + }, net.IPv4(1, 2, 3, 4), "mx.example.com", "foo@example.com", true, false) + + // Test: Mixed configuration (some lists with response blocks, some without) -> both work correctly + test(map[string]mockdns.Zone{ + "4.3.2.1.zen.example.org.": { + A: []string{"127.0.0.11"}, + }, + "4.3.2.1.legacy.example.org.": { + A: []string{"127.0.0.1"}, + }, + }, []List{ + { + Zone: "zen.example.org", + ClientIPv4: true, + ResponseRules: []ResponseRule{ + { + Networks: []net.IPNet{ + {IP: net.IPv4(127, 0, 0, 11), Mask: net.IPv4Mask(255, 255, 255, 255)}, + }, + Score: 5, + Message: "Listed in PBL", + }, + }, + }, + { + Zone: "legacy.example.org", + ClientIPv4: true, + ScoreAdj: 3, + }, + }, net.IPv4(1, 2, 3, 4), "mx.example.com", "foo@example.com", false, true) // 5 + 3 = 8, quarantine but not reject +} diff --git a/internal/check/rspamd/rspamd.go b/internal/check/rspamd/rspamd.go index 559c94b6..ede0c151 100644 --- a/internal/check/rspamd/rspamd.go +++ b/internal/check/rspamd/rspamd.go @@ -57,6 +57,8 @@ type Check struct { errorRespAction modconfig.FailAction addHdrAction modconfig.FailAction rewriteSubjAction modconfig.FailAction + rejectAction modconfig.FailAction + softRejectAction modconfig.FailAction client *http.Client } @@ -117,6 +119,15 @@ func (c *Check) Configure(inlineArgs []string, cfg *config.Map) error { func() (interface{}, error) { return modconfig.FailAction{Quarantine: true}, nil }, modconfig.FailActionDirective, &c.rewriteSubjAction) + cfg.Custom("reject_action", false, false, + func() (interface{}, error) { + return modconfig.FailAction{Reject: true}, nil + }, modconfig.FailActionDirective, &c.rejectAction) + cfg.Custom("soft_reject_action", false, false, + func() (interface{}, error) { + return modconfig.FailAction{Reject: true}, nil + }, modconfig.FailActionDirective, &c.softRejectAction) + cfg.StringList("flags", false, false, []string{"pass_all"}, &flags) if _, err := cfg.Process(); err != nil { return err @@ -320,7 +331,7 @@ func (s *state) CheckBody(ctx context.Context, hdr textproto.Header, body buffer Header: hdrAdd, }) case "soft reject": - return module.CheckResult{ + return s.c.softRejectAction.Apply(module.CheckResult{ Reject: true, Reason: &exterrors.SMTPError{ Code: 450, @@ -329,9 +340,9 @@ func (s *state) CheckBody(ctx context.Context, hdr textproto.Header, body buffer CheckName: modName, Misc: map[string]interface{}{"action": "soft reject"}, }, - } + }) case "reject": - return module.CheckResult{ + return s.c.rejectAction.Apply(module.CheckResult{ Reject: true, Reason: &exterrors.SMTPError{ Code: 550, @@ -340,7 +351,7 @@ func (s *state) CheckBody(ctx context.Context, hdr textproto.Header, body buffer CheckName: modName, Misc: map[string]interface{}{"action": "reject"}, }, - } + }) } s.log.Msg("unhandled action", "action", respData.Action) diff --git a/internal/cli/app.go b/internal/cli/app.go index fc4273d7..59108965 100644 --- a/internal/cli/app.go +++ b/internal/cli/app.go @@ -1,7 +1,6 @@ package maddycli import ( - "flag" "fmt" "os" "strings" @@ -30,10 +29,6 @@ databases used by it (all other subcommands). } app.ExitErrHandler = func(c *cli.Context, err error) { cli.HandleExitCoder(err) - if err != nil { - log.Println(err) - cli.OsExiter(1) - } } app.EnableBashCompletion = true app.Commands = []*cli.Command{ @@ -66,9 +61,6 @@ databases used by it (all other subcommands). func AddGlobalFlag(f cli.Flag) { app.Flags = append(app.Flags, f) - if err := f.Apply(flag.CommandLine); err != nil { - log.Println("GlobalFlag", f, "could not be mapped to stdlib flag:", err) - } } func AddSubcommand(cmd *cli.Command) { @@ -83,15 +75,27 @@ func AddSubcommand(cmd *cli.Command) { return cmd.Action(c) } app.Flags = append(app.Flags, cmd.Flags...) - for _, f := range cmd.Flags { - if err := f.Apply(flag.CommandLine); err != nil { - log.Println("GlobalFlag", f, "could not be mapped to stdlib flag:", err) - } - } } } +// RunWithoutExit is like Run but returns exit code instead of calling os.Exit +// To be used in maddy.cover. +func RunWithoutExit() int { + code := 0 + + cli.OsExiter = func(c int) { code = c } + defer func() { + cli.OsExiter = os.Exit + }() + + Run() + + return code +} + func Run() { + mapStdlibFlags(app) + // Actual entry point is registered in maddy.go. // Print help when called via maddyctl executable. To be removed diff --git a/internal/cli/ctl/imapacct.go b/internal/cli/ctl/imapacct.go index ee7d19c2..2541a228 100644 --- a/internal/cli/ctl/imapacct.go +++ b/internal/cli/ctl/imapacct.go @@ -81,6 +81,11 @@ creates a set of default folder (mailboxes) with special-use attribute set.`, EnvVars: []string{"MADDY_CFGBLOCK"}, Value: "local_mailboxes", }, + &cli.BoolFlag{ + Name: "no-specialuse", + Usage: "Do not create special-use folders", + Value: false, + }, &cli.StringFlag{ Name: "sent-name", Usage: "Name of special mailbox for sent messages, use empty string to not create any", @@ -235,6 +240,10 @@ func imapAcctCreate(be module.Storage, ctx *cli.Context) error { fmt.Fprintf(os.Stderr, "Note: Storage backend does not support SPECIAL-USE IMAP extension") } + if ctx.Bool("no-specialuse") { + return nil + } + createMbox := func(name, specialUseAttr string) error { if suu == nil { return act.CreateMailbox(name) diff --git a/internal/cli/ctl/moduleinit.go b/internal/cli/ctl/moduleinit.go index 1129a516..5519d0e3 100644 --- a/internal/cli/ctl/moduleinit.go +++ b/internal/cli/ctl/moduleinit.go @@ -21,20 +21,52 @@ package ctl import ( "errors" "fmt" - "io" "os" "github.com/foxcpp/maddy" "github.com/foxcpp/maddy/framework/container" + "github.com/foxcpp/maddy/framework/log" "github.com/foxcpp/maddy/framework/module" "github.com/foxcpp/maddy/internal/updatepipe" "github.com/urfave/cli/v2" ) -func closeIfNeeded(i interface{}) { - if c, ok := i.(io.Closer); ok { - c.Close() +func closeIfNeeded(i any) { + if c, ok := i.(module.LifetimeModule); ok { + if err := c.Stop(); err != nil { + log.DefaultLogger.Error("failed to stop module", err) + } + } +} + +type managedStorage struct { + module.ManageableStorage + started bool +} + +func (m *managedStorage) Close() error { + if !m.started { + return nil + } + if lm, ok := m.ManageableStorage.(module.LifetimeModule); ok { + return lm.Stop() + } + return nil +} + +type managedUserDB struct { + module.PlainUserDB + started bool +} + +func (m *managedUserDB) Close() error { + if !m.started { + return nil + } + if lm, ok := m.PlainUserDB.(module.LifetimeModule); ok { + return lm.Stop() } + return nil } func getCfgBlockModule(ctx *cli.Context) (*container.C, module.Module, error) { @@ -92,6 +124,14 @@ func openStorage(ctx *cli.Context) (module.Storage, error) { return nil, cli.Exit(fmt.Sprintf("Error: configuration block %s is not an IMAP storage", ctx.String("cfg-block")), 2) } + started := false + if lt, ok := storage.(module.LifetimeModule); ok { + if err := lt.Start(); err != nil { + return nil, err + } + started = true + } + if updStore, ok := mod.(updatepipe.Backend); ok { if err := updStore.EnableUpdatePipe(updatepipe.ModePush); err != nil && !errors.Is(err, os.ErrNotExist) { fmt.Fprintf(os.Stderr, "Failed to initialize update pipe, do not remove messages from mailboxes open by clients: %v\n", err) @@ -100,6 +140,11 @@ func openStorage(ctx *cli.Context) (module.Storage, error) { fmt.Fprintf(os.Stderr, "No update pipe support, do not remove messages from mailboxes open by clients\n") } + if started { + if ms, ok := storage.(module.ManageableStorage); ok { + return &managedStorage{ManageableStorage: ms, started: started}, nil + } + } return storage, nil } @@ -114,5 +159,16 @@ func openUserDB(ctx *cli.Context) (module.PlainUserDB, error) { return nil, cli.Exit(fmt.Sprintf("Error: configuration block %s is not a local credentials store", ctx.String("cfg-block")), 2) } + started := false + if lt, ok := userDB.(module.LifetimeModule); ok { + if err := lt.Start(); err != nil { + return nil, err + } + started = true + } + + if started { + return &managedUserDB{PlainUserDB: userDB, started: started}, nil + } return userDB, nil } diff --git a/internal/cli/extflag.go b/internal/cli/extflag.go new file mode 100644 index 00000000..8cfc27c3 --- /dev/null +++ b/internal/cli/extflag.go @@ -0,0 +1,60 @@ +package maddycli + +import ( + "flag" + + "github.com/urfave/cli/v2" +) + +// extFlag implements cli.Flag via standard flag.Flag. +type extFlag struct { + f *flag.Flag +} + +func (e *extFlag) Apply(fs *flag.FlagSet) error { + fs.Var(e.f.Value, e.f.Name, e.f.Usage) + return nil +} + +func (e *extFlag) Names() []string { + return []string{e.f.Name} +} + +func (e *extFlag) IsSet() bool { + return false +} + +func (e *extFlag) String() string { + return cli.FlagStringer(e) +} + +func (e *extFlag) IsVisible() bool { + return true +} + +func (e *extFlag) TakesValue() bool { + return false +} + +func (e *extFlag) GetUsage() string { + return e.f.Usage +} + +func (e *extFlag) GetValue() string { + return e.f.Value.String() +} + +func (e *extFlag) GetDefaultText() string { + return e.f.DefValue +} + +func (e *extFlag) GetEnvVars() []string { + return nil +} + +func mapStdlibFlags(app *cli.App) { + // Modified AllowExtFlags from cli lib with -test.* exception removed. + flag.VisitAll(func(f *flag.Flag) { + app.Flags = append(app.Flags, &extFlag{f}) + }) +} diff --git a/internal/endpoint/dovecot_sasld/dovecot_sasl.go b/internal/endpoint/dovecot_sasld/dovecot_sasl.go index f4dee1a7..74a452b2 100644 --- a/internal/endpoint/dovecot_sasld/dovecot_sasl.go +++ b/internal/endpoint/dovecot_sasld/dovecot_sasl.go @@ -80,6 +80,7 @@ func (endp *Endpoint) Configure(_ []string, cfg *config.Map) error { } endp.srv = dovecotsasl.NewServer() + endp.saslAuth.Log.Debug = endp.log.Debug endp.srv.Log = stdlog.New(&endp.log, "", 0) for _, mech := range endp.saslAuth.SASLMechanisms() { diff --git a/internal/endpoint/imap/imap.go b/internal/endpoint/imap/imap.go index 13116225..b73908d8 100644 --- a/internal/endpoint/imap/imap.go +++ b/internal/endpoint/imap/imap.go @@ -108,6 +108,8 @@ func (endp *Endpoint) Configure(_ []string, cfg *config.Map) error { return err } + endp.saslAuth.Log.Debug = endp.Log.Debug + addresses := make([]config.Endpoint, 0, len(endp.addrs)) for _, addr := range endp.addrs { saddr, err := config.ParseEndpoint(addr) @@ -165,7 +167,9 @@ func (endp *Endpoint) Start() error { } if err := endp.setupListeners(endp.endpoints); err != nil { - endp.Stop() + if err := endp.Stop(); err != nil { + endp.Log.Error("failed to stop after setupListeners error", err) + } return err } return nil diff --git a/internal/endpoint/openmetrics/om.go b/internal/endpoint/openmetrics/om.go index 44ac64db..e7e4a791 100644 --- a/internal/endpoint/openmetrics/om.go +++ b/internal/endpoint/openmetrics/om.go @@ -85,7 +85,9 @@ func (e *Endpoint) Start() error { for _, endp := range e.endpoints { l, err := netresource.Listen(endp.Network(), endp.Address()) if err != nil { - e.Stop() + if err := e.Stop(); err != nil { + e.logger.Error("failed to stop after failed listen", err) + } return fmt.Errorf("%s: %v", modName, err) } diff --git a/internal/endpoint/smtp/metrics.go b/internal/endpoint/smtp/metrics.go index 509241bc..8c9a7188 100644 --- a/internal/endpoint/smtp/metrics.go +++ b/internal/endpoint/smtp/metrics.go @@ -26,7 +26,7 @@ var ( Namespace: "maddy", Subsystem: "smtp", Name: "started_transactions", - Help: "Amount of SMTP trasanactions started", + Help: "Amount of SMTP transactions started", }, []string{"module"}, ) @@ -35,7 +35,7 @@ var ( Namespace: "maddy", Subsystem: "smtp", Name: "smtp_completed_transactions", - Help: "Amount of SMTP trasanactions successfully completed", + Help: "Amount of SMTP transactions successfully completed", }, []string{"module"}, ) @@ -44,7 +44,7 @@ var ( Namespace: "maddy", Subsystem: "smtp", Name: "aborted_transactions", - Help: "Amount of SMTP trasanactions aborted", + Help: "Amount of SMTP transactions aborted", }, []string{"module"}, ) diff --git a/internal/endpoint/smtp/session.go b/internal/endpoint/smtp/session.go index ed842f98..01e36e73 100644 --- a/internal/endpoint/smtp/session.go +++ b/internal/endpoint/smtp/session.go @@ -103,11 +103,6 @@ func (s *Session) AuthMechanisms() []string { } func (s *Session) Auth(mech string) (sasl.Server, error) { - // Executed before authentication and session initialization. - if err := s.endp.pipeline.RunEarlyChecks(s.sessionCtx, &s.connState); err != nil { - return nil, s.endp.wrapErr("", true, "AUTH", err) - } - return s.endp.saslAuth.CreateSASL(mech, s.connState.RemoteAddr, func(identity string, data auth.ContextData) error { s.connState.AuthUser = identity s.connState.AuthPassword = data.Password @@ -176,19 +171,7 @@ func (s *Session) AuthPlain(username, password string) error { failedLogins.WithLabelValues(s.endp.name).Inc() - if exterrors.IsTemporary(err) { - return &smtp.SMTPError{ - Code: 454, - EnhancedCode: smtp.EnhancedCode{4, 7, 0}, - Message: "Temporary authentication failure", - } - } - - return &smtp.SMTPError{ - Code: 535, - EnhancedCode: smtp.EnhancedCode{5, 7, 8}, - Message: "Invalid credentials", - } + return s.endp.authErrorMap(err) } s.connState.AuthUser = username diff --git a/internal/endpoint/smtp/smtp.go b/internal/endpoint/smtp/smtp.go index 8c1b9616..cabde482 100644 --- a/internal/endpoint/smtp/smtp.go +++ b/internal/endpoint/smtp/smtp.go @@ -38,6 +38,7 @@ import ( modconfig "github.com/foxcpp/maddy/framework/config/module" tls2 "github.com/foxcpp/maddy/framework/config/tls" "github.com/foxcpp/maddy/framework/dns" + "github.com/foxcpp/maddy/framework/exterrors" "github.com/foxcpp/maddy/framework/future" "github.com/foxcpp/maddy/framework/log" "github.com/foxcpp/maddy/framework/module" @@ -75,9 +76,6 @@ type Endpoint struct { sessionCnt atomic.Int32 shutdownTimeout time.Duration - authNormalize authz.NormalizeFunc - authMap module.Table - listenersWg sync.WaitGroup Log log.Logger @@ -287,6 +285,9 @@ func (endp *Endpoint) setConfig(cfg *config.Map) error { return err } + endp.saslAuth.Log.Debug = endp.Log.Debug + endp.saslAuth.ErrorMap = endp.authErrorMap + // INTERNATIONALIZATION: See RFC 6531 Section 3.3. endp.serv.Domain, err = idna.ToASCII(hostname) if err != nil { @@ -308,8 +309,6 @@ func (endp *Endpoint) setConfig(cfg *config.Map) error { return fmt.Errorf("%s: auth. provider must be set for submission endpoint", endp.name) } } - endp.saslAuth.AuthNormalize = endp.authNormalize - endp.saslAuth.AuthMap = endp.authMap if ioDebug { endp.serv.Debug = endp.Log.DebugWriter() @@ -321,12 +320,30 @@ func (endp *Endpoint) setConfig(cfg *config.Map) error { func (endp *Endpoint) Start() error { if err := endp.setupListeners(endp.endpoints); err != nil { - endp.Stop() + if err := endp.Stop(); err != nil { + endp.Log.Error("failed to Stop after setupListeners fail", err) + } return err } return nil } +func (endp *Endpoint) authErrorMap(err error) error { + if exterrors.IsTemporary(err) { + return &smtp.SMTPError{ + Code: 454, + EnhancedCode: smtp.EnhancedCode{4, 7, 0}, + Message: "Temporary authentication failure", + } + } + + return &smtp.SMTPError{ + Code: 535, + EnhancedCode: smtp.EnhancedCode{5, 7, 8}, + Message: "Invalid credentials", + } +} + func (endp *Endpoint) setupListeners(addresses []config.Endpoint) error { for _, addr := range addresses { var l net.Listener @@ -429,7 +446,9 @@ func (endp *Endpoint) Stop() error { ctx, cancel := context.WithTimeout(context.Background(), endp.shutdownTimeout) defer cancel() - endp.serv.Shutdown(ctx) + if err := endp.serv.Shutdown(ctx); err != nil { + return err + } endp.listenersWg.Wait() diff --git a/internal/endpoint/smtp/smtp_test.go b/internal/endpoint/smtp/smtp_test.go index d6de22dc..5fc79703 100644 --- a/internal/endpoint/smtp/smtp_test.go +++ b/internal/endpoint/smtp/smtp_test.go @@ -37,6 +37,7 @@ import ( "github.com/foxcpp/maddy/internal/auth" "github.com/foxcpp/maddy/internal/msgpipeline" "github.com/foxcpp/maddy/internal/testutils" + "github.com/stretchr/testify/assert" ) var testPort string @@ -146,13 +147,17 @@ func submitMsgOpts(t *testing.T, cl *smtp.Client, from string, rcpts []string, o func TestSMTPDelivery(t *testing.T) { tgt := testutils.Target{} endp := testEndpoint(t, "smtp", nil, &tgt, nil, nil) - defer endp.Stop() + defer func() { + assert.NoError(t, endp.Stop()) + }() cl, err := smtp.Dial("127.0.0.1:" + testPort) if err != nil { t.Fatal(err) } - defer cl.Close() + defer func() { + assert.NoError(t, cl.Close()) + }() err = submitMsg(t, cl, "sender@example.org", []string{"rcpt1@example.com", "rcpt2@example.com"}, testMsg) if err != nil { @@ -184,7 +189,9 @@ func TestSMTPDelivery(t *testing.T) { func TestSMTPDelivery_rDNSError(t *testing.T) { tgt := testutils.Target{} endp := testEndpoint(t, "smtp", nil, &tgt, nil, nil) - defer endp.Stop() + defer func() { + assert.NoError(t, endp.Stop()) + }() endp.resolver.(*mockdns.Resolver).Zones["1.0.0.127.in-addr.arpa."] = mockdns.Zone{ Err: &net.DNSError{ @@ -199,7 +206,9 @@ func TestSMTPDelivery_rDNSError(t *testing.T) { if err != nil { t.Fatal(err) } - defer cl.Close() + defer func() { + assert.NoError(t, cl.Close()) + }() err = submitMsg(t, cl, "sender@example.org", []string{"rcpt1@example.com", "rcpt2@example.com"}, testMsg) if err != nil { @@ -228,13 +237,17 @@ func TestSMTPDelivery_EarlyCheck_Fail(t *testing.T) { }, }, }, nil) - defer endp.Stop() + defer func() { + assert.NoError(t, endp.Stop()) + }() cl, err := smtp.Dial("127.0.0.1:" + testPort) if err != nil { t.Fatal(err) } - defer cl.Close() + defer func() { + assert.NoError(t, cl.Close()) + }() err = cl.Mail("sender@example.org", nil) if err == nil { @@ -268,13 +281,17 @@ func TestSMTPDeliver_CheckError(t *testing.T) { }, }, nil) endp.deferServerReject = false - defer endp.Stop() + defer func() { + assert.NoError(t, endp.Stop()) + }() cl, err := smtp.Dial("127.0.0.1:" + testPort) if err != nil { t.Fatal(err) } - defer cl.Close() + defer func() { + assert.NoError(t, cl.Close()) + }() err = cl.Mail("sender@example.org", nil) if err == nil { @@ -307,13 +324,17 @@ func TestSMTPDeliver_CheckError_Deferred(t *testing.T) { }, }, nil) endp.deferServerReject = true - defer endp.Stop() + defer func() { + assert.NoError(t, endp.Stop()) + }() cl, err := smtp.Dial("127.0.0.1:" + testPort) if err != nil { t.Fatal(err) } - defer cl.Close() + defer func() { + assert.NoError(t, cl.Close()) + }() err = cl.Mail("sender@example.org", nil) if err != nil { @@ -346,13 +367,17 @@ func TestSMTPDeliver_CheckError_Deferred(t *testing.T) { func TestSMTPDelivery_Multi(t *testing.T) { tgt := testutils.Target{} endp := testEndpoint(t, "smtp", nil, &tgt, nil, nil) - defer endp.Stop() + defer func() { + assert.NoError(t, endp.Stop()) + }() cl, err := smtp.Dial("127.0.0.1:" + testPort) if err != nil { t.Fatal(err) } - defer cl.Close() + defer func() { + assert.NoError(t, cl.Close()) + }() err = submitMsg(t, cl, "sender1@example.org", []string{"rcpt1@example.com", "rcpt2@example.com"}, testMsg) if err != nil { @@ -384,13 +409,17 @@ func TestSMTPDelivery_Multi(t *testing.T) { func TestSMTPDelivery_AbortData(t *testing.T) { tgt := testutils.Target{} endp := testEndpoint(t, "smtp", nil, &tgt, nil, nil) - defer endp.Stop() + defer func() { + assert.NoError(t, endp.Stop()) + }() cl, err := smtp.Dial("127.0.0.1:" + testPort) if err != nil { t.Fatal(err) } - defer cl.Close() + defer func() { + _ = cl.Close() + }() if err := cl.Hello("mx.example.org"); err != nil { t.Fatal(err) @@ -410,7 +439,7 @@ func TestSMTPDelivery_AbortData(t *testing.T) { } // Then.. Suddenly, close the connection without sending the final dot. - cl.Close() + assert.NoError(t, cl.Close()) time.Sleep(250 * time.Millisecond) @@ -422,13 +451,17 @@ func TestSMTPDelivery_AbortData(t *testing.T) { func TestSMTPDelivery_EmptyMessage(t *testing.T) { tgt := testutils.Target{} endp := testEndpoint(t, "smtp", nil, &tgt, nil, nil) - defer endp.Stop() + defer func() { + assert.NoError(t, endp.Stop()) + }() cl, err := smtp.Dial("127.0.0.1:" + testPort) if err != nil { t.Fatal(err) } - defer cl.Close() + defer func() { + assert.NoError(t, cl.Close()) + }() if err := cl.Hello("mx.example.org"); err != nil { t.Fatal(err) @@ -461,13 +494,17 @@ func TestSMTPDelivery_EmptyMessage(t *testing.T) { func TestSMTPDelivery_AbortLogout(t *testing.T) { tgt := testutils.Target{} endp := testEndpoint(t, "smtp", nil, &tgt, nil, nil) - defer endp.Stop() + defer func() { + assert.NoError(t, endp.Stop()) + }() cl, err := smtp.Dial("127.0.0.1:" + testPort) if err != nil { t.Fatal(err) } - defer cl.Close() + defer func() { + _ = cl.Close() + }() if err := cl.Hello("mx.example.org"); err != nil { t.Fatal(err) @@ -480,7 +517,7 @@ func TestSMTPDelivery_AbortLogout(t *testing.T) { } // Then.. Suddenly, close the connection. - cl.Close() + assert.NoError(t, cl.Close()) time.Sleep(250 * time.Millisecond) @@ -492,13 +529,17 @@ func TestSMTPDelivery_AbortLogout(t *testing.T) { func TestSMTPDelivery_Reset(t *testing.T) { tgt := testutils.Target{} endp := testEndpoint(t, "smtp", nil, &tgt, nil, nil) - defer endp.Stop() + defer func() { + assert.NoError(t, endp.Stop()) + }() cl, err := smtp.Dial("127.0.0.1:" + testPort) if err != nil { t.Fatal(err) } - defer cl.Close() + defer func() { + assert.NoError(t, cl.Close()) + }() if err := cl.Mail("from-garbage@example.org", nil); err != nil { t.Fatal(err) @@ -527,13 +568,17 @@ func TestSMTPDelivery_Reset(t *testing.T) { func TestSMTPDelivery_SubmissionAuthRequire(t *testing.T) { tgt := testutils.Target{} endp := testEndpoint(t, "submission", &module.Dummy{}, &tgt, nil, nil) - defer endp.Stop() + defer func() { + assert.NoError(t, endp.Stop()) + }() cl, err := smtp.Dial("127.0.0.1:" + testPort) if err != nil { t.Fatal(err) } - defer cl.Close() + defer func() { + assert.NoError(t, cl.Close()) + }() if err := cl.Mail("from-garbage@example.org", nil); err == nil { t.Fatal("Expected an error, got none") @@ -543,13 +588,17 @@ func TestSMTPDelivery_SubmissionAuthRequire(t *testing.T) { func TestSMTPDelivery_SubmissionAuthOK(t *testing.T) { tgt := testutils.Target{} endp := testEndpoint(t, "submission", &module.Dummy{}, &tgt, nil, nil) - defer endp.Stop() + defer func() { + assert.NoError(t, endp.Stop()) + }() cl, err := smtp.Dial("127.0.0.1:" + testPort) if err != nil { t.Fatal(err) } - defer cl.Close() + defer func() { + assert.NoError(t, cl.Close()) + }() if err := cl.Auth(sasl.NewPlainClient("", "user", "password")); err != nil { t.Fatal(err) diff --git a/internal/endpoint/smtp/smtputf8_test.go b/internal/endpoint/smtp/smtputf8_test.go index 2f157aba..ff16be6c 100644 --- a/internal/endpoint/smtp/smtputf8_test.go +++ b/internal/endpoint/smtp/smtputf8_test.go @@ -27,6 +27,7 @@ import ( "github.com/foxcpp/maddy/framework/exterrors" "github.com/foxcpp/maddy/framework/module" "github.com/foxcpp/maddy/internal/testutils" + "github.com/stretchr/testify/assert" ) func TestSMTPUTF8_MangleStatusMessage(t *testing.T) { @@ -43,14 +44,18 @@ func TestSMTPUTF8_MangleStatusMessage(t *testing.T) { }, }, nil) endp.deferServerReject = false - defer endp.Stop() + defer func() { + assert.NoError(t, endp.Stop()) + }() defer testutils.WaitForConnsClose(t, endp.serv) cl, err := smtp.Dial("127.0.0.1:" + testPort) if err != nil { t.Fatal(err) } - defer cl.Close() + defer func() { + assert.NoError(t, cl.Close()) + }() err = cl.Mail("sender@example.org", nil) if err == nil { @@ -73,7 +78,9 @@ func TestSMTP_RejectNonASCIIFrom(t *testing.T) { tgt := testutils.Target{} endp := testEndpoint(t, "smtp", nil, &tgt, nil, nil) endp.deferServerReject = false - defer endp.Stop() + defer func() { + assert.NoError(t, endp.Stop()) + }() defer testutils.WaitForConnsClose(t, endp.serv) cl, err := smtp.Dial("127.0.0.1:" + testPort) @@ -100,14 +107,18 @@ func TestSMTPUTF8_NormalizeCaseFoldFrom(t *testing.T) { tgt := testutils.Target{} endp := testEndpoint(t, "smtp", nil, &tgt, nil, nil) endp.deferServerReject = false - defer endp.Stop() + defer func() { + assert.NoError(t, endp.Stop()) + }() defer testutils.WaitForConnsClose(t, endp.serv) cl, err := smtp.Dial("127.0.0.1:" + testPort) if err != nil { t.Fatal(err) } - defer cl.Close() + defer func() { + assert.NoError(t, cl.Close()) + }() err = submitMsgOpts(t, cl, "foo@E\u0301.example.org", []string{"rcpt@example.com"}, &smtp.MailOptions{ UTF8: true, @@ -127,14 +138,18 @@ func TestSMTP_RejectNonASCIIRcpt(t *testing.T) { tgt := testutils.Target{} endp := testEndpoint(t, "smtp", nil, &tgt, nil, nil) endp.deferServerReject = false - defer endp.Stop() + defer func() { + assert.NoError(t, endp.Stop()) + }() defer testutils.WaitForConnsClose(t, endp.serv) cl, err := smtp.Dial("127.0.0.1:" + testPort) if err != nil { t.Fatal(err) } - defer cl.Close() + defer func() { + assert.NoError(t, cl.Close()) + }() err = submitMsg(t, cl, "x@example.org", []string{"ѣ@example.org"}, testMsg) @@ -154,14 +169,18 @@ func TestSMTPUTF8_NormalizeCaseFoldRcpt(t *testing.T) { tgt := testutils.Target{} endp := testEndpoint(t, "smtp", nil, &tgt, nil, nil) endp.deferServerReject = false - defer endp.Stop() + defer func() { + assert.NoError(t, endp.Stop()) + }() defer testutils.WaitForConnsClose(t, endp.serv) cl, err := smtp.Dial("127.0.0.1:" + testPort) if err != nil { t.Fatal(err) } - defer cl.Close() + defer func() { + assert.NoError(t, cl.Close()) + }() err = submitMsgOpts(t, cl, "x@example.org", []string{"foo@E\u0301.example.org"}, &smtp.MailOptions{ UTF8: true, @@ -191,14 +210,18 @@ func TestSMTPUTF8_NoMangleStatusMessage(t *testing.T) { }, }, nil) endp.deferServerReject = false - defer endp.Stop() + defer func() { + assert.NoError(t, endp.Stop()) + }() defer testutils.WaitForConnsClose(t, endp.serv) cl, err := smtp.Dial("127.0.0.1:" + testPort) if err != nil { t.Fatal(err) } - defer cl.Close() + defer func() { + assert.NoError(t, cl.Close()) + }() err = cl.Mail("sender@example.org", &smtp.MailOptions{ UTF8: true, @@ -222,14 +245,18 @@ func TestSMTPUTF8_NoMangleStatusMessage(t *testing.T) { func TestSMTPUTF8_Received_EHLO_ALabel(t *testing.T) { tgt := testutils.Target{} endp := testEndpoint(t, "smtp", nil, &tgt, nil, nil) - defer endp.Stop() + defer func() { + assert.NoError(t, endp.Stop()) + }() defer testutils.WaitForConnsClose(t, endp.serv) cl, err := smtp.Dial("127.0.0.1:" + testPort) if err != nil { t.Fatal(err) } - defer cl.Close() + defer func() { + assert.NoError(t, cl.Close()) + }() if err := cl.Hello("凱凱.invalid"); err != nil { t.Fatal(err) @@ -256,7 +283,9 @@ func TestSMTPUTF8_Received_EHLO_ALabel(t *testing.T) { func TestSMTPUTF8_Received_rDNS_ALabel(t *testing.T) { tgt := testutils.Target{} endp := testEndpoint(t, "smtp", nil, &tgt, nil, nil) - defer endp.Stop() + defer func() { + assert.NoError(t, endp.Stop()) + }() defer testutils.WaitForConnsClose(t, endp.serv) endp.resolver.(*mockdns.Resolver).Zones["1.0.0.127.in-addr.arpa."] = mockdns.Zone{ @@ -267,7 +296,9 @@ func TestSMTPUTF8_Received_rDNS_ALabel(t *testing.T) { if err != nil { t.Fatal(err) } - defer cl.Close() + defer func() { + assert.NoError(t, cl.Close()) + }() err = submitMsg(t, cl, "sender@example.org", []string{"rcpt1@example.com", "rcpt2@example.com"}, testMsg) if err != nil { @@ -290,7 +321,9 @@ func TestSMTPUTF8_Received_rDNS_ALabel(t *testing.T) { func TestSMTPUTF8_Received_rDNS_ULabel(t *testing.T) { tgt := testutils.Target{} endp := testEndpoint(t, "smtp", nil, &tgt, nil, nil) - defer endp.Stop() + defer func() { + assert.NoError(t, endp.Stop()) + }() defer testutils.WaitForConnsClose(t, endp.serv) endp.resolver.(*mockdns.Resolver).Zones["1.0.0.127.in-addr.arpa."] = mockdns.Zone{ @@ -301,7 +334,9 @@ func TestSMTPUTF8_Received_rDNS_ULabel(t *testing.T) { if err != nil { t.Fatal(err) } - defer cl.Close() + defer func() { + assert.NoError(t, cl.Close()) + }() err = submitMsgOpts(t, cl, "sender@example.org", []string{"rcpt1@example.com", "rcpt2@example.com"}, &smtp.MailOptions{ UTF8: true, @@ -326,14 +361,18 @@ func TestSMTPUTF8_Received_rDNS_ULabel(t *testing.T) { func TestSMTPUTF8_Received_EHLO_ULabel(t *testing.T) { tgt := testutils.Target{} endp := testEndpoint(t, "smtp", nil, &tgt, nil, nil) - defer endp.Stop() + defer func() { + assert.NoError(t, endp.Stop()) + }() defer testutils.WaitForConnsClose(t, endp.serv) cl, err := smtp.Dial("127.0.0.1:" + testPort) if err != nil { t.Fatal(err) } - defer cl.Close() + defer func() { + assert.NoError(t, cl.Close()) + }() if err := cl.Hello("凱凱.invalid"); err != nil { t.Fatal(err) diff --git a/internal/endpoint/smtp/submission_test.go b/internal/endpoint/smtp/submission_test.go index beb4cf5e..91f030cf 100644 --- a/internal/endpoint/smtp/submission_test.go +++ b/internal/endpoint/smtp/submission_test.go @@ -26,6 +26,7 @@ import ( "github.com/emersion/go-message/textproto" "github.com/emersion/go-smtp" "github.com/foxcpp/maddy/framework/module" + "github.com/stretchr/testify/assert" ) func init() { @@ -54,9 +55,9 @@ func TestSubmissionPrepare(t *testing.T) { // Synchronize the endpoint initialization. // Otherwise Close will race with Serve called by setupListeners. cl, _ := smtp.Dial("127.0.0.1:" + testPort) - cl.Close() + assert.NoError(t, cl.Close()) - endp.Stop() + assert.NoError(t, endp.Stop()) }() session, err := endp.NewSession(nil) diff --git a/internal/libdns/gcore.go b/internal/libdns/gcore.go new file mode 100644 index 00000000..69d0da5e --- /dev/null +++ b/internal/libdns/gcore.go @@ -0,0 +1,33 @@ +//go:build libdns_gcore || !libdns_separate + +package libdns + +import ( + "fmt" + + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/module" + "github.com/libdns/gcore" +) + +func init() { + module.Register("libdns.gcore", func(modName, instName string) (module.Module, error) { + p := gcore.Provider{} + return &ProviderModule{ + RecordDeleter: &p, + RecordAppender: &p, + setConfig: func(c *config.Map) { + c.String("api_key", false, false, "", &p.APIKey) + }, + afterConfig: func() error { + if p.APIKey == "" { + return fmt.Errorf("libdns.gcore: api_key should be specified") + } + return nil + }, + + instName: instName, + modName: modName, + }, nil + }) +} diff --git a/internal/table/sqlite3.go b/internal/sqlite/is.go similarity index 80% rename from internal/table/sqlite3.go rename to internal/sqlite/is.go index 8b22794b..953179a1 100644 --- a/internal/table/sqlite3.go +++ b/internal/sqlite/is.go @@ -1,9 +1,6 @@ -//go:build !nosqlite3 && cgo -// +build !nosqlite3,cgo - /* Maddy Mail Server - Composable all-in-one email server. -Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors +Copyright © 2019-2026 Max Mazurov , Maddy Mail Server contributors This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -19,6 +16,8 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . */ -package table +package sqliteprovider -import _ "github.com/mattn/go-sqlite3" +func IsSqliteDriver(name string) bool { + return name == "sqlite" || name == "sqlite3" +} diff --git a/internal/sqlite/modernc_sqlite3.go b/internal/sqlite/modernc_sqlite3.go new file mode 100644 index 00000000..6e31cf73 --- /dev/null +++ b/internal/sqlite/modernc_sqlite3.go @@ -0,0 +1,35 @@ +//go:build (!nosqlite3 && !cgo) || modernc + +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2026 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package sqliteprovider + +import _ "modernc.org/sqlite" + +const ( + IsAvailable = true + IsTranspiled = true +) + +func MapDriverName(n string) string { + if n == "sqlite3" { + return "sqlite" + } + return n +} diff --git a/internal/storage/imapsql/no_sqlite3.go b/internal/sqlite/no_sqlite3.go similarity index 79% rename from internal/storage/imapsql/no_sqlite3.go rename to internal/sqlite/no_sqlite3.go index 525f8e41..17682ae8 100644 --- a/internal/storage/imapsql/no_sqlite3.go +++ b/internal/sqlite/no_sqlite3.go @@ -1,9 +1,8 @@ //go:build nosqlite3 -// +build nosqlite3 /* Maddy Mail Server - Composable all-in-one email server. -Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors +Copyright © 2019-2026 Max Mazurov , Maddy Mail Server contributors This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -19,6 +18,13 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . */ -package imapsql +package sqliteprovider -const sqliteImpl = "missing" +const ( + IsAvailable = false + IsTranspiled = false +) + +func MapDriverName(n string) string { + return n +} diff --git a/internal/storage/imapsql/sqlite3.go b/internal/sqlite/sqlite3.go similarity index 73% rename from internal/storage/imapsql/sqlite3.go rename to internal/sqlite/sqlite3.go index 599f39d7..38fb1dee 100644 --- a/internal/storage/imapsql/sqlite3.go +++ b/internal/sqlite/sqlite3.go @@ -1,9 +1,8 @@ -//go:build !nosqlite3 && cgo -// +build !nosqlite3,cgo +//go:build !nosqlite3 && cgo && !modernc /* Maddy Mail Server - Composable all-in-one email server. -Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors +Copyright © 2019-2026 Max Mazurov , Maddy Mail Server contributors This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -19,8 +18,18 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . */ -package imapsql +package sqliteprovider import _ "github.com/mattn/go-sqlite3" -const sqliteImpl = "cgo" +const ( + IsAvailable = true + IsTranspiled = false +) + +func MapDriverName(n string) string { + if n == "sqlite" { + return "sqlite3" + } + return n +} diff --git a/internal/storage/imapsql/imapsql.go b/internal/storage/imapsql/imapsql.go index ddee1358..89657aaf 100644 --- a/internal/storage/imapsql/imapsql.go +++ b/internal/storage/imapsql/imapsql.go @@ -48,6 +48,7 @@ import ( "github.com/foxcpp/maddy/framework/log" "github.com/foxcpp/maddy/framework/module" "github.com/foxcpp/maddy/internal/authz" + sqliteprovider "github.com/foxcpp/maddy/internal/sqlite" "github.com/foxcpp/maddy/internal/updatepipe" "github.com/foxcpp/maddy/internal/updatepipe/pubsub" @@ -172,16 +173,16 @@ func (store *Storage) Configure(inlineArgs []string, cfg *config.Map) error { return errors.New("imapsql: driver is required") } - if driver == "sqlite3" { - if sqliteImpl == "modernc" { - store.Log.Println("using transpiled SQLite (modernc.org/sqlite), this is experimental") - driver = "sqlite" - } else if sqliteImpl == "cgo" { + if sqliteprovider.IsSqliteDriver(driver) { + if sqliteprovider.IsTranspiled { + store.Log.Println("using transpiled SQLite (modernc.org/sqlite)") + } else if sqliteprovider.IsAvailable { store.Log.Debugln("using cgo SQLite") - } else if sqliteImpl == "missing" { + } else { return errors.New("imapsql: SQLite is not supported, recompile without no_sqlite3 tag set") } } + driver = sqliteprovider.MapDriverName(driver) deliveryNormFunc, ok := authz.NormalizeFuncs[deliveryNormalize] if !ok { @@ -301,7 +302,7 @@ func (store *Storage) EnableUpdatePipe(mode updatepipe.BackendMode) error { } switch store.driver { - case "sqlite3": + case "sqlite3", "sqlite": dbId := sha1.Sum([]byte(strings.Join(store.dsn, " "))) sockPath := filepath.Join( config.RuntimeDirectory, diff --git a/internal/table/file_test.go b/internal/table/file_test.go index 5c7389e3..50d2e1cc 100644 --- a/internal/table/file_test.go +++ b/internal/table/file_test.go @@ -26,6 +26,7 @@ import ( "github.com/foxcpp/maddy/framework/config" "github.com/foxcpp/maddy/internal/testutils" + "github.com/stretchr/testify/assert" ) func TestReadFile(t *testing.T) { @@ -36,8 +37,18 @@ func TestReadFile(t *testing.T) { if err != nil { t.Fatal(err) } - defer os.Remove(f.Name()) - defer f.Close() + defer func(name string) { + err := os.Remove(name) + if err != nil { + t.Log(err) + } + }(f.Name()) + defer func(f *os.File) { + err := f.Close() + if err != nil { + t.Log(err) + } + }(f) if _, err := f.WriteString(file); err != nil { t.Fatal(err) } @@ -91,12 +102,20 @@ func TestFileReload(t *testing.T) { if err != nil { t.Fatal(err) } - defer os.Remove(f.Name()) + defer func(name string) { + err := os.Remove(name) + if err != nil { + t.Log(err) + } + }(f.Name()) if _, err := f.WriteString(file); err != nil { - f.Close() + _ = f.Close() + t.Fatal(err) + } + err = f.Close() + if err != nil { t.Fatal(err) } - f.Close() mod, err := NewFile("", "") if err != nil { @@ -107,7 +126,9 @@ func TestFileReload(t *testing.T) { t.Fatal(err) } m.log = testutils.Logger(t, "file_map") - defer m.Stop() + defer func() { + assert.NoError(t, m.Stop()) + }() if err := mod.Configure([]string{f.Name()}, &config.Map{Block: config.Node{}}); err != nil { t.Fatal(err) @@ -145,12 +166,17 @@ func TestFileReload_Broken(t *testing.T) { if err != nil { t.Fatal(err) } - defer os.Remove(f.Name()) + defer func(name string) { + err := os.Remove(name) + if err != nil { + t.Fatal(err) + } + }(f.Name()) if _, err := f.WriteString(file); err != nil { - f.Close() + assert.NoError(t, f.Close()) t.Fatal(err) } - f.Close() + assert.NoError(t, f.Close()) mod, err := NewFile("", "") if err != nil { @@ -161,7 +187,12 @@ func TestFileReload_Broken(t *testing.T) { t.Fatal(err) } m.log = testutils.Logger(t, FileModName) - defer m.Stop() + defer func(m *File) { + err := m.Stop() + if err != nil { + t.Fatal(err) + } + }(m) if err := mod.Configure([]string{f.Name()}, &config.Map{Block: config.Node{}}); err != nil { t.Fatal(err) @@ -174,7 +205,12 @@ func TestFileReload_Broken(t *testing.T) { if _, err := f2.WriteString(":"); err != nil { t.Fatal(err) } - defer f2.Close() + defer func(f2 *os.File) { + err := f2.Close() + if err != nil { + t.Fatal(err) + } + }(f2) time.Sleep(3 * reloadInterval) @@ -195,10 +231,13 @@ func TestFileReload_Removed(t *testing.T) { t.Fatal(err) } if _, err := f.WriteString(file); err != nil { - f.Close() + _ = f.Close() + t.Fatal(err) + } + err = f.Close() + if err != nil { t.Fatal(err) } - f.Close() mod, err := NewFile("", "") if err != nil { @@ -209,13 +248,21 @@ func TestFileReload_Removed(t *testing.T) { t.Fatal(err) } m.log = testutils.Logger(t, FileModName) - defer m.Stop() + defer func(m *File) { + err := m.Stop() + if err != nil { + t.Fatal(err) + } + }(m) if err := mod.Configure([]string{f.Name()}, &config.Map{Block: config.Node{}}); err != nil { t.Fatal(err) } - os.Remove(f.Name()) + err = os.Remove(f.Name()) + if err != nil { + t.Fatal(err) + } time.Sleep(3 * reloadInterval) diff --git a/internal/table/sql_query.go b/internal/table/sql_query.go index 0fe5c64f..ec2c3d23 100644 --- a/internal/table/sql_query.go +++ b/internal/table/sql_query.go @@ -21,11 +21,13 @@ package table import ( "context" "database/sql" + "errors" "fmt" "strings" "github.com/foxcpp/maddy/framework/config" "github.com/foxcpp/maddy/framework/module" + sqliteprovider "github.com/foxcpp/maddy/internal/sqlite" _ "github.com/lib/pq" ) @@ -89,6 +91,7 @@ func (s *SQL) Configure(inlineArgs []string, cfg *config.Map) error { if driver == "postgres" && s.namedArgs { return config.NodeErr(cfg.Block, "PostgreSQL driver does not support named_args") } + driver = sqliteprovider.MapDriverName(driver) db, err := sql.Open(driver, strings.Join(dsnParts, " ")) if err != nil { @@ -156,7 +159,7 @@ func (s *SQL) Lookup(ctx context.Context, val string) (string, bool, error) { row = s.lookup.QueryRowContext(ctx, val) } if err := row.Scan(&repl); err != nil { - if err == sql.ErrNoRows { + if errors.Is(err, sql.ErrNoRows) { return "", false, nil } return "", false, fmt.Errorf("%s: lookup %s: %w", s.modName, val, err) diff --git a/internal/target/queue/queue.go b/internal/target/queue/queue.go index 663046b3..8657110d 100644 --- a/internal/target/queue/queue.go +++ b/internal/target/queue/queue.go @@ -89,7 +89,6 @@ import ( // partialError describes state of partially successful message delivery. type partialError struct { - // Underlying error objects for each recipient. Errs map[string]error @@ -123,13 +122,12 @@ type Queue struct { location string hostname string autogenMsgDomain string - wheel *TimeWheel + wheel *TimeWheel[queueSlot] dsnPipeline module.DeliveryTarget // Retry delay is calculated using the following formula: // initialRetryTime * retryTimeScale ^ (TriesCount - 1) - initialRetryTime time.Duration retryTimeScale float64 maxTries int @@ -213,6 +211,9 @@ func (q *Queue) Configure(inlineArgs []string, cfg *config.Map) error { cfg.Bool("debug", true, false, &q.Log.Debug) cfg.Int("max_tries", false, false, 20, &q.maxTries) cfg.Int("max_parallelism", false, false, 16, &q.maxParallelism) + cfg.Duration("post_init_delay", false, false, q.postInitDelay, &q.postInitDelay) + cfg.Duration("initial_retry_time", false, false, q.initialRetryTime, &q.initialRetryTime) + cfg.Float("retry_time_scale", false, false, q.retryTimeScale, &q.retryTimeScale) cfg.String("location", false, false, q.location, &q.location) cfg.Custom("target", false, true, nil, modconfig.DeliveryDirective, &q.Target) cfg.String("hostname", true, true, "", &q.hostname) @@ -251,7 +252,7 @@ func (q *Queue) Start() error { } func (q *Queue) start(maxParallelism int) error { - q.wheel = NewTimeWheel(q.dispatch) + q.wheel = NewTimeWheel[queueSlot](q.dispatch) q.deliverySemaphore = make(chan struct{}, maxParallelism) if err := q.readDiskQueue(); err != nil { @@ -263,13 +264,18 @@ func (q *Queue) start(maxParallelism int) error { return nil } -func (q *Queue) Stop() error { +func (q *Queue) EarlyStop() error { + // We must ensure queue state is consistent on disk before we proceed + // with configuration reload. q.wheel.Close() q.deliveryWg.Wait() - return nil } +func (q *Queue) Stop() error { + return q.EarlyStop() +} + // discardBroken changes the name of metadata file to have .meta_broken // extension. // @@ -285,8 +291,8 @@ func (q *Queue) discardBroken(id string) { } } -func (q *Queue) dispatch(value TimeSlot) { - slot := value.Value.(queueSlot) +func (q *Queue) dispatch(ctx context.Context, value TimeSlot[queueSlot]) { + slot := value.Value q.Log.Debugln("starting delivery for", slot.ID) @@ -331,7 +337,7 @@ func (q *Queue) dispatch(value TimeSlot) { body = slot.Body } - q.tryDelivery(meta, hdr, body) + q.tryDelivery(ctx, meta, hdr, body) }() } @@ -375,10 +381,10 @@ func toSMTPErr(err error) *smtp.SMTPError { return res } -func (q *Queue) tryDelivery(meta *QueueMetadata, header textproto.Header, body buffer.Buffer) { +func (q *Queue) tryDelivery(ctx context.Context, meta *QueueMetadata, header textproto.Header, body buffer.Buffer) { dl := target.DeliveryLogger(q.Log, meta.MsgMeta) - partialErr := q.deliver(meta, header, body) + partialErr := q.deliver(ctx, meta, header, body) dl.Debugf("errors: %v", partialErr.Errs) // While iterating the list of recipients we also pick the smallest tries count @@ -462,7 +468,7 @@ func (q *Queue) tryDelivery(meta *QueueMetadata, header textproto.Header, body b }) } -func (q *Queue) deliver(meta *QueueMetadata, header textproto.Header, body buffer.Buffer) partialError { +func (q *Queue) deliver(ctx context.Context, meta *QueueMetadata, header textproto.Header, body buffer.Buffer) partialError { dl := target.DeliveryLogger(q.Log, meta.MsgMeta) perr := partialError{ Errs: map[string]error{}, @@ -473,7 +479,7 @@ func (q *Queue) deliver(meta *QueueMetadata, header textproto.Header, body buffe msgMeta.ID = msgMeta.ID + "-" + strconv.FormatInt(time.Now().Unix(), 16) dl.Debugf("using message ID = %s", msgMeta.ID) - msgCtx, msgTask := trace.NewTask(context.Background(), "Queue delivery") + msgCtx, msgTask := trace.NewTask(ctx, "Queue delivery") defer msgTask.End() mailCtx, mailTask := trace.NewTask(msgCtx, "MAIL FROM") @@ -488,6 +494,15 @@ func (q *Queue) deliver(meta *QueueMetadata, header textproto.Header, body buffe } dl.Debugf("target.StartDelivery OK") + // Check in case delivery implementation is actually + // context-unaware. + if err := mailCtx.Err(); err != nil { + for _, rcpt := range meta.To { + perr.Errs[rcpt] = err + } + return perr + } + var acceptedRcpts []string for _, rcpt := range meta.To { rcptCtx, rcptTask := trace.NewTask(msgCtx, "RCPT TO") @@ -499,6 +514,15 @@ func (q *Queue) deliver(meta *QueueMetadata, header textproto.Header, body buffe acceptedRcpts = append(acceptedRcpts, rcpt) } rcptTask.End() + + // Check in case delivery implementation is actually + // context-unaware. + if err := mailCtx.Err(); err != nil { + for _, rcpt := range meta.To { + perr.Errs[rcpt] = err + } + return perr + } } if len(acceptedRcpts) == 0 { @@ -515,6 +539,10 @@ func (q *Queue) deliver(meta *QueueMetadata, header textproto.Header, body buffe } } + // At this point, it is too late to abort delivery. We should complete + // it or fail it consistently. + msgCtx = context.WithoutCancel(msgCtx) + bodyCtx, bodyTask := trace.NewTask(msgCtx, "DATA") defer bodyTask.End() @@ -639,6 +667,9 @@ func (q *Queue) removeFromDisk(msgMeta *module.MsgMetadata) { if err := os.Remove(metaPath); err != nil { dl.Error("failed to remove meta-data from disk", err) } + + queuedMsgs.WithLabelValues(q.name, q.location).Dec() + dl.Debugf("removed message from disk") } @@ -708,6 +739,8 @@ func (q *Queue) readDiskQueue() error { ID: id, }) loadedCount++ + + queuedMsgs.WithLabelValues(q.name, q.location).Inc() } if loadedCount != 0 { @@ -766,6 +799,8 @@ func (q *Queue) storeNewMessage(meta *QueueMetadata, header textproto.Header, bo return nil, err } + queuedMsgs.WithLabelValues(q.name, q.location).Inc() + return buffer.FileBuffer{Path: bodyPath, LenHint: body.Len()}, nil } diff --git a/internal/target/queue/queue_test.go b/internal/target/queue/queue_test.go index 205ba891..6eb7a101 100644 --- a/internal/target/queue/queue_test.go +++ b/internal/target/queue/queue_test.go @@ -39,6 +39,7 @@ import ( "github.com/foxcpp/maddy/framework/log" "github.com/foxcpp/maddy/framework/module" "github.com/foxcpp/maddy/internal/testutils" + "github.com/stretchr/testify/assert" ) // newTestQueue returns properly initialized Queue object usable for testing. @@ -245,7 +246,7 @@ func TestQueueDelivery(t *testing.T) { // Wait for the delivery to complete and stop processing. msg := readMsgChanTimeout(t, dt.committed, 5*time.Second) - q.Stop() + assert.NoError(t, q.Stop()) testutils.CheckMsgID(t, msg, "tester@example.com", []string{"tester1@example.org", "tester2@example.org"}, "") @@ -269,7 +270,7 @@ func TestQueueDelivery_PermanentFail_NonPartial(t *testing.T) { // Queue will abort a delivery if it fails for all recipients. readMsgChanTimeout(t, dt.aborted, 5*time.Second) - q.Stop() + assert.NoError(t, q.Stop()) // Delivery is failed permanently, hence no retry should be rescheduled. checkQueueDir(t, q, []string{}) @@ -296,7 +297,7 @@ func TestQueueDelivery_PermanentFail_Partial(t *testing.T) { // Here delivery fails for recipients too, but this is reported using PartialDelivery. readMsgChanTimeout(t, dt.aborted, 5*time.Second) - q.Stop() + assert.NoError(t, q.Stop()) checkQueueDir(t, q, []string{}) } @@ -322,7 +323,7 @@ func TestQueueDelivery_TemporaryFail(t *testing.T) { msg := readMsgChanTimeout(t, dt.committed, 5*time.Second) testutils.CheckMsgID(t, msg, "tester@example.com", []string{"tester1@example.org", "tester2@example.org"}, "") - q.Stop() + assert.NoError(t, q.Stop()) // No more retries scheduled, queue storage is clear. defer checkQueueDir(t, q, []string{}) } @@ -355,7 +356,7 @@ func TestQueueDelivery_TemporaryFail_Partial(t *testing.T) { msg = readMsgChanTimeout(t, dt.committed, 5000*time.Second) testutils.CheckMsgID(t, msg, "tester@example.com", []string{"tester2@example.org"}, "") - q.Stop() + assert.NoError(t, q.Stop()) // No more retries scheduled, queue storage is clear. checkQueueDir(t, q, []string{}) } @@ -395,7 +396,7 @@ func TestQueueDelivery_MultipleAttempts(t *testing.T) { msg = readMsgChanTimeout(t, dt.committed, 5*time.Second) testutils.CheckMsgID(t, msg, "tester@example.com", []string{"tester2@example.org"}, "") - q.Stop() + assert.NoError(t, q.Stop()) // No more retries should be scheduled. checkQueueDir(t, q, []string{}) } @@ -420,7 +421,7 @@ func TestQueueDelivery_PermanentRcptReject(t *testing.T) { msg := readMsgChanTimeout(t, dt.committed, 5*time.Second) testutils.CheckMsgID(t, msg, "tester@example.org", []string{"tester2@example.org"}, "") - q.Stop() + assert.NoError(t, q.Stop()) // No more retries should be scheduled. checkQueueDir(t, q, []string{}) } @@ -454,7 +455,7 @@ func TestQueueDelivery_TemporaryRcptReject(t *testing.T) { msg = readMsgChanTimeout(t, dt.committed, 5*time.Second) testutils.CheckMsgID(t, msg, "tester@example.com", []string{"tester1@example.org"}, "") - q.Stop() + assert.NoError(t, q.Stop()) // No more retries should be scheduled. checkQueueDir(t, q, []string{}) } @@ -488,7 +489,7 @@ func TestQueueDelivery_SerializationRoundtrip(t *testing.T) { testutils.CheckMsgID(t, msg, "tester@example.com", []string{"tester2@example.org"}, "") // Then stop it. - q.Stop() + assert.NoError(t, q.Stop()) // Make sure it is saved. checkQueueDir(t, q, []string{deliveryID}) @@ -501,7 +502,7 @@ func TestQueueDelivery_SerializationRoundtrip(t *testing.T) { testutils.CheckMsgID(t, msg, "tester@example.com", []string{"tester1@example.org"}, "") // Close it again. - q.Stop() + assert.NoError(t, q.Stop()) // No more retries should be scheduled. checkQueueDir(t, q, []string{}) } @@ -535,7 +536,7 @@ func TestQueueDelivery_DeserlizationCleanUp(t *testing.T) { msg := readMsgChanTimeout(t, dt.committed, 5*time.Second) testutils.CheckMsgID(t, msg, "tester@example.com", []string{"tester2@example.org"}, "") - q.Stop() + assert.NoError(t, q.Stop()) if err := os.Remove(filepath.Join(q.location, deliveryID+fileSuffix)); err != nil { t.Fatal(err) @@ -543,7 +544,7 @@ func TestQueueDelivery_DeserlizationCleanUp(t *testing.T) { // Dangling files should be removed during load. q = newTestQueueDir(t, &dt, q.location) - q.Stop() + assert.NoError(t, q.Stop()) // Nothing should be left. checkQueueDir(t, q, []string{}) diff --git a/internal/target/queue/timewheel.go b/internal/target/queue/timewheel.go index 060804b2..cc385e3c 100644 --- a/internal/target/queue/timewheel.go +++ b/internal/target/queue/timewheel.go @@ -20,17 +20,18 @@ package queue import ( "container/list" + "context" "sync" "sync/atomic" "time" ) -type TimeSlot struct { +type TimeSlot[Value any] struct { Time time.Time - Value interface{} + Value Value } -type TimeWheel struct { +type TimeWheel[Value any] struct { stopped uint32 slots *list.List @@ -38,39 +39,41 @@ type TimeWheel struct { updateNotify chan time.Time stopNotify chan struct{} + tickerCtx context.Context + tickerCancel context.CancelFunc - dispatch func(TimeSlot) + dispatch func(context.Context, TimeSlot[Value]) } -func NewTimeWheel(dispatch func(TimeSlot)) *TimeWheel { - tw := &TimeWheel{ +func NewTimeWheel[Value any](dispatch func(context.Context, TimeSlot[Value])) *TimeWheel[Value] { + ctx, cancel := context.WithCancel(context.Background()) + + tw := &TimeWheel[Value]{ slots: list.New(), stopNotify: make(chan struct{}), + tickerCtx: ctx, + tickerCancel: cancel, updateNotify: make(chan time.Time), dispatch: dispatch, } - go tw.tick() + go tw.tick(context.Background()) return tw } -func (tw *TimeWheel) Add(target time.Time, value interface{}) { +func (tw *TimeWheel[Value]) Add(target time.Time, value Value) { if atomic.LoadUint32(&tw.stopped) == 1 { // Already stopped, ignore. return } - if value == nil { - panic("can't insert nil objects into TimeWheel queue") - } - tw.slotsLock.Lock() - tw.slots.PushBack(TimeSlot{Time: target, Value: value}) + tw.slots.PushBack(TimeSlot[Value]{Time: target, Value: value}) tw.slotsLock.Unlock() tw.updateNotify <- target } -func (tw *TimeWheel) Close() { +func (tw *TimeWheel[Value]) Close() { atomic.StoreUint32(&tw.stopped, 1) // Idempotent Close is convenient sometimes. @@ -78,6 +81,8 @@ func (tw *TimeWheel) Close() { return } + tw.tickerCancel() + tw.stopNotify <- struct{}{} <-tw.stopNotify @@ -86,16 +91,16 @@ func (tw *TimeWheel) Close() { close(tw.updateNotify) } -func (tw *TimeWheel) tick() { +func (tw *TimeWheel[Value]) tick(ctx context.Context) { for { now := time.Now() // Look for list element closest to now. tw.slotsLock.Lock() - var closestSlot TimeSlot + var closestSlot TimeSlot[Value] var closestEl *list.Element for e := tw.slots.Front(); e != nil; e = e.Next() { - slot := e.Value.(TimeSlot) - if slot.Time.Sub(now) < closestSlot.Time.Sub(now) || closestSlot.Value == nil { + slot := e.Value.(TimeSlot[Value]) + if slot.Time.Sub(now) < closestSlot.Time.Sub(now) || closestEl == nil { closestSlot = slot closestEl = e } @@ -124,7 +129,7 @@ func (tw *TimeWheel) tick() { tw.slots.Remove(closestEl) tw.slotsLock.Unlock() - tw.dispatch(closestSlot) + tw.dispatch(ctx, closestSlot) break selectloop case newTarget := <-tw.updateNotify: diff --git a/internal/target/queue/timewheel_test.go b/internal/target/queue/timewheel_test.go index d758b603..9beb171e 100644 --- a/internal/target/queue/timewheel_test.go +++ b/internal/target/queue/timewheel_test.go @@ -19,6 +19,7 @@ along with this program. If not, see . package queue import ( + "context" "testing" "time" ) @@ -26,9 +27,9 @@ import ( func TestTimeWheelAdd(t *testing.T) { t.Parallel() - called := make(chan TimeSlot) + called := make(chan TimeSlot[int]) - w := NewTimeWheel(func(slot TimeSlot) { + w := NewTimeWheel[int](func(ctx context.Context, slot TimeSlot[int]) { called <- slot }) defer w.Close() @@ -36,7 +37,7 @@ func TestTimeWheelAdd(t *testing.T) { w.Add(time.Now().Add(1*time.Second), 1) slot := <-called - if val, _ := slot.Value.(int); val != 1 { + if slot.Value != 1 { t.Errorf("Wrong slot value: %v", slot.Value) } } @@ -44,9 +45,9 @@ func TestTimeWheelAdd(t *testing.T) { func TestTimeWheelAdd_Ordering(t *testing.T) { t.Parallel() - called := make(chan TimeSlot) + called := make(chan TimeSlot[int]) - w := NewTimeWheel(func(slot TimeSlot) { + w := NewTimeWheel[int](func(ctx context.Context, slot TimeSlot[int]) { called <- slot }) defer w.Close() @@ -55,11 +56,11 @@ func TestTimeWheelAdd_Ordering(t *testing.T) { w.Add(time.Now().Add(1250*time.Millisecond), 2) slot := <-called - if val, _ := slot.Value.(int); val != 1 { + if slot.Value != 1 { t.Errorf("Wrong first slot value: %v", slot.Value) } slot = <-called - if val, _ := slot.Value.(int); val != 2 { + if slot.Value != 2 { t.Errorf("Wrong second slot value: %v", slot.Value) } } @@ -67,9 +68,9 @@ func TestTimeWheelAdd_Ordering(t *testing.T) { func TestTimeWheelAdd_Restart(t *testing.T) { t.Parallel() - called := make(chan TimeSlot) + called := make(chan TimeSlot[int]) - w := NewTimeWheel(func(slot TimeSlot) { + w := NewTimeWheel[int](func(ctx context.Context, slot TimeSlot[int]) { called <- slot }) defer w.Close() @@ -78,11 +79,11 @@ func TestTimeWheelAdd_Restart(t *testing.T) { w.Add(time.Now().Add(500*time.Millisecond), 2) slot := <-called - if val, _ := slot.Value.(int); val != 2 { + if slot.Value != 2 { t.Errorf("Wrong first slot value: %v", slot.Value) } slot = <-called - if val, _ := slot.Value.(int); val != 1 { + if slot.Value != 1 { t.Errorf("Wrong second slot value: %v", slot.Value) } } @@ -90,9 +91,9 @@ func TestTimeWheelAdd_Restart(t *testing.T) { func TestTimeWheelAdd_MissingGotoBug(t *testing.T) { t.Parallel() - called := make(chan TimeSlot) + called := make(chan TimeSlot[int]) - w := NewTimeWheel(func(slot TimeSlot) { + w := NewTimeWheel[int](func(ctx context.Context, slot TimeSlot[int]) { called <- slot }) defer w.Close() @@ -101,7 +102,7 @@ func TestTimeWheelAdd_MissingGotoBug(t *testing.T) { w.Add(time.Now().Add(500*time.Millisecond), 2) // should correctly restart slot := <-called - if val, _ := slot.Value.(int); val != 2 { + if slot.Value != 2 { t.Errorf("Wrong first slot value: %v", slot.Value) } } @@ -109,9 +110,9 @@ func TestTimeWheelAdd_MissingGotoBug(t *testing.T) { func TestTimeWheelAdd_EmptyUpdWait(t *testing.T) { t.Parallel() - called := make(chan TimeSlot) + called := make(chan TimeSlot[int]) - w := NewTimeWheel(func(slot TimeSlot) { + w := NewTimeWheel[int](func(ctx context.Context, slot TimeSlot[int]) { called <- slot }) defer w.Close() @@ -121,7 +122,7 @@ func TestTimeWheelAdd_EmptyUpdWait(t *testing.T) { w.Add(time.Now().Add(1*time.Second), 1) slot := <-called - if val, _ := slot.Value.(int); val != 1 { + if slot.Value != 1 { t.Errorf("Wrong slot value: %v", slot.Value) } } diff --git a/internal/target/remote/mxauth_test.go b/internal/target/remote/mxauth_test.go index 978567c5..f979a53d 100644 --- a/internal/target/remote/mxauth_test.go +++ b/internal/target/remote/mxauth_test.go @@ -32,6 +32,7 @@ import ( "github.com/foxcpp/maddy/framework/dns" "github.com/foxcpp/maddy/framework/module" "github.com/foxcpp/maddy/internal/testutils" + "github.com/stretchr/testify/assert" ) func TestRemoteDelivery_AuthMX_MTASTS(t *testing.T) { @@ -63,7 +64,9 @@ func TestRemoteDelivery_AuthMX_MTASTS(t *testing.T) { testSTSPolicy(t, zones, mtastsGet), }) tgt.tlsConfig = clientCfg - defer tgt.Stop() + defer func() { + assert.NoError(t, tgt.Stop()) + }() testutils.DoTestDelivery(t, tgt, "test@example.com", []string{"test@example.invalid"}) be.CheckMsg(t, 0, "test@example.com", []string{"test@example.invalid"}) @@ -108,7 +111,9 @@ func TestRemoteDelivery_MTASTS_SkipNonMatching(t *testing.T) { &localPolicy{minMXLevel: module.MX_MTASTS}, }) tgt.tlsConfig = clientCfg - defer tgt.Stop() + defer func() { + assert.NoError(t, tgt.Stop()) + }() testutils.DoTestDelivery(t, tgt, "test@example.com", []string{"test@example.invalid"}) be.CheckMsg(t, 0, "test@example.com", []string{"test@example.invalid"}) @@ -148,7 +153,9 @@ func TestRemoteDelivery_AuthMX_MTASTS_Fail(t *testing.T) { &localPolicy{minMXLevel: module.MX_MTASTS}, }) tgt.tlsConfig = clientCfg - defer tgt.Stop() + defer func() { + assert.NoError(t, tgt.Stop()) + }() _, err := testutils.DoTestDeliveryErr(t, tgt, "test@example.com", []string{"test@example.invalid"}) if err == nil { @@ -189,7 +196,9 @@ func TestRemoteDelivery_AuthMX_MTASTS_NoTLS(t *testing.T) { testSTSPolicy(t, zones, mtastsGet), &localPolicy{minMXLevel: module.MX_MTASTS}, }) - defer tgt.Stop() + defer func() { + assert.NoError(t, tgt.Stop()) + }() _, err := testutils.DoTestDeliveryErr(t, tgt, "test@example.com", []string{"test@example.invalid"}) if err == nil { @@ -230,7 +239,12 @@ func TestRemoteDelivery_AuthMX_MTASTS_RequirePKIX(t *testing.T) { testSTSPolicy(t, zones, mtastsGet), &localPolicy{minMXLevel: module.MX_MTASTS}, }) - defer tgt.Stop() + defer func(tgt *Target) { + err := tgt.Stop() + if err != nil { + t.Fatal(err) + } + }(tgt) _, err := testutils.DoTestDeliveryErr(t, tgt, "test@example.com", []string{"test@example.invalid"}) if err == nil { @@ -280,7 +294,9 @@ func TestRemoteDelivery_AuthMX_MTASTS_NoPolicy(t *testing.T) { testSTSPolicy(t, zones, mtastsGet), &localPolicy{minMXLevel: module.MX_MTASTS}, }) - defer tgt.Stop() + defer func() { + assert.NoError(t, tgt.Stop()) + }() _, err := testutils.DoTestDeliveryErr(t, tgt, "test@example.com", []string{"test@example.invalid"}) if err == nil { @@ -323,7 +339,9 @@ func TestRemoteDelivery_AuthMX_DNSSEC(t *testing.T) { extResolver.Cfg.Port = strconv.Itoa(addr.Port) tgt := testTarget(t, zones, extResolver, nil) - defer tgt.Stop() + defer func() { + assert.NoError(t, tgt.Stop()) + }() testutils.DoTestDelivery(t, tgt, "test@example.com", []string{"test@example.invalid"}) be.CheckMsg(t, 0, "test@example.com", []string{"test@example.invalid"}) @@ -365,7 +383,9 @@ func TestRemoteDelivery_AuthMX_DNSSEC_Fail(t *testing.T) { tgt := testTarget(t, zones, extResolver, []module.MXAuthPolicy{ &localPolicy{minMXLevel: module.MX_DNSSEC}, }) - defer tgt.Stop() + defer func() { + assert.NoError(t, tgt.Stop()) + }() _, err = testutils.DoTestDeliveryErr(t, tgt, "test@example.com", []string{"test@example.invalid"}) if err == nil { @@ -407,7 +427,9 @@ func TestRemoteDelivery_REQUIRETLS(t *testing.T) { testSTSPolicy(t, zones, mtastsGet), }) tgt.tlsConfig = clientCfg - defer tgt.Stop() + defer func() { + assert.NoError(t, tgt.Stop()) + }() testutils.DoTestDeliveryMeta(t, tgt, "test@example.com", []string{"test@example.invalid"}, &module.MsgMetadata{ OriginalFrom: "test@example.com", @@ -448,7 +470,9 @@ func TestRemoteDelivery_REQUIRETLS_Fail(t *testing.T) { testSTSPolicy(t, zones, mtastsGet), }) tgt.tlsConfig = clientCfg - defer tgt.Stop() + defer func() { + assert.NoError(t, tgt.Stop()) + }() if _, err := testutils.DoTestDeliveryErrMeta(t, tgt, "test@example.com", []string{"test@example.invalid"}, &module.MsgMetadata{ OriginalFrom: "test@example.com", @@ -494,7 +518,9 @@ func TestRemoteDelivery_REQUIRETLS_Relaxed(t *testing.T) { }) tgt.relaxedREQUIRETLS = true tgt.tlsConfig = clientCfg - defer tgt.Stop() + defer func() { + assert.NoError(t, tgt.Stop()) + }() testutils.DoTestDeliveryMeta(t, tgt, "test@example.com", []string{"test@example.invalid"}, &module.MsgMetadata{ OriginalFrom: "test@example.com", @@ -531,7 +557,9 @@ func TestRemoteDelivery_REQUIRETLS_Relaxed_NoMXAuth(t *testing.T) { }) tgt.relaxedREQUIRETLS = true tgt.tlsConfig = clientCfg - defer tgt.Stop() + defer func() { + assert.NoError(t, tgt.Stop()) + }() if _, err := testutils.DoTestDeliveryErrMeta(t, tgt, "test@example.com", []string{"test@example.invalid"}, &module.MsgMetadata{ OriginalFrom: "test@example.com", @@ -577,7 +605,9 @@ func TestRemoteDelivery_REQUIRETLS_Relaxed_NoTLS(t *testing.T) { }) tgt.relaxedREQUIRETLS = true tgt.tlsConfig = nil - defer tgt.Stop() + defer func() { + assert.NoError(t, tgt.Stop()) + }() if _, err := testutils.DoTestDeliveryErrMeta(t, tgt, "test@example.com", []string{"test@example.invalid"}, &module.MsgMetadata{ OriginalFrom: "test@example.com", @@ -628,7 +658,9 @@ func TestRemoteDelivery_REQUIRETLS_Relaxed_TLSFail(t *testing.T) { srv.TLSConfig.MinVersion = tls.VersionTLS11 srv.TLSConfig.MaxVersion = tls.VersionTLS11 tgt.tlsConfig = clientCfg - defer tgt.Stop() + defer func() { + assert.NoError(t, tgt.Stop()) + }() if _, err := testutils.DoTestDeliveryErrMeta(t, tgt, "test@example.com", []string{"test@example.invalid"}, &module.MsgMetadata{ OriginalFrom: "test@example.com", diff --git a/internal/target/remote/remote.go b/internal/target/remote/remote.go index 79b683fd..e300edd8 100644 --- a/internal/target/remote/remote.go +++ b/internal/target/remote/remote.go @@ -147,7 +147,7 @@ func (rt *Target) Configure(inlineArgs []string, cfg *config.Map) error { MaxKeys: 5000, MaxConnsPerKey: 5, // basically, max. amount of idle connections in cache MaxConnLifetimeSec: 150, // 2.5 mins, half of recommended idle time from RFC 5321 - StaleKeyLifetimeSec: 60 * 5, // should be bigger than MaxConnLifetimeSec + StaleKeyLifetimeSec: 60 * 4, // make sure that cleanup runs before recommended idle time from RFC 5321 } cfg.Int("conn_max_idle_count", false, false, 5, &poolCfg.MaxConnsPerKey) cfg.Int64("conn_max_idle_time", false, false, 150, &poolCfg.MaxConnLifetimeSec) diff --git a/internal/target/remote/remote_test.go b/internal/target/remote/remote_test.go index 99b2492a..b21f97e0 100644 --- a/internal/target/remote/remote_test.go +++ b/internal/target/remote/remote_test.go @@ -40,6 +40,7 @@ import ( "github.com/foxcpp/maddy/internal/limits" "github.com/foxcpp/maddy/internal/smtpconn/pool" "github.com/foxcpp/maddy/internal/testutils" + "github.com/stretchr/testify/assert" ) // .invalid TLD is used here to make sure if there is something wrong about @@ -64,7 +65,7 @@ func testTarget(t *testing.T, zones map[string]mockdns.Zone, extResolver *dns.Ex MaxKeys: 5000, MaxConnsPerKey: 5, // basically, max. amount of idle connections in cache MaxConnLifetimeSec: 150, // 2.5 mins, half of recommended idle time from RFC 5321 - StaleKeyLifetimeSec: 60 * 5, // should be bigger than MaxConnLifetimeSec + StaleKeyLifetimeSec: 60 * 4, // make sure that cleanup runs before recommended idle time from RFC 5321 }), } @@ -129,7 +130,9 @@ func TestRemoteDelivery(t *testing.T) { } tgt := testTarget(t, zones, nil, nil) - defer tgt.Stop() + defer func() { + assert.NoError(t, tgt.Stop()) + }() testutils.DoTestDelivery(t, tgt, "test@example.com", []string{"test@example.invalid"}) be.CheckMsg(t, 0, "test@example.com", []string{"test@example.invalid"}) @@ -146,7 +149,9 @@ func TestRemoteDelivery_NoMXFallback(t *testing.T) { } tgt := testTarget(t, zones, nil, nil) - defer tgt.Stop() + defer func() { + assert.NoError(t, tgt.Stop()) + }() delivery, err := tgt.StartDelivery(context.Background(), &module.MsgMetadata{ID: "test..."}, "test@example.com") if err != nil { @@ -176,7 +181,9 @@ func TestRemoteDelivery_EmptySender(t *testing.T) { } tgt := testTarget(t, zones, nil, nil) - defer tgt.Stop() + defer func() { + assert.NoError(t, tgt.Stop()) + }() testutils.DoTestDelivery(t, tgt, "", []string{"test@example.invalid"}) be.CheckMsg(t, 0, "", []string{"test@example.invalid"}) @@ -202,7 +209,9 @@ func TestRemoteDelivery_IPLiteral(t *testing.T) { } tgt := testTarget(t, zones, nil, nil) - defer tgt.Stop() + defer func() { + assert.NoError(t, tgt.Stop()) + }() testutils.DoTestDelivery(t, tgt, "test@example.com", []string{"test@[127.0.0.1]"}) be.CheckMsg(t, 0, "test@example.com", []string{"test@[127.0.0.1]"}) @@ -219,7 +228,9 @@ func TestRemoteDelivery_FallbackMX(t *testing.T) { } tgt := testTarget(t, zones, nil, nil) - defer tgt.Stop() + defer func() { + assert.NoError(t, tgt.Stop()) + }() testutils.DoTestDelivery(t, tgt, "test@example.com", []string{"test@example.invalid"}) be.CheckMsg(t, 0, "test@example.com", []string{"test@example.invalid"}) @@ -239,7 +250,9 @@ func TestRemoteDelivery_BodyNonAtomic(t *testing.T) { } tgt := testTarget(t, zones, nil, nil) - defer tgt.Stop() + defer func() { + assert.NoError(t, tgt.Stop()) + }() c := multipleErrs{ errs: map[string]error{}, @@ -267,7 +280,9 @@ func TestRemoteDelivery_Abort(t *testing.T) { } tgt := testTarget(t, zones, nil, nil) - defer tgt.Stop() + defer func() { + assert.NoError(t, tgt.Stop()) + }() delivery, err := tgt.StartDelivery(context.Background(), &module.MsgMetadata{ID: "test..."}, "test@example.com") if err != nil { @@ -297,7 +312,9 @@ func TestRemoteDelivery_CommitWithoutBody(t *testing.T) { } tgt := testTarget(t, zones, nil, nil) - defer tgt.Stop() + defer func() { + assert.NoError(t, tgt.Stop()) + }() delivery, err := tgt.StartDelivery(context.Background(), &module.MsgMetadata{ID: "test..."}, "test@example.com") if err != nil { @@ -334,7 +351,9 @@ func TestRemoteDelivery_MAILFROMErr(t *testing.T) { } tgt := testTarget(t, zones, nil, nil) - defer tgt.Stop() + defer func() { + assert.NoError(t, tgt.Stop()) + }() delivery, err := tgt.StartDelivery(context.Background(), &module.MsgMetadata{ID: "test..."}, "test@example.com") if err != nil { @@ -360,7 +379,9 @@ func TestRemoteDelivery_NoMX(t *testing.T) { } tgt := testTarget(t, zones, nil, nil) - defer tgt.Stop() + defer func() { + assert.NoError(t, tgt.Stop()) + }() delivery, err := tgt.StartDelivery(context.Background(), &module.MsgMetadata{ID: "test..."}, "test@example.com") if err != nil { @@ -390,7 +411,9 @@ func TestRemoteDelivery_NullMX(t *testing.T) { } tgt := testTarget(t, zones, nil, nil) - defer tgt.Stop() + defer func() { + assert.NoError(t, tgt.Stop()) + }() delivery, err := tgt.StartDelivery(context.Background(), &module.MsgMetadata{ID: "test..."}, "test@example.com") if err != nil { @@ -419,7 +442,9 @@ func TestRemoteDelivery_Quarantined(t *testing.T) { } tgt := testTarget(t, zones, nil, nil) - defer tgt.Stop() + defer func() { + assert.NoError(t, tgt.Stop()) + }() meta := module.MsgMetadata{ID: "test..."} @@ -467,7 +492,9 @@ func TestRemoteDelivery_MAILFROMErr_Repeated(t *testing.T) { } tgt := testTarget(t, zones, nil, nil) - defer tgt.Stop() + defer func() { + assert.NoError(t, tgt.Stop()) + }() delivery, err := tgt.StartDelivery(context.Background(), &module.MsgMetadata{ID: "test..."}, "test@example.com") if err != nil { @@ -507,7 +534,9 @@ func TestRemoteDelivery_RcptErr(t *testing.T) { } tgt := testTarget(t, zones, nil, nil) - defer tgt.Stop() + defer func() { + assert.NoError(t, tgt.Stop()) + }() delivery, err := tgt.StartDelivery(context.Background(), &module.MsgMetadata{ID: "test..."}, "test@example.com") if err != nil { @@ -558,7 +587,9 @@ func TestRemoteDelivery_DownMX(t *testing.T) { } tgt := testTarget(t, zones, nil, nil) - defer tgt.Stop() + defer func() { + assert.NoError(t, tgt.Stop()) + }() testutils.DoTestDelivery(t, tgt, "test@example.com", []string{"test@example.invalid"}) be.CheckMsg(t, 0, "test@example.com", []string{"test@example.invalid"}) @@ -581,7 +612,9 @@ func TestRemoteDelivery_AllMXDown(t *testing.T) { } tgt := testTarget(t, zones, nil, nil) - defer tgt.Stop() + defer func() { + assert.NoError(t, tgt.Stop()) + }() _, err := testutils.DoTestDeliveryErr(t, tgt, "test@example.com", []string{"test@example.invalid"}) if err == nil { @@ -612,7 +645,9 @@ func TestRemoteDelivery_Split(t *testing.T) { } tgt := testTarget(t, zones, nil, nil) - defer tgt.Stop() + defer func() { + assert.NoError(t, tgt.Stop()) + }() testutils.DoTestDelivery(t, tgt, "test@example.com", []string{"test@example.invalid", "test@example2.invalid"}) @@ -651,7 +686,9 @@ func TestRemoteDelivery_Split_Fail(t *testing.T) { } tgt := testTarget(t, zones, nil, nil) - defer tgt.Stop() + defer func() { + assert.NoError(t, tgt.Stop()) + }() delivery, err := tgt.StartDelivery(context.Background(), &module.MsgMetadata{ID: "test..."}, "test@example.com") if err != nil { @@ -704,7 +741,9 @@ func TestRemoteDelivery_BodyErr(t *testing.T) { } tgt := testTarget(t, zones, nil, nil) - defer tgt.Stop() + defer func() { + assert.NoError(t, tgt.Stop()) + }() delivery, err := tgt.StartDelivery(context.Background(), &module.MsgMetadata{ID: "test..."}, "test@example.com") if err != nil { @@ -758,7 +797,9 @@ func TestRemoteDelivery_Split_BodyErr(t *testing.T) { } tgt := testTarget(t, zones, nil, nil) - defer tgt.Stop() + defer func() { + assert.NoError(t, tgt.Stop()) + }() delivery, err := tgt.StartDelivery(context.Background(), &module.MsgMetadata{ID: "test..."}, "test@example.com") if err != nil { @@ -814,7 +855,9 @@ func TestRemoteDelivery_Split_BodyErr_NonAtomic(t *testing.T) { } tgt := testTarget(t, zones, nil, nil) - defer tgt.Stop() + defer func() { + assert.NoError(t, tgt.Stop()) + }() delivery, err := tgt.StartDelivery(context.Background(), &module.MsgMetadata{ID: "test..."}, "test@example.com") if err != nil { @@ -874,7 +917,9 @@ func TestRemoteDelivery_TLSErrFallback(t *testing.T) { tgt := testTarget(t, zones, nil, nil) tgt.tlsConfig = clientCfg - defer tgt.Stop() + defer func() { + assert.NoError(t, tgt.Stop()) + }() testutils.DoTestDelivery(t, tgt, "test@example.com", []string{"test@example.invalid"}) be.CheckMsg(t, 0, "test@example.com", []string{"test@example.invalid"}) @@ -896,7 +941,9 @@ func TestRemoteDelivery_RequireTLS_Missing(t *testing.T) { tgt := testTarget(t, zones, nil, []module.MXAuthPolicy{ &localPolicy{minTLSLevel: module.TLSEncrypted}, }) - defer tgt.Stop() + defer func() { + assert.NoError(t, tgt.Stop()) + }() _, err := testutils.DoTestDeliveryErr(t, tgt, "test@example.com", []string{"test@example.invalid"}) if err == nil { @@ -921,7 +968,9 @@ func TestRemoteDelivery_RequireTLS_Present(t *testing.T) { &localPolicy{minTLSLevel: module.TLSEncrypted}, }) tgt.tlsConfig = clientCfg - defer tgt.Stop() + defer func() { + assert.NoError(t, tgt.Stop()) + }() testutils.DoTestDelivery(t, tgt, "test@example.com", []string{"test@example.invalid"}) be.CheckMsg(t, 0, "test@example.com", []string{"test@example.invalid"}) @@ -950,7 +999,9 @@ func TestRemoteDelivery_RequireTLS_NoErrFallback(t *testing.T) { &localPolicy{minTLSLevel: module.TLSEncrypted}, }) tgt.tlsConfig = clientCfg - defer tgt.Stop() + defer func() { + assert.NoError(t, tgt.Stop()) + }() _, err := testutils.DoTestDeliveryErr(t, tgt, "test@example.com", []string{"test@example.invalid"}) if err == nil { @@ -975,7 +1026,9 @@ func TestRemoteDelivery_TLS_FallbackNoVerify(t *testing.T) { tgt := testTarget(t, zones, nil, []module.MXAuthPolicy{ &localPolicy{minTLSLevel: module.TLSEncrypted}, }) - defer tgt.Stop() + defer func() { + assert.NoError(t, tgt.Stop()) + }() testutils.DoTestDelivery(t, tgt, "test@example.com", []string{"test@example.invalid"}) be.CheckMsg(t, 0, "test@example.com", []string{"test@example.invalid"}) @@ -1008,7 +1061,9 @@ func TestRemoteDelivery_TLS_FallbackPlaintext(t *testing.T) { tgt := testTarget(t, zones, nil, nil) tgt.tlsConfig = clientCfg - defer tgt.Stop() + defer func() { + assert.NoError(t, tgt.Stop()) + }() testutils.DoTestDelivery(t, tgt, "test@example.com", []string{"test@example.invalid"}) be.CheckMsg(t, 0, "test@example.com", []string{"test@example.invalid"}) @@ -1041,7 +1096,9 @@ func TestRemoteDelivery_ConnReuse(t *testing.T) { tgt := testTarget(t, zones, nil, nil) tgt.connReuseLimit = 5 - defer tgt.Stop() + defer func() { + assert.NoError(t, tgt.Stop()) + }() testutils.DoTestDelivery(t, tgt, "test@example.com", []string{"test@example.invalid"}) testutils.DoTestDelivery(t, tgt, "test@example.com", []string{"test@example.invalid"}) diff --git a/internal/target/smtp/smtp_downstream.go b/internal/target/smtp/smtp_downstream.go index 8c50b71b..d48b56ad 100644 --- a/internal/target/smtp/smtp_downstream.go +++ b/internal/target/smtp/smtp_downstream.go @@ -56,7 +56,7 @@ type Downstream struct { hostname string endpoints []config.Endpoint saslFactory saslClientFactory - tlsConfig tls.Config + tlsConfig *tls.Config connectTimeout time.Duration commandTimeout time.Duration @@ -119,7 +119,7 @@ func (u *Downstream) Configure(inlineArgs []string, cfg *config.Map) error { return nil, nil }, saslAuthDirective, &u.saslFactory) cfg.Custom("tls_client", true, false, func() (interface{}, error) { - return tls.Config{}, nil + return &tls.Config{}, nil }, tls2.TLSClientBlock, &u.tlsConfig) cfg.Duration("connect_timeout", false, false, 5*time.Minute, &u.connectTimeout) cfg.Duration("command_timeout", false, false, 5*time.Minute, &u.commandTimeout) @@ -227,9 +227,9 @@ func (d *delivery) connect(ctx context.Context) error { for _, endp := range d.u.endpoints { var err error if d.u.lmtp { - _, err = conn.ConnectLMTP(ctx, endp, d.u.starttls, &d.u.tlsConfig) + _, err = conn.ConnectLMTP(ctx, endp, d.u.starttls, d.u.tlsConfig) } else { - _, err = conn.Connect(ctx, endp, d.u.starttls, &d.u.tlsConfig) + _, err = conn.Connect(ctx, endp, d.u.starttls, d.u.tlsConfig) } if err != nil { if len(d.u.endpoints) != 1 { diff --git a/internal/target/smtp/smtp_downstream_test.go b/internal/target/smtp/smtp_downstream_test.go index 31ef2954..f0f58c66 100644 --- a/internal/target/smtp/smtp_downstream_test.go +++ b/internal/target/smtp/smtp_downstream_test.go @@ -221,7 +221,7 @@ func TestDownstreamDelivery_StartTLS(t *testing.T) { Port: testPort, }, }, - tlsConfig: *clientCfg.Clone(), + tlsConfig: clientCfg.Clone(), starttls: true, log: testutils.Logger(t, "target.smtp"), } diff --git a/maddy.go b/maddy.go index c60360e2..23eea317 100644 --- a/maddy.go +++ b/maddy.go @@ -112,6 +112,11 @@ func init() { Value: filepath.Join(ConfigDirectory, "maddy.conf"), }, ) + maddycli.AddGlobalFlag(&cli.BoolFlag{ + Name: "debug", + Usage: "enable debug logging early", + Destination: &log.DefaultLogger.Debug, + }) maddycli.AddSubcommand(&cli.Command{ Name: "verify-config", Usage: "Check configuration file for errors", @@ -128,11 +133,6 @@ func init() { Name: "run", Usage: "Start the server", Flags: []cli.Flag{ - &cli.BoolFlag{ - Name: "debug", - Usage: "enable debug logging early", - Destination: &log.DefaultLogger.Debug, - }, &cli.StringFlag{ Name: "libexec", Value: DefaultLibexecDirectory, @@ -207,7 +207,11 @@ func Run(c *cli.Context) error { defer log.DefaultLogger.Out.Close() defer hooks.RunHooks(hooks.EventShutdown) - hooks.AddHook(hooks.EventShutdown, netresource.CloseAllListeners) + defer func() { + if err := netresource.CloseAllListeners(); err != nil { + log.DefaultLogger.Error("CloseAllListeners failed", err) + } + }() if err := moduleMain(c.Path("config")); err != nil { systemdStatusErr(err) @@ -305,7 +309,7 @@ func ensureDirectoryWritable(path string) error { return err } testFile.Close() - return os.Remove(testFile.Name()) + return os.RemoveAll(testFile.Name()) } func ReadGlobals(c *container.C, cfg []config.Node) (map[string]interface{}, []config.Node, error) { @@ -373,8 +377,14 @@ func moduleStart(c *container.C) error { return c.Lifetime.StartAll() } -func moduleStop(c *container.C) { - c.Lifetime.StopAll() +func moduleStop(c *container.C, earlyStop bool) error { + if earlyStop { + if err := c.Lifetime.EarlyStopAll(); err != nil { + log.DefaultLogger.Error("early stop failed", err) + } + } + + return c.Lifetime.StopAll() } func moduleMain(configPath string) error { @@ -412,7 +422,9 @@ func moduleMain(configPath string) error { asyncStopWg.Wait() systemdStatus(SDStopping, "Waiting for current configuration to stop...") - moduleStop(c) + if err := moduleStop(c, true); err != nil { + c.DefaultLogger.Msg("moduleStop failed", err) + } c.DefaultLogger.Msg("server stopped") return nil @@ -431,6 +443,12 @@ func moduleReload(oldContainer *container.C, configPath string, asyncStopWg *syn oldContainer.DefaultLogger.Msg("configuration loaded") + if err := oldContainer.Lifetime.EarlyStopAll(); err != nil { + oldContainer.DefaultLogger.Error("failed to early-stop old server", err) + container.Global = oldContainer + return oldContainer + } + netresource.ResetListenersUsage() oldContainer.DefaultLogger.Msg("starting new server") if err := moduleStart(newContainer); err != nil { @@ -439,17 +457,23 @@ func moduleReload(oldContainer *container.C, configPath string, asyncStopWg *syn return oldContainer } - newContainer.DefaultLogger.Msg("server started", "version", Version) + newContainer.DefaultLogger.Msg("new server started", "version", Version) systemdStatus(SDReloading, "New configuration running. Waiting for old connections and transactions to finish...") asyncStopWg.Add(1) go func() { defer asyncStopWg.Done() - defer netresource.CloseUnusedListeners() + defer func() { + if err := netresource.CloseUnusedListeners(); err != nil { + oldContainer.DefaultLogger.Error("CloseUnusedListeners failed", err) + } + }() oldContainer.DefaultLogger.Msg("stopping old server") - moduleStop(oldContainer) + if err := moduleStop(oldContainer, false); err != nil { + oldContainer.DefaultLogger.Error("moduleStop failed", err) + } oldContainer.DefaultLogger.Msg("old server stopped") systemdStatus(SDReloading, "Configuration running.") diff --git a/tests/build_cover.sh b/tests/build_cover.sh index 929511c2..b724fd5a 100755 --- a/tests/build_cover.sh +++ b/tests/build_cover.sh @@ -2,4 +2,4 @@ if [ -z "$GO" ]; then GO=go fi -exec $GO test -tags 'cover_main debugflags' -coverpkg 'github.com/foxcpp/maddy,github.com/foxcpp/maddy/pkg/...,github.com/foxcpp/maddy/internal/...' -cover -covermode atomic -c cover_test.go -o maddy.cover +exec $GO test -race -tags 'cover_main debugflags' -coverpkg 'github.com/foxcpp/maddy,github.com/foxcpp/maddy/pkg/...,github.com/foxcpp/maddy/internal/...' -cover -covermode atomic -c cover_test.go -o maddy.cover diff --git a/tests/cover_test.go b/tests/cover_test.go index d47c8840..298e3e9f 100644 --- a/tests/cover_test.go +++ b/tests/cover_test.go @@ -42,8 +42,10 @@ import ( "os" "testing" - "github.com/foxcpp/maddy" - "github.com/urfave/cli/v2" + _ "github.com/foxcpp/maddy" // To register run command + _ "github.com/foxcpp/maddy/internal/cli/ctl" // To register other CLI commands. + + maddycli "github.com/foxcpp/maddy/internal/cli" ) func TestMain(m *testing.M) { @@ -56,16 +58,14 @@ func TestMain(m *testing.M) { panic(err) } + // Skip flag parsing and make flag.Parse no-op so when + // m.Run calls it it will not error out on maddy flags. + args := os.Args + os.Args = []string{"command"} flag.Parse() + os.Args = args - app := cli.NewApp() - // maddycli wrapper registers all necessary flags with flag.CommandLine by default - ctx := cli.NewContext(app, flag.CommandLine, nil) - err = maddy.Run(ctx) - code := 0 - if ec, ok := err.(cli.ExitCoder); ok { - code = ec.ExitCode() - } + code := maddycli.RunWithoutExit() if err := os.Chdir(wd); err != nil { panic(err) diff --git a/tests/dovecot_sasl_test.go b/tests/dovecot_sasl_test.go index af424e81..140b1820 100644 --- a/tests/dovecot_sasl_test.go +++ b/tests/dovecot_sasl_test.go @@ -1,10 +1,8 @@ //go:build integration && (darwin || dragonfly || freebsd || linux || netbsd || openbsd || solaris) -// +build integration -// +build darwin dragonfly freebsd linux netbsd openbsd solaris /* Maddy Mail Server - Composable all-in-one email server. -Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors +Copyright © 2019-2026 Max Mazurov , Maddy Mail Server contributors This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -26,9 +24,9 @@ package tests_test import ( "bufio" + "bytes" "errors" "flag" - "io/ioutil" "os" "os/exec" "os/user" @@ -47,7 +45,8 @@ func init() { flag.StringVar(&DovecotExecutable, "integration.dovecot", "dovecot", "path to dovecot executable for interop tests") } -const dovecotConf = `base_dir = $ROOT/run/ +const dovecotConf = ` +base_dir = $ROOT/run/ state_dir = $ROOT/lib/ log_path = /dev/stderr ssl = no @@ -56,12 +55,14 @@ default_internal_user = $USER default_internal_group = $GROUP default_login_user = $USER +auth_failure_delay = 0 + passdb { driver = passwd-file args = $ROOT/passwd } -userdb { +userdb file { driver = passwd-file args = $ROOT/passwd } @@ -78,7 +79,7 @@ protocols = imap service imap-login { chroot = inet_listener imap { - address = 127.0.0.1 + listen = 127.0.0.1 port = 0 } } @@ -95,8 +96,64 @@ auth_verbose_passwords = yes mail_debug = yes ` +const dovecotConf24 = `dovecot_config_version = 2.4.0 +dovecot_storage_version = 2.4.0 + +base_dir = $ROOT/run/ +state_dir = $ROOT/lib/ +mail_plugin_dir = $ROOT/lib/ +login_plugin_dir = $ROOT/lib/ +log_path = /dev/stderr +ssl = no + +default_internal_user = $USER +default_internal_group = $GROUP +default_login_user = $USER + +auth_failure_delay = 0 + +passdb file { + driver = passwd-file + passwd_file_path = $ROOT/passwd +} + +userdb file { + driver = passwd-file + passwd_file_path = $ROOT/passwd +} + +service auth { + unix_listener auth { + mode = 0666 + } +} + +# Turn on debugging information, to help troubleshooting issues. +auth_verbose = yes +auth_debug = yes +auth_debug_passwords = yes +auth_verbose_passwords = yes +mail_debug = yes +` + const dovecotPasswd = `tester:{plain}123456:1000:1000::/home/user` +func isDovecot24(t *testing.T, dovecotExec string) bool { + cmd := exec.Command(dovecotExec, "--version") + var stdout bytes.Buffer + cmd.Stdout = &stdout + if err := cmd.Run(); err != nil { + t.Fatal(err) + } + + version, _, _ := strings.Cut(stdout.String(), "-") + t.Log("Dovecot version:", stdout.String()) + + parts := strings.SplitN(version, ".", 3) + + return len(parts) >= 2 && parts[0] == "2" && parts[1] >= "4" +} + func runDovecot(t *testing.T) (string, *exec.Cmd) { dovecotExec, err := exec.LookPath(DovecotExecutable) if err != nil { @@ -117,15 +174,20 @@ func runDovecot(t *testing.T) (string, *exec.Cmd) { t.Fatal(err) } + dovecotConfTemplate := dovecotConf + if isDovecot24(t, dovecotExec) { + dovecotConfTemplate = dovecotConf24 + } + dovecotConf := strings.NewReplacer( "$ROOT", tempDir, "$USER", curUser.Username, - "$GROUP", curGroup.Name).Replace(dovecotConf) - err = ioutil.WriteFile(filepath.Join(tempDir, "dovecot.conf"), []byte(dovecotConf), os.ModePerm) + "$GROUP", curGroup.Name).Replace(dovecotConfTemplate) + err = os.WriteFile(filepath.Join(tempDir, "dovecot.conf"), []byte(dovecotConf), os.ModePerm) if err != nil { t.Fatal(err) } - err = ioutil.WriteFile(filepath.Join(tempDir, "passwd"), []byte(dovecotPasswd), os.ModePerm) + err = os.WriteFile(filepath.Join(tempDir, "passwd"), []byte(dovecotPasswd), os.ModePerm) if err != nil { t.Fatal(err) } @@ -147,9 +209,14 @@ func runDovecot(t *testing.T) (string, *exec.Cmd) { for scnr.Scan() { line := scnr.Text() - // One of messages printed near completing initialization. + // One of messages printed near completing initialization (Dovecot 2.3 or older) if strings.Contains(line, "starting up for imap") { - time.Sleep(500*time.Millisecond) + time.Sleep(500 * time.Millisecond) + ready <- struct{}{} + } + // Dovecot 2.4+ + if strings.Contains(line, "starting up without any protocols") { + time.Sleep(500 * time.Millisecond) ready <- struct{}{} } diff --git a/tests/imapsql_test.go b/tests/imapsql_test.go index 69f9e7ed..776d7423 100644 --- a/tests/imapsql_test.go +++ b/tests/imapsql_test.go @@ -1,5 +1,4 @@ -//go:build integration && cgo && !nosqlite3 -// +build integration,cgo,!nosqlite3 +//go:build integration /* Maddy Mail Server - Composable all-in-one email server. diff --git a/tests/multiple_domains_test.go b/tests/multiple_domains_test.go new file mode 100644 index 00000000..6b29c3fa --- /dev/null +++ b/tests/multiple_domains_test.go @@ -0,0 +1,340 @@ +//go:build integration + +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2025 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package tests_test + +import ( + "testing" + + "github.com/foxcpp/maddy/tests" +) + +// Test cases based on https://maddy.email/multiple-domains/ + +func TestMultipleDomains_SeparateNamespace(tt *testing.T) { + tt.Parallel() + t := tests.NewT(tt) + t.DNS(nil) + t.Port("submission") + t.Port("imap") + t.Config(` + tls off + hostname test.maddy.email + + auth.pass_table local_authdb { + table sql_table { + driver sqlite3 + dsn credentials.db + table_name passwords + } + } + storage.imapsql local_mailboxes { + driver sqlite3 + dsn imapsql.db + } + + submission tcp://0.0.0.0:{env:TEST_PORT_submission} { + auth &local_authdb + reject + } + imap tcp://127.0.0.1:{env:TEST_PORT_imap} { + auth &local_authdb + storage &local_mailboxes + } + `) + + t.MustRunCLIGroup( + []string{"creds", "create", "-p", "user1", "user1@test1.maddy.email"}, + []string{"creds", "create", "-p", "user2", "user2@test1.maddy.email"}, + []string{"creds", "create", "-p", "user3", "user1@test2.maddy.email"}, + []string{"imap-acct", "create", "--no-specialuse", "user1@test1.maddy.email"}, + []string{"imap-acct", "create", "--no-specialuse", "user2@test1.maddy.email"}, + []string{"imap-acct", "create", "--no-specialuse", "user1@test2.maddy.email"}, + ) + t.Run(2) + + user1 := t.Conn("imap") + defer user1.Close() + user1.ExpectPattern(`\* OK *`) + user1.Writeln(`. LOGIN user1@test1.maddy.email user1`) + user1.ExpectPattern(`. OK *`) + user1.Writeln(`. CREATE user1`) + user1.ExpectPattern(`. OK *`) + + user1SMTP := t.Conn("submission") + defer user1SMTP.Close() + user1SMTP.SMTPNegotation("localhost", []string{"AUTH PLAIN"}, nil) + user1SMTP.SMTPPlainAuth("user1@test1.maddy.email", "user1", true) + + user2 := t.Conn("imap") + defer user2.Close() + user2.ExpectPattern(`\* OK *`) + user2.Writeln(`. LOGIN user2@test1.maddy.email user2`) + user2.ExpectPattern(`. OK *`) + user2.Writeln(`. CREATE user2`) + user2.ExpectPattern(`. OK *`) + + user2SMTP := t.Conn("submission") + defer user2SMTP.Close() + user2SMTP.SMTPNegotation("localhost", []string{"AUTH PLAIN"}, nil) + user2SMTP.SMTPPlainAuth("user2@test1.maddy.email", "user2", true) + + user3 := t.Conn("imap") + defer user3.Close() + user3.ExpectPattern(`\* OK *`) + user3.Writeln(`. LOGIN user1@test2.maddy.email user3`) + user3.ExpectPattern(`. OK *`) + user3.Writeln(`. CREATE user3`) + user3.ExpectPattern(`. OK *`) + + user3SMTP := t.Conn("submission") + defer user3SMTP.Close() + user3SMTP.SMTPNegotation("localhost", []string{"AUTH PLAIN"}, nil) + user3SMTP.SMTPPlainAuth("user1@test2.maddy.email", "user3", true) + + user1.Writeln(`. LIST "" "*"`) + user1.Expect(`* LIST (\HasNoChildren) "." INBOX`) + user1.Expect(`* LIST (\HasNoChildren) "." "user1"`) + user1.ExpectPattern(". OK *") + + user2.Writeln(`. LIST "" "*"`) + user2.Expect(`* LIST (\HasNoChildren) "." INBOX`) + user2.Expect(`* LIST (\HasNoChildren) "." "user2"`) + user2.ExpectPattern(". OK *") + + user3.Writeln(`. LIST "" "*"`) + user3.Expect(`* LIST (\HasNoChildren) "." INBOX`) + user3.Expect(`* LIST (\HasNoChildren) "." "user3"`) + user3.ExpectPattern(". OK *") +} + +func TestMultipleDomains_SharedCredentials_DistinctMailboxes(tt *testing.T) { + tt.Parallel() + t := tests.NewT(tt) + t.DNS(nil) + t.Port("submission") + t.Port("imap") + t.Config(` + tls off + hostname test.maddy.email + auth_map email_localpart + + auth.pass_table local_authdb { + table sql_table { + driver sqlite3 + dsn credentials.db + table_name passwords + } + } + storage.imapsql local_mailboxes { + driver sqlite3 + dsn imapsql.db + } + + submission tcp://0.0.0.0:{env:TEST_PORT_submission} { + auth &local_authdb + reject + } + imap tcp://127.0.0.1:{env:TEST_PORT_imap} { + auth &local_authdb + storage &local_mailboxes + } + `) + + t.MustRunCLIGroup( + []string{"creds", "create", "-p", "user1", "user1"}, + []string{"creds", "create", "-p", "user2", "user2"}, + []string{"imap-acct", "create", "--no-specialuse", "user1@test1.maddy.email"}, + []string{"imap-acct", "create", "--no-specialuse", "user2@test1.maddy.email"}, + []string{"imap-acct", "create", "--no-specialuse", "user1@test2.maddy.email"}, + ) + t.Run(2) + + user1 := t.Conn("imap") + defer user1.Close() + user1.ExpectPattern(`\* OK *`) + user1.Writeln(`. LOGIN user1@test1.maddy.email user1`) + user1.ExpectPattern(`. OK *`) + user1.Writeln(`. CREATE user1`) + user1.ExpectPattern(`. OK *`) + + user1SMTP := t.Conn("submission") + defer user1SMTP.Close() + user1SMTP.SMTPNegotation("localhost", []string{"AUTH PLAIN"}, nil) + user1SMTP.SMTPPlainAuth("user1@test1.maddy.email", "user1", true) + + user2 := t.Conn("imap") + defer user2.Close() + user2.ExpectPattern(`\* OK *`) + user2.Writeln(`. LOGIN user2@test1.maddy.email user2`) + user2.ExpectPattern(`. OK *`) + user2.Writeln(`. CREATE user2`) + user2.ExpectPattern(`. OK *`) + + user2SMTP := t.Conn("submission") + defer user2SMTP.Close() + user2SMTP.SMTPNegotation("localhost", []string{"AUTH PLAIN"}, nil) + user2SMTP.SMTPPlainAuth("user2@test1.maddy.email", "user2", true) + + user3 := t.Conn("imap") + defer user3.Close() + user3.ExpectPattern(`\* OK *`) + user3.Writeln(`. LOGIN user1@test2.maddy.email user1`) + user3.ExpectPattern(`. OK *`) + user3.Writeln(`. CREATE user3`) + user3.ExpectPattern(`. OK *`) + + user3SMTP := t.Conn("submission") + defer user3SMTP.Close() + user3SMTP.SMTPNegotation("localhost", []string{"AUTH PLAIN"}, nil) + user3SMTP.SMTPPlainAuth("user1@test2.maddy.email", "user1", true) + + user1.Writeln(`. LIST "" "*"`) + user1.Expect(`* LIST (\HasNoChildren) "." INBOX`) + user1.Expect(`* LIST (\HasNoChildren) "." "user1"`) + user1.ExpectPattern(". OK *") + + user2.Writeln(`. LIST "" "*"`) + user2.Expect(`* LIST (\HasNoChildren) "." INBOX`) + user2.Expect(`* LIST (\HasNoChildren) "." "user2"`) + user2.ExpectPattern(". OK *") + + user3.Writeln(`. LIST "" "*"`) + user3.Expect(`* LIST (\HasNoChildren) "." INBOX`) + user3.Expect(`* LIST (\HasNoChildren) "." "user3"`) + user3.ExpectPattern(". OK *") +} + +func TestMultipleDomains_SharedCredentials_SharedMailboxes(tt *testing.T) { + tt.Parallel() + t := tests.NewT(tt) + t.DNS(nil) + t.Port("submission") + t.Port("imap") + t.Config(` + tls off + hostname test.maddy.email + auth_map email_localpart_optional + + auth.pass_table local_authdb { + table sql_table { + driver sqlite3 + dsn credentials.db + table_name passwords + } + } + storage.imapsql local_mailboxes { + driver sqlite3 + dsn imapsql.db + + delivery_map email_localpart_optional + } + + submission tcp://0.0.0.0:{env:TEST_PORT_submission} { + auth &local_authdb + reject + } + imap tcp://127.0.0.1:{env:TEST_PORT_imap} { + auth &local_authdb + storage &local_mailboxes + + storage_map email_localpart_optional + } + `) + + t.MustRunCLIGroup( + []string{"creds", "create", "-p", "user1", "user1"}, + []string{"creds", "create", "-p", "user2", "user2"}, + []string{"imap-acct", "create", "--no-specialuse", "user1"}, + []string{"imap-acct", "create", "--no-specialuse", "user2"}, + ) + t.Run(2) + + user1 := t.Conn("imap") + defer user1.Close() + user1.ExpectPattern(`\* OK *`) + user1.Writeln(`. LOGIN user1 user1`) + user1.ExpectPattern(`. OK *`) + user1.Writeln(`. CREATE user1`) + user1.ExpectPattern(`. OK *`) + + user1SMTP := t.Conn("submission") + defer user1SMTP.Close() + user1SMTP.SMTPNegotation("localhost", []string{"AUTH PLAIN"}, nil) + user1SMTP.SMTPPlainAuth("user1", "user1", true) + + user2 := t.Conn("imap") + defer user2.Close() + user2.ExpectPattern(`\* OK *`) + user2.Writeln(`. LOGIN user2@test1.maddy.email user2`) + user2.ExpectPattern(`. OK *`) + user2.Writeln(`. CREATE user2`) + user2.ExpectPattern(`. OK *`) + + user2SMTP := t.Conn("submission") + defer user2SMTP.Close() + user2SMTP.SMTPNegotation("localhost", []string{"AUTH PLAIN"}, nil) + user2SMTP.SMTPPlainAuth("user2", "user2", true) + + user12 := t.Conn("imap") + defer user12.Close() + user12.ExpectPattern(`\* OK *`) + user12.Writeln(`. LOGIN user1@test2.maddy.email user1`) + user12.ExpectPattern(`. OK *`) + user12.Writeln(`. CREATE user12`) + user12.ExpectPattern(`. OK *`) + + user13 := t.Conn("imap") + defer user13.Close() + user13.ExpectPattern(`\* OK *`) + user13.Writeln(`. LOGIN user1@test.maddy.email user1`) + user13.ExpectPattern(`. OK *`) + user13.Writeln(`. CREATE user13`) + user13.ExpectPattern(`. OK *`) + + user12SMTP := t.Conn("submission") + defer user12SMTP.Close() + user12SMTP.SMTPNegotation("localhost", []string{"AUTH PLAIN"}, nil) + user12SMTP.SMTPPlainAuth("user1", "user1", true) + + user13SMTP := t.Conn("submission") + defer user13SMTP.Close() + user13SMTP.SMTPNegotation("localhost", []string{"AUTH PLAIN"}, nil) + user13SMTP.SMTPPlainAuth("user1@test.maddy.email", "user1", true) + + user1.Writeln(`. LIST "" "*"`) + user1.Expect(`* LIST (\HasNoChildren) "." INBOX`) + user1.Expect(`* LIST (\HasNoChildren) "." "user1"`) + user1.Expect(`* LIST (\HasNoChildren) "." "user12"`) + user1.Expect(`* LIST (\HasNoChildren) "." "user13"`) + user1.ExpectPattern(". OK *") + + user2.Writeln(`. LIST "" "*"`) + user2.Expect(`* LIST (\HasNoChildren) "." INBOX`) + user2.Expect(`* LIST (\HasNoChildren) "." "user2"`) + user2.ExpectPattern(". OK *") + + user12.Writeln(`. LIST "" "*"`) + user12.Expect(`* LIST (\HasNoChildren) "." INBOX`) + user12.Expect(`* LIST (\HasNoChildren) "." "user1"`) + user12.Expect(`* LIST (\HasNoChildren) "." "user12"`) + user12.Expect(`* LIST (\HasNoChildren) "." "user13"`) + user12.ExpectPattern(". OK *") +} diff --git a/internal/storage/imapsql/modernc_sqlite3.go b/tests/reload_non_unix.go similarity index 78% rename from internal/storage/imapsql/modernc_sqlite3.go rename to tests/reload_non_unix.go index 696b4c0a..a7511634 100644 --- a/internal/storage/imapsql/modernc_sqlite3.go +++ b/tests/reload_non_unix.go @@ -1,9 +1,8 @@ -//go:build !nosqlite3 && !cgo -// +build !nosqlite3,!cgo +//go:build !unix /* Maddy Mail Server - Composable all-in-one email server. -Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors +Copyright © 2019-2026 Max Mazurov , Maddy Mail Server contributors This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -19,8 +18,8 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . */ -package imapsql +package tests -import _ "modernc.org/sqlite" - -const sqliteImpl = "modernc" +func (t *T) reloadConfig() { + t.Skip("Tests for config reload are not available") +} diff --git a/tests/reload_test.go b/tests/reload_test.go new file mode 100644 index 00000000..be3a479b --- /dev/null +++ b/tests/reload_test.go @@ -0,0 +1,255 @@ +//go:build unix && integration + +// Can't reload on Windows, yet + +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2026 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package tests_test + +import ( + "testing" + "time" + + sqliteprovider "github.com/foxcpp/maddy/internal/sqlite" + "github.com/foxcpp/maddy/tests" +) + +func TestSmtpPipelineSwitch(tt *testing.T) { + if !sqliteprovider.IsTranspiled { + tt.Skip("Test is unstable with original SQLite") + } + + tt.Parallel() + t := tests.NewT(tt) + + t.DNS(nil) + t.Port("smtp") + t.Config(` + smtp tcp://127.0.0.1:{env:TEST_PORT_smtp} { + hostname maddy.test + tls off + + reject + } + `) + t.Run(1) + defer t.Close() + + conn1 := t.Conn("smtp") + defer conn1.Close() + conn1.SMTPNegotation("localhost", nil, nil) + conn1.Writeln("MAIL FROM:") + conn1.ExpectPattern("2*") + conn1.Writeln("RCPT TO:") + conn1.ExpectPattern("5*") // REJECTED + conn1.Writeln("RSET") + conn1.ExpectPattern("2*") + + t.Config(` + smtp tcp://127.0.0.1:{env:TEST_PORT_smtp} { + hostname maddy.test + tls off + + deliver_to dummy + } + `) + + conn2 := t.Conn("smtp") + defer conn2.Close() + conn2.SMTPNegotation("localhost", nil, nil) + conn2.Writeln("MAIL FROM:") + conn2.ExpectPattern("2*") + conn2.Writeln("RCPT TO:") + conn2.ExpectPattern("2*") + conn2.Writeln("DATA") + conn2.ExpectPattern("354 *") + conn2.Writeln("From: ") + conn2.Writeln("To: ") + conn2.Writeln("Subject: Hi!") + conn2.Writeln("") + conn2.Writeln("Hi!") + conn2.Writeln(".") + conn2.ExpectPattern("2*") // DISCARDED + + conn1.Writeln("MAIL FROM:") + conn1.ExpectPattern("2*") + conn1.Writeln("RCPT TO:") + conn1.ExpectPattern("5*") // Still REJECTED (running on old server). + conn1.Writeln("RSET") + conn1.ExpectPattern("2*") +} + +func TestImapStorageSwitch(tt *testing.T) { + if !sqliteprovider.IsTranspiled { + tt.Skip("Test is unstable with original SQLite") + } + + tt.Parallel() + t := tests.NewT(tt) + + t.DNS(nil) + t.Port("smtp") + t.Port("imap") + t.Config(` + storage.imapsql test_store { + driver sqlite3 + dsn imapsql.db + } + + imap tcp://127.0.0.1:{env:TEST_PORT_imap} { + tls off + + auth dummy + storage &test_store + } + + smtp tcp://127.0.0.1:{env:TEST_PORT_smtp} { + hostname maddy.test + tls off + + deliver_to &test_store + } + `) + t.Run(1) + defer t.Close() + + imapConn := t.Conn("imap") + defer imapConn.Close() + imapConn.ExpectPattern(`\* OK *`) + imapConn.Writeln(". LOGIN testusr@maddy.test 1234") + imapConn.ExpectPattern(". OK *") + imapConn.Writeln(". SELECT INBOX") + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`. OK *`) + + conn1 := t.Conn("smtp") + defer conn1.Close() + conn1.SMTPNegotation("localhost", nil, nil) + conn1.Writeln("MAIL FROM:") + conn1.ExpectPattern("2*") + conn1.Writeln("RCPT TO:") + conn1.ExpectPattern("2*") + conn1.Writeln("DATA") + conn1.ExpectPattern("354 *") + conn1.Writeln("From: ") + conn1.Writeln("To: ") + conn1.Writeln("Subject: Store 1") + conn1.Writeln("") + conn1.Writeln("Hi!") + conn1.Writeln(".") + conn1.ExpectPattern("2*") // Goes to storage 1 + + t.Config(` + storage.imapsql test_store { + driver sqlite3 + dsn imapsql2.db + } + + imap tcp://127.0.0.1:{env:TEST_PORT_imap} { + tls off + + auth dummy + storage &test_store + } + + smtp tcp://127.0.0.1:{env:TEST_PORT_smtp} { + hostname maddy.test + tls off + + deliver_to &test_store + } + `) + + imapConn2 := t.Conn("imap") + defer imapConn2.Close() + imapConn2.ExpectPattern(`\* OK *`) + imapConn2.Writeln(". LOGIN testusr2@maddy.test 1234") + imapConn2.ExpectPattern(". OK *") + + time.Sleep(500 * time.Millisecond) + + conn2 := t.Conn("smtp") + defer conn2.Close() + conn2.SMTPNegotation("localhost", nil, nil) + conn2.Writeln("MAIL FROM:") + conn2.ExpectPattern("2*") + conn2.Writeln("RCPT TO:") + conn2.ExpectPattern("2*") + conn2.Writeln("DATA") + conn2.ExpectPattern("354 *") + conn2.Writeln("From: ") + conn2.Writeln("To: ") + conn2.Writeln("Subject: Store 2") + conn2.Writeln("") + conn2.Writeln("Hi!") + conn2.Writeln(".") + conn2.ExpectPattern("2*") // Goes to storage 2 + + imapConn.Writeln(". NOOP") + imapConn.ExpectPattern(`\* 1 EXISTS`) + imapConn.ExpectPattern(`\* 1 RECENT`) + imapConn.ExpectPattern(". OK *") + + // Old connection sees message in store 1. + imapConn.Writeln(". FETCH 1 (BODY.PEEK[])") + imapConn.ExpectPattern(`\* 1 FETCH (BODY\[\] {*}*`) + imapConn.Expect(`Delivered-To: testusr@maddy.test`) + imapConn.Expect(`Return-Path: `) + imapConn.ExpectPattern(`Received: from localhost (client.maddy.test \[` + tests.DefaultSourceIP.String() + `\]) by maddy.test`) + imapConn.ExpectPattern(` (envelope-sender ) with ESMTP id *; *`) + imapConn.ExpectPattern(` *`) + imapConn.Expect("From: ") + imapConn.Expect("To: ") + imapConn.Expect("Subject: Store 1") + imapConn.Expect("") + imapConn.Expect("Hi!") + imapConn.Expect(")") + imapConn.ExpectPattern(`. OK *`) + + // New connection sees message in store 2. + imapConn2.Writeln(". SELECT INBOX") + imapConn2.ExpectPattern(`\* *`) + imapConn2.ExpectPattern(`\* *`) + imapConn2.ExpectPattern(`\* *`) + imapConn2.ExpectPattern(`\* *`) + imapConn2.ExpectPattern(`\* *`) + imapConn2.ExpectPattern(`\* *`) + imapConn2.ExpectPattern(`\* *`) + imapConn2.ExpectPattern(`. OK *`) + imapConn2.Writeln(". FETCH 1 (BODY.PEEK[])") + imapConn2.ExpectPattern(`\* 1 FETCH (BODY\[\] {*}*`) + imapConn2.Expect(`Delivered-To: testusr2@maddy.test`) + imapConn2.Expect(`Return-Path: `) + imapConn2.ExpectPattern(`Received: from localhost (client.maddy.test \[` + tests.DefaultSourceIP.String() + `\]) by maddy.test`) + imapConn2.ExpectPattern(` (envelope-sender ) with ESMTP id *; *`) + imapConn2.ExpectPattern(` *`) + imapConn2.Expect("From: ") + imapConn2.Expect("To: ") + imapConn2.Expect("Subject: Store 2") + imapConn2.Expect("") + imapConn2.Expect("Hi!") + imapConn2.Expect(")") + imapConn2.ExpectPattern(`. OK *`) + +} diff --git a/tests/reload_unix.go b/tests/reload_unix.go new file mode 100644 index 00000000..303f4e48 --- /dev/null +++ b/tests/reload_unix.go @@ -0,0 +1,42 @@ +//go:build unix + +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2026 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package tests + +import ( + "syscall" + "time" +) + +func (t *T) reloadConfig() { + err := t.servProc.Process.Signal(syscall.SIGUSR2) + if err != nil { + t.Fatal("Failed to send SIGUSR2:", err) + } + + t.Log("waiting for server to reload...") + + select { + case <-t.reloadedChan: + case <-time.After(5 * time.Second): + t.killServer() + t.Fatal("Server reload is taking too long, killed") + } +} diff --git a/tests/t.go b/tests/t.go index 2243662a..4e8d4fc1 100644 --- a/tests/t.go +++ b/tests/t.go @@ -25,6 +25,7 @@ package tests import ( "bufio" + "bytes" "flag" "fmt" "math/rand" @@ -34,6 +35,7 @@ import ( "path/filepath" "strconv" "strings" + "sync" "testing" "time" @@ -58,13 +60,16 @@ type T struct { portsRev map[uint16]string servProc *exec.Cmd + + reloadedChan chan struct{} } func NewT(t *testing.T) *T { return &T{ - T: t, - ports: map[string]uint16{}, - portsRev: map[uint16]string{}, + T: t, + ports: map[string]uint16{}, + portsRev: map[uint16]string{}, + reloadedChan: make(chan struct{}, 1), } } @@ -73,11 +78,21 @@ func NewT(t *testing.T) *T { func (t *T) Config(cfg string) { t.Helper() + t.cfg = cfg + if t.servProc != nil { - panic("tests: Config called after Run") - } + t.Log("Reloading configuration for running server...") - t.cfg = cfg + configPreable := "state_dir " + filepath.Join(t.testDir, "statedir") + "\n" + + "runtime_dir " + filepath.Join(t.testDir, "runtimedir") + "\n\n" + + err := os.WriteFile(filepath.Join(t.testDir, "maddy.conf"), []byte(configPreable+t.cfg), os.ModePerm) + if err != nil { + t.Fatal("Test configuration failed:", err) + } + + t.reloadConfig() + } } // DNS sets the DNS zones to emulate for the tested server instance. @@ -99,12 +114,25 @@ func (t *T) DNS(zones map[string]mockdns.Zone) { t.dnsServ.Close() } - dnsServ, err := mockdns.NewServer(zones, false) + dnsServ, err := mockdns.NewServerWithLogger(zones, t, false) if err != nil { t.Fatal("Test configuration failed:", err) } dnsServ.Log = t t.dnsServ = dnsServ + + t.Cleanup(func() { + if t.dnsServ == nil { + return + } + + // Shutdown the DNS server after maddy to make sure it will not spend time + // timing out queries. + if err := t.dnsServ.Close(); err != nil { + t.Log("Unable to stop the DNS server:", err) + } + t.dnsServ = nil + }) } // Port allocates the random TCP port for use by test. It will made accessible @@ -129,14 +157,7 @@ func (t *T) Env(kv string) { t.env = append(t.env, kv) } -// Run completes the configuration of test environment and starts the test server. -// -// T.Close should be called by the end of test to release any resources and -// shutdown the server. -// -// The parameter waitListeners specifies the amount of listeners the server is -// supposed to configure. Run() will block before all of them are up. -func (t *T) Run(waitListeners int) { +func (t *T) ensureCanRun() { if t.cfg == "" { panic("tests: Run called without configuration set") } @@ -149,63 +170,63 @@ func (t *T) Run(waitListeners int) { } // Setup file system, create statedir, runtimedir, write out config. - testDir, err := os.MkdirTemp("", "maddy-tests-") - if err != nil { - t.Fatal("Test configuration failed:", err) - } - t.testDir = testDir - - t.Log("Using", t.testDir) - - defer func() { - if !t.Failed() { - return + if t.testDir == "" { + testDir, err := os.MkdirTemp("", "maddy-tests-") + if err != nil { + t.Fatal("Test configuration failed:", err) } + t.testDir = testDir + t.Log("using", t.testDir) - // Clean-up on test failure (if Run failed somewhere) - - t.dnsServ.Close() - t.dnsServ = nil + if err := os.MkdirAll(filepath.Join(t.testDir, "statedir"), os.ModePerm); err != nil { + t.Fatal("Test configuration failed:", err) + } + if err := os.MkdirAll(filepath.Join(t.testDir, "runtimedir"), os.ModePerm); err != nil { + t.Fatal("Test configuration failed:", err) + } - os.RemoveAll(t.testDir) - t.testDir = "" - }() + t.Cleanup(func() { + if !t.Failed() { + return + } - if err := os.MkdirAll(filepath.Join(t.testDir, "statedir"), os.ModePerm); err != nil { - t.Fatal("Test configuration failed:", err) - } - if err := os.MkdirAll(filepath.Join(t.testDir, "runtimedir"), os.ModePerm); err != nil { - t.Fatal("Test configuration failed:", err) + t.Log("removing", t.testDir) + os.RemoveAll(t.testDir) + t.testDir = "" + }) } configPreable := "state_dir " + filepath.Join(t.testDir, "statedir") + "\n" + - "runtime_dir " + filepath.Join(t.testDir, "runtime") + "\n\n" + "runtime_dir " + filepath.Join(t.testDir, "runtimedir") + "\n\n" - err = os.WriteFile(filepath.Join(t.testDir, "maddy.conf"), []byte(configPreable+t.cfg), os.ModePerm) + err := os.WriteFile(filepath.Join(t.testDir, "maddy.conf"), []byte(configPreable+t.cfg), os.ModePerm) if err != nil { t.Fatal("Test configuration failed:", err) } +} +func (t *T) buildCmd(additionalArgs ...string) *exec.Cmd { // Assigning 0 by default will make outbound SMTP unusable. remoteSmtp := "0" if port := t.ports["remote_smtp"]; port != 0 { remoteSmtp = strconv.Itoa(int(port)) } - cmd := exec.Command(TestBinary, - "-config", filepath.Join(t.testDir, "maddy.conf"), + args := []string{"-config", filepath.Join(t.testDir, "maddy.conf"), "-debug.smtpport", remoteSmtp, "-debug.dnsoverride", t.dnsServ.LocalAddr().String(), - "-log", "stderr") + "-log", "/tmp/test.log"} if CoverageOut != "" { - cmd.Args = append(cmd.Args, "-test.coverprofile", CoverageOut+"."+strconv.FormatInt(time.Now().UnixNano(), 16)) + args = append(args, "-test.coverprofile", CoverageOut+"."+strconv.FormatInt(time.Now().UnixNano(), 16)) } if DebugLog { - cmd.Args = append(cmd.Args, "-debug") + args = append(args, "-debug") } - t.Logf("launching %v", cmd.Args) + args = append(args, additionalArgs...) + + cmd := exec.Command(TestBinary, args...) pwd, err := os.Getwd() if err != nil { @@ -217,53 +238,121 @@ func (t *T) Run(waitListeners int) { cmd.Env = append(cmd.Env, "TEST_PWD="+pwd, "TEST_STATE_DIR="+filepath.Join(t.testDir, "statedir"), - "TEST_RUNTIME_DIR="+filepath.Join(t.testDir, "statedir"), + "TEST_RUNTIME_DIR="+filepath.Join(t.testDir, "runtimedir"), ) for name, port := range t.ports { cmd.Env = append(cmd.Env, fmt.Sprintf("TEST_PORT_%s=%d", name, port)) } cmd.Env = append(cmd.Env, t.env...) + return cmd +} + +func (t *T) MustRunCLIGroup(args ...[]string) { + t.ensureCanRun() + + wg := sync.WaitGroup{} + for _, arg := range args { + wg.Add(1) + go func() { + defer wg.Done() + + _, err := t.RunCLI(arg...) + if err != nil { + t.Printf("maddy %v: %v", arg, err) + t.Fail() + } + }() + } + wg.Wait() +} + +func (t *T) MustRunCLI(args ...string) string { + s, err := t.RunCLI(args...) + if err != nil { + t.Fatalf("maddy %v: %v", args, err) + } + return s +} + +func (t *T) RunCLI(args ...string) (string, error) { + t.ensureCanRun() + cmd := t.buildCmd(args...) + + var stderr, stdout bytes.Buffer + cmd.Stderr = &stderr + cmd.Stdout = &stdout + + t.Log("launching maddy", cmd.Args) + if err := cmd.Run(); err != nil { + t.Log("Stderr:", stderr.String()) + t.Fatal("Test configuration failed:", err) + } + + t.Log("Stderr:", stderr.String()) + + return stdout.String(), nil +} + +// Run completes the configuration of test environment and starts the test server. +// +// T.Close should be called by the end of test to release any resources and +// shutdown the server. +// +// The parameter waitListeners specifies the amount of listeners the server is +// supposed to configure. Run() will block before all of them are up. +func (t *T) Run(waitListeners int) { + t.ensureCanRun() + cmd := t.buildCmd("run") + // Capture maddy log and redirect it. logOut, err := cmd.StderrPipe() if err != nil { t.Fatal("Test configuration failed:", err) } + t.Log("launching maddy", cmd.Args) if err := cmd.Start(); err != nil { t.Fatal("Test configuration failed:", err) } - // Log scanning goroutine checks for the "listening" messages and sends 'true' - // on the channel each time. - listeningMsg := make(chan bool) + serverStarted := make(chan bool) go func() { defer logOut.Close() - defer close(listeningMsg) + defer close(serverStarted) scnr := bufio.NewScanner(logOut) for scnr.Scan() { line := scnr.Text() - if strings.Contains(line, "listening on") { - listeningMsg <- true - line += " (test runner>listener wait trigger<)" + t.Log("maddy:", line) + + if strings.HasPrefix(line, "server started") { + serverStarted <- true } - t.Log("maddy:", line) + if strings.HasPrefix(line, "new server started") { + select { + case t.reloadedChan <- struct{}{}: + t.Log("server reload confirmed, continuing test") + default: + t.Log("unexpected reloads detected") + t.Fail() + } + } } if err := scnr.Err(); err != nil { t.Log("stderr I/O error:", err) } }() - for i := 0; i < waitListeners; i++ { - if !<-listeningMsg { - t.Fatal("Log ended before all expected listeners are up. Start-up error?") - } + if !<-serverStarted { + t.Fatal("Log ended before all expected listeners are up. Start-up error?") } t.servProc = cmd + + t.Cleanup(t.killServer) } func (t *T) StateDir() string { @@ -274,7 +363,7 @@ func (t *T) RuntimeDir() string { return filepath.Join(t.testDir, "statedir") } -func (t *T) Close() { +func (t *T) killServer() { if err := t.servProc.Process.Signal(os.Interrupt); err != nil { t.Log("Unable to kill the server process:", err) os.RemoveAll(t.testDir) @@ -299,13 +388,10 @@ func (t *T) Close() { t.Log("Failed to remove test directory:", err) } t.testDir = "" +} - // Shutdown the DNS server after maddy to make sure it will not spend time - // timing out queries. - if err := t.dnsServ.Close(); err != nil { - t.Log("Unable to stop the DNS server:", err) - } - t.dnsServ = nil +func (t *T) Close() { + t.Log("close is no-op") } // Printf implements Logger interfaces used by some libraries.