feat: fixed Message-ID per email and an effect handoff API for durable jobs

For the durable jobs outbox (QRun-IO/qqq#1191, spec section 2.13):
- Each queued message stores one RFC 5322 Message-ID (email_message.rfc_message_id,
  unique), derived from the idempotency key (else a stable per-message seed),
  and reuses it on every attempt. SMTP, Postmark, SendGrid and Mailgun send it.
  SES receives it in the raw MIME but replaces it; this is documented, and
  X-QQQ-Email-Id stays on every transport.
- EmailEffects.send(request, effectKey) hands an email to qbit-email's outbox
  exactly once per key: QUEUED_OR_DUPLICATE, or REFUSED (suppressed or
  invalid; never retried).
- findByIdempotencyKey / findByRfcMessageId look an email up for a reconcile.
- docs/DURABLE-JOBS.md has the outcome mapping and the binding class to add
  when QQQ RC.7 ships (EmailJobEffectTarget implements core's EffectTarget;
  no SNAPSHOT dependency now).
- Postgres upgrade SQL for existing installs; checkstyle now also covers test
  sources (the repo standard), with the existing violations fixed.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
james.maesandClaude Opus 5.5 committed 2026-10-14 08:58:59 -05:00
1 parent 351a8070db
commit 40f6935c19
35 files changed
+870 -84

No files matched your search

@@ -21,18 +21,34 @@
package com.kingsrook.qbits.email;
import java.net.ServerSocket;
import java.time.*;
import java.util.*;
import java.util.concurrent.*;
import java.time.Clock;
import java.time.Duration;
import java.time.Instant;
import java.time.ZoneOffset;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CyclicBarrier;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import com.icegreen.greenmail.util.GreenMail;
import com.icegreen.greenmail.util.ServerSetup;
import com.kingsrook.qbits.email.delivery.*;
import com.kingsrook.qbits.email.delivery.EmailDispatcher;
import com.kingsrook.qbits.email.metadata.EmailTables;
import com.kingsrook.qbits.email.outbox.*;
import com.kingsrook.qbits.email.outbox.EmailCategory;
import com.kingsrook.qbits.email.outbox.EmailRequest;
import com.kingsrook.qbits.email.outbox.EmailSendResult;
import com.kingsrook.qbits.email.outbox.EmailService;
import com.kingsrook.qbits.email.processes.EmailProcesses;
import com.kingsrook.qbits.email.support.*;
import com.kingsrook.qbits.email.transport.*;
import com.kingsrook.qbits.email.support.EmailStore;
import com.kingsrook.qbits.email.support.SystemWrites;
import com.kingsrook.qbits.email.transport.EmailTransport;
import com.kingsrook.qbits.email.transport.EmailTransportException;
import com.kingsrook.qbits.email.transport.EmailTransportRegistry;
import com.kingsrook.qbits.email.transport.EmailTransportResult;
import com.kingsrook.qbits.email.transport.SmtpEmailTransport;
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
import com.kingsrook.qqq.backend.core.actions.messaging.SendMessageAction;
import com.kingsrook.qqq.backend.core.alerts.AlertResolution;
@@ -41,16 +57,23 @@ import com.kingsrook.qqq.backend.core.alerts.AlertSinkInterface;
import com.kingsrook.qqq.backend.core.alerts.AlertsInstanceMetaData;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.messaging.*;
import com.kingsrook.qqq.backend.core.model.actions.messaging.email.*;
import com.kingsrook.qqq.backend.core.model.actions.messaging.Content;
import com.kingsrook.qqq.backend.core.model.actions.messaging.MultiParty;
import com.kingsrook.qqq.backend.core.model.actions.messaging.Party;
import com.kingsrook.qqq.backend.core.model.actions.messaging.SendMessageInput;
import com.kingsrook.qqq.backend.core.model.actions.messaging.email.EmailContentRole;
import com.kingsrook.qqq.backend.core.model.actions.messaging.email.EmailPartyRole;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import jakarta.mail.Multipart;
import org.junit.jupiter.api.*;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;
import static org.assertj.core.api.Assertions.*;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
/*******************************************************************************
** Email Delivery Test.
@@ -398,6 +421,39 @@ class EmailDeliveryTest
assertThat(H2Support.all(EmailTables.ATTEMPT)).hasSize(1);
}
/*******************************************************************************
** A lost acknowledgement retries the stored identity through the real SMTP sink.
*******************************************************************************/
@Test
void smtpRetryPreservesStoredMessageId() throws Exception
{
var queued = EmailService.send(request().withIdempotencyKey("job:1:row:mail"));
String stored = message(queued.messageId()).getValueString("rfcMessageId");
AtomicInteger calls = new AtomicInteger();
var dispatcher = fake((envelope, settings) ->
{
assertThat(envelope.getRfcMessageId()).isEqualTo(stored);
var sent = new SmtpEmailTransport().send(envelope, settings);
if(calls.incrementAndGet() == 1)
{
throw new EmailTransportException("Lost acknowledgement", false);
}
return (sent);
});
assertThat(dispatcher.sendNow(queued.messageId())).isEqualTo(EmailDispatcher.SendResult.RETRY_WAIT);
EmailRuntime.setClock(Clock.fixed(now.plusSeconds(301), ZoneOffset.UTC));
assertThat(dispatcher.sendNow(queued.messageId())).isEqualTo(EmailDispatcher.SendResult.SENT);
assertThat(mail.waitForIncomingEmail(2000, 2)).isTrue();
assertThat(mail.getReceivedMessages()).hasSize(2);
for(var received : mail.getReceivedMessages())
{
assertThat(received.getMessageID()).isEqualTo(stored);
assertThat(received.getHeader("X-QQQ-Email-Id")).containsExactly(queued.messageId().toString());
}
assertThat(message(queued.messageId()).getValueString("rfcMessageId")).isEqualTo(stored);
assertThat(H2Support.all(EmailTables.ATTEMPT)).hasSize(2);
}
/*******************************************************************************
** Smtp Authentication And Secret Resolution Only At Send.
*******************************************************************************/
@@ -477,8 +533,8 @@ class EmailDeliveryTest
var second = EmailService.send(request());
fake((e, s) ->
{
throw new EmailTransportException("throttle", false).withRetryAfter(Duration.ofHours(2));
})
throw new EmailTransportException("throttle", false).withRetryAfter(Duration.ofHours(2));
})
.runOnce();
assertThat(message(second.messageId()).getValueInstant("nextAttemptDate"))
.isEqualTo(now.plusSeconds(3960));
@@ -494,8 +550,8 @@ class EmailDeliveryTest
assertThat(
fake((e, s) ->
{
throw new EmailTransportException("rejected", true);
})
throw new EmailTransportException("rejected", true);
})
.sendNow(queued.messageId()))
.isEqualTo(EmailDispatcher.SendResult.DEAD_LETTER);
var future = EmailService.send(request());
@@ -30,9 +30,9 @@ import java.util.Set;
import com.kingsrook.qbits.email.app.EmailHome;
import com.kingsrook.qbits.email.metadata.EmailEnums;
import com.kingsrook.qbits.email.metadata.EmailTables;
import com.kingsrook.qbits.email.outbox.EmailCategory;
import com.kingsrook.qbits.email.render.ContrastCheck;
import com.kingsrook.qbits.email.render.EmailLayout;
import com.kingsrook.qbits.email.outbox.EmailCategory;
import com.kingsrook.qbits.email.support.SystemWrites;
import com.kingsrook.qqq.backend.core.actions.dashboard.RenderWidgetAction;
import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
@@ -66,7 +66,9 @@ class EmailHomePolishTest
@BeforeEach
void setup() throws Exception
{
H2Support.setup(c -> { });
H2Support.setup(c ->
{
});
EmailRuntime.setClock(Clock.fixed(NOW, ZoneOffset.UTC));
H2Support.provider(25);
}
@@ -20,14 +20,24 @@
package com.kingsrook.qbits.email;
import java.util.*;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import com.kingsrook.qbits.email.app.EmailApp;
import com.kingsrook.qbits.email.metadata.*;
import com.kingsrook.qbits.email.outbox.*;
import com.kingsrook.qbits.email.permissions.*;
import com.kingsrook.qbits.email.processes.*;
import com.kingsrook.qbits.email.support.*;
import com.kingsrook.qqq.backend.core.actions.tables.*;
import com.kingsrook.qbits.email.metadata.EmailTableCustomizers;
import com.kingsrook.qbits.email.metadata.EmailTables;
import com.kingsrook.qbits.email.outbox.AdHocEmailRenderer;
import com.kingsrook.qbits.email.outbox.EmailRequest;
import com.kingsrook.qbits.email.outbox.EmailSendResult;
import com.kingsrook.qbits.email.outbox.EmailService;
import com.kingsrook.qbits.email.permissions.EmailPermissionChecker;
import com.kingsrook.qbits.email.processes.EmailProcesses;
import com.kingsrook.qbits.email.support.EmailStore;
import com.kingsrook.qbits.email.support.SystemWrites;
import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction;
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
@@ -36,10 +46,14 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.fields.*;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability;
import org.junit.jupiter.api.*;
import static org.assertj.core.api.Assertions.*;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
/*******************************************************************************
** Email Metadata Test.
@@ -412,7 +426,7 @@ class EmailMetadataTest
"MAILGUN",
"GMAIL_API",
"future"))
{
{
assertThat(
new EmailQBitConfig()
.withAllowedEnvNames(Set.of("CUSTOM"))
@@ -20,20 +20,28 @@
package com.kingsrook.qbits.email;
import java.time.*;
import java.util.*;
import com.icegreen.greenmail.util.*;
import com.kingsrook.qbits.email.delivery.*;
import com.kingsrook.qbits.email.metadata.*;
import com.kingsrook.qbits.email.outbox.*;
import com.kingsrook.qbits.email.processes.*;
import com.kingsrook.qbits.email.support.*;
import java.time.Clock;
import java.time.Duration;
import java.time.Instant;
import java.time.ZoneOffset;
import java.util.Map;
import com.icegreen.greenmail.util.GreenMail;
import com.icegreen.greenmail.util.ServerSetup;
import com.kingsrook.qbits.email.metadata.EmailTables;
import com.kingsrook.qbits.email.outbox.EmailRequest;
import com.kingsrook.qbits.email.outbox.EmailService;
import com.kingsrook.qbits.email.processes.EmailProcesses;
import com.kingsrook.qbits.email.support.EmailStore;
import com.kingsrook.qbits.email.support.SystemWrites;
import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.model.actions.processes.*;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import org.junit.jupiter.api.*;
import static org.assertj.core.api.Assertions.*;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
/*******************************************************************************
** Email Processes Test.
@@ -109,7 +109,7 @@ class EmailReviewFixesTest
var schema = new VariableSchema("[{\"name\":\"items\",\"type\":\"LIST\"},{\"name\":\"value\",\"type\":\"STRING\"}]");
var loop = engine.parse("{{#each items}}{{#each items}}{{/each}}{{/each}}", schema);
assertThatThrownBy(() -> loop.render(Map.of("items", Collections.nCopies(100, "")), EmailTemplateEngine.Context.TEXT)).hasMessage("Template output is too large");
assertThatThrownBy(() -> engine.parse("{{value}}", schema).render(Map.of("value", "\u00e9".repeat(600_000)), EmailTemplateEngine.Context.TEXT))
assertThatThrownBy(() -> engine.parse("{{value}}", schema).render(Map.of("value", "é".repeat(600_000)), EmailTemplateEngine.Context.TEXT))
.hasMessage("Template output is too large");
assertThat(engine.parse("{{#if value}}".repeat(8) + "ok" + "{{/if}}".repeat(8), schema).render(Map.of("value", true), EmailTemplateEngine.Context.TEXT)).isEqualTo("ok");
assertThatThrownBy(() -> engine.parse("{{#if value}}".repeat(9) + "{{/if}}".repeat(9), schema)).hasMessageContaining("8 levels");
@@ -179,7 +179,10 @@ class EmailReviewFixesTest
{
assertThatThrownBy(() -> EmailTemplateEngine.validateHtml(source)).hasMessageContaining("must be quoted");
}
EmailRuntime.config().withMjmlCompiler(source -> { throw new AssertionError("Unsafe MJML reached compiler"); });
EmailRuntime.config().withMjmlCompiler(source ->
{
throw new AssertionError("Unsafe MJML reached compiler");
});
QRecord mjml = EmailTemplatesTest.draft("includes", "MJML", "[]", "{}", "<mjml><mj-include path='secret'/></mjml>");
assertThatThrownBy(() -> EmailVersions.activate(mjml.getValueInteger("id"))).hasMessage("MJML includes are not allowed");
assertThat(preview(mjml)).contains("MJML includes are not allowed");
@@ -192,7 +195,10 @@ class EmailReviewFixesTest
void viewerNeverCompiles() throws Exception
{
QRecord version = EmailTemplatesTest.draft("viewer", "MJML", "[]", "{}", "<mjml/>");
EmailRuntime.config().withMjmlCompiler(source -> { throw new AssertionError("Viewer compiled MJML"); });
EmailRuntime.config().withMjmlCompiler(source ->
{
throw new AssertionError("Viewer compiled MJML");
});
QContext.getQSession().setPermissions(Set.of("email.view"));
assertThat(preview(version)).contains("Not compiled yet. Activate the version or use Preview as a manager.");
QContext.getQSession().setPermissions(Set.of("email.manage"));
@@ -270,7 +276,10 @@ class EmailReviewFixesTest
QRecord original = EmailStore.get(EmailTables.MESSAGE, first.messageId(), null);
SystemWrites.updateOne(EmailTables.MESSAGE, new QRecord().withValue("id", second.messageId()).withValue("htmlBody", original.getValue("htmlBody")));
var dispatcher = new com.kingsrook.qbits.email.delivery.EmailDispatcher(new com.kingsrook.qbits.email.transport.EmailTransportRegistry()
.withTransport("SMTP", (envelope, settings) -> { throw new AssertionError("Swapped ciphertext reached transport"); }));
.withTransport("SMTP", (envelope, settings) ->
{
throw new AssertionError("Swapped ciphertext reached transport");
}));
assertThat(dispatcher.sendNow(second.messageId())).isEqualTo(com.kingsrook.qbits.email.delivery.EmailDispatcher.SendResult.RETRY_WAIT);
}
@@ -148,7 +148,10 @@ class EmailTemplatesTest
assertThat(((RawHTML) preview.getWidgetData()).getHtml()).contains("<p>Email</p>");
assertThat(EmailStore.get(EmailTables.VERSION, version.getValueInteger("id"), null).getValue("compiledHtml")).isNull();
EmailVersions.activate(version.getValueInteger("id"));
EmailRuntime.config().withMjmlCompiler(source -> { throw new AssertionError("Compiler ran at send time"); });
EmailRuntime.config().withMjmlCompiler(source ->
{
throw new AssertionError("Compiler ran at send time");
});
var result = EmailService.send(new EmailRequest().withTemplateName("mjml").withLocale("fr").withTo("p@example.com"));
assertThat(result.status()).isEqualTo(EmailSendResult.Status.QUEUED);
QRecord row = EmailStore.get(EmailTables.MESSAGE, result.messageId(), null);
@@ -295,7 +298,10 @@ class EmailTemplatesTest
EmailRuntime.config().withSecretResolver(name -> null);
assertThat(com.kingsrook.qbits.email.render.EmailUnsubscribeLinks.create("1", "address", "NOTIFICATION")).isNull();
assertThatThrownBy(() -> com.kingsrook.qbits.email.render.EmailUnsubscribeLinks.create("1", "address", "MARKETING")).hasMessageContaining("QQQ_EMAIL_UNSUBSCRIBE_KEY");
EmailRuntime.config().withSecretResolver(name -> { throw new IllegalArgumentException("resolver detail"); });
EmailRuntime.config().withSecretResolver(name ->
{
throw new IllegalArgumentException("resolver detail");
});
assertThatThrownBy(() -> com.kingsrook.qbits.email.render.EmailUnsubscribeLinks.create("1", "person@example.com", "NOTIFICATION"))
.hasMessage("Opt-out email needs publicBaseUrl and QQQ_EMAIL_UNSUBSCRIBE_KEY");
}
@@ -85,7 +85,7 @@ public final class H2Support
*******************************************************************************/
public static QInstance instance(RDBMSBackendMetaData backend, Consumer<EmailQBitConfig> tweak)
throws Exception
{
{
QInstance instance = new QInstance();
instance.withInstanceDefaultAuthentication(
new QAuthenticationMetaData()
@@ -66,7 +66,7 @@ public final class ProcessRunner
public static Lines runWithValues(
String process, java.util.Map<String, Serializable> values, Serializable... ids)
throws Exception
{
{
RunProcessInput input = new RunProcessInput();
input.setProcessName(process);
input.addValue("recordsParam", "recordIds");
@@ -0,0 +1,212 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.email.outbox;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.util.HexFormat;
import java.util.List;
import com.kingsrook.qbits.email.H2Support;
import com.kingsrook.qbits.email.metadata.EmailTables;
import com.kingsrook.qbits.email.support.SystemWrites;
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
/*******************************************************************************
** Durable handoff semantics and stored identity using the real H2 outbox.
*******************************************************************************/
class EmailEffectsTest
{
/*******************************************************************************
** Prepare a provider without opening a transport connection.
*******************************************************************************/
@BeforeEach
void setup() throws Exception
{
H2Support.setup(config ->
{
});
H2Support.provider(25);
}
/*******************************************************************************
** Release the per-thread context.
*******************************************************************************/
@AfterEach
void cleanup()
{
QContext.clear();
}
/*******************************************************************************
** A plain message uses the configured sender.
*******************************************************************************/
private EmailRequest request()
{
return (new EmailRequest().withTo("person@example.com").withSubject("Effect").withTextBody("Body"));
}
/*******************************************************************************
** Replays return the same persisted reference without another row.
*******************************************************************************/
@Test
void handoffDeduplicatesAndLookupsReturnCurrentReferences() throws Exception
{
String key = "job:2:row:email";
EmailRequest request = request();
var first = EmailEffects.send(request, key);
var second = EmailEffects.send(request(), key);
assertThat(request.getIdempotencyKey()).isEqualTo(key);
assertThat(first.outcome()).isEqualTo(EmailEffectOutcome.QUEUED_OR_DUPLICATE);
assertThat(second).isEqualTo(first);
assertThat(H2Support.all(EmailTables.MESSAGE)).hasSize(1);
String token = HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(key.getBytes(StandardCharsets.UTF_8))).substring(0, 32);
assertThat(first.message().rfcMessageId()).isEqualTo("<" + token + "@example.com>");
assertThat(first.message().status()).isEqualTo("QUEUED");
assertThat(first.message().providerMessageId()).isNull();
SystemWrites.updateOne(EmailTables.MESSAGE, new QRecord().withValue("id", first.message().id())
.withValue("status", "SENT").withValue("providerMessageId", "provider-reference"));
QContext.init(QContext.getQInstance(), new QSession().withPermissions(List.of()));
var found = EmailEffects.findByIdempotencyKey(key).orElseThrow();
assertThat(found.id()).isEqualTo(first.message().id());
assertThat(found.status()).isEqualTo("SENT");
assertThat(found.providerMessageId()).isEqualTo("provider-reference");
assertThat(EmailEffects.findByRfcMessageId(first.message().rfcMessageId())).contains(found);
assertThat(EmailEffects.findByIdempotencyKey("missing")).isEmpty();
assertThat(EmailEffects.findByRfcMessageId("<missing@example.com>")).isEmpty();
assertThat(EmailEffects.findByIdempotencyKey(null)).isEmpty();
assertThat(EmailEffects.findByRfcMessageId(null)).isEmpty();
assertThat(EmailEffects.findByIdempotencyKey(" ")).isEmpty();
assertThat(EmailEffects.findByRfcMessageId(" ")).isEmpty();
}
/*******************************************************************************
** Conflicting or unusable requests never enqueue or change an existing key.
*******************************************************************************/
@Test
void refusesConflictingKeysAndInvalidRequests() throws Exception
{
EmailRequest conflict = request().withIdempotencyKey("other");
assertThat(EmailEffects.send(conflict, "effect").outcome()).isEqualTo(EmailEffectOutcome.REFUSED);
assertThat(conflict.getIdempotencyKey()).isEqualTo("other");
assertThat(EmailEffects.send(request(), null).outcome()).isEqualTo(EmailEffectOutcome.REFUSED);
assertThat(EmailEffects.send(request(), " ").outcome()).isEqualTo(EmailEffectOutcome.REFUSED);
assertThat(EmailEffects.send(request(), "a".repeat(501)).outcome()).isEqualTo(EmailEffectOutcome.REFUSED);
assertThat(EmailEffects.send(null, "effect").outcome()).isEqualTo(EmailEffectOutcome.REFUSED);
var invalid = EmailEffects.send(request().withSubject(null), "invalid");
assertThat(invalid.outcome()).isEqualTo(EmailEffectOutcome.REFUSED);
assertThat(invalid.message()).isNull();
assertThat(H2Support.all(EmailTables.MESSAGE)).isEmpty();
assertThat(EmailEffects.send(request().withIdempotencyKey("same"), "same").outcome()).isEqualTo(EmailEffectOutcome.QUEUED_OR_DUPLICATE);
}
/*******************************************************************************
** A suppressed audit row stays refused on replay, including duplicate results.
*******************************************************************************/
@Test
void suppressedRecipientsAreRefusedOnEveryHandoff() throws Exception
{
H2Support.insert(EmailTables.SUPPRESSION, new QRecord().withValue("address", "person@example.com")
.withValue("scope", "ALL").withValue("reason", "HARD_BOUNCE"));
var first = EmailEffects.send(request(), "suppressed");
var second = EmailEffects.send(request(), "suppressed");
assertThat(first.outcome()).isEqualTo(EmailEffectOutcome.REFUSED);
assertThat(first.message().status()).isEqualTo("SUPPRESSED");
assertThat(second).isEqualTo(first);
assertThat(H2Support.all(EmailTables.MESSAGE)).hasSize(1);
}
/*******************************************************************************
** Never acknowledge an uncommitted or deferred handoff to the jobs dispatcher.
*******************************************************************************/
@Test
void callerTransactionsAreRefused() throws Exception
{
try(QBackendTransaction transaction = QBackendTransaction.openFor(new InsertInput(EmailTables.MESSAGE)))
{
assertThat(EmailEffects.send(request().withTransaction(transaction, "rdbms"), "same-backend").outcome()).isEqualTo(EmailEffectOutcome.REFUSED);
assertThat(EmailEffects.send(request().withTransaction(transaction, "other"), "other-backend").outcome()).isEqualTo(EmailEffectOutcome.REFUSED);
transaction.commit();
}
assertThat(H2Support.all(EmailTables.MESSAGE)).isEmpty();
}
/*******************************************************************************
** Database uniqueness failures must escape, rather than become terminal refusals.
*******************************************************************************/
@Test
void storageFailurePropagatesAndRfcIdIsUnique() throws Exception
{
var first = EmailEffects.send(request(), "collision");
SystemWrites.updateOne(EmailTables.MESSAGE, new QRecord().withValue("id", first.message().id()).withValue("idempotencyKey", "changed"));
assertThatThrownBy(() -> EmailEffects.send(request(), "collision")).isInstanceOf(QException.class);
assertThat(H2Support.all(EmailTables.MESSAGE)).hasSize(1);
assertThat(EmailEffects.findByIdempotencyKey("collision")).isEmpty();
}
/*******************************************************************************
** Identity hashing is deterministic, ignores display names and has a safe fallback.
*******************************************************************************/
@Test
void generationAndUnkeyedMessages() throws Exception
{
assertThat(EmailService.messageId("key", "Display <sender@EXAMPLE.com>"))
.isEqualTo(EmailService.messageId("key", "sender@example.com"));
for(String from : new String[] {null, "local", "broken <", "sender@[]"})
{
assertThat(EmailService.messageId("key", from)).matches("<[0-9a-f]{32}@qbit-email\\.local>");
}
var first = EmailService.send(request());
var second = EmailService.send(request());
var rows = H2Support.all(EmailTables.MESSAGE);
assertThat(rows).hasSize(2);
assertThat(rows.get(0).getValueString("rfcMessageId")).matches("<[0-9a-f]{32}@example\\.com>")
.isNotEqualTo(rows.get(1).getValueString("rfcMessageId"));
assertThat(first.messageId()).isNotEqualTo(second.messageId());
}
}
@@ -26,8 +26,12 @@ import org.json.JSONObject;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.CsvSource;
import static com.kingsrook.qbits.email.tracking.TrackingTestSupport.*;
import static org.assertj.core.api.Assertions.*;
import static com.kingsrook.qbits.email.tracking.TrackingTestSupport.NOW;
import static com.kingsrook.qbits.email.tracking.TrackingTestSupport.message;
import static com.kingsrook.qbits.email.tracking.TrackingTestSupport.provider;
import static com.kingsrook.qbits.email.tracking.TrackingTestSupport.resource;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
/*******************************************************************************
** Provider-specific normalization contracts use realistic fixture envelopes.
@@ -47,8 +47,12 @@ import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.CsvSource;
import org.junit.jupiter.params.provider.ValueSource;
import static com.kingsrook.qbits.email.tracking.TrackingTestSupport.*;
import static org.assertj.core.api.Assertions.*;
import static com.kingsrook.qbits.email.tracking.TrackingTestSupport.NOW;
import static com.kingsrook.qbits.email.tracking.TrackingTestSupport.SECRET;
import static com.kingsrook.qbits.email.tracking.TrackingTestSupport.message;
import static com.kingsrook.qbits.email.tracking.TrackingTestSupport.provider;
import static com.kingsrook.qbits.email.tracking.TrackingTestSupport.resource;
import static org.assertj.core.api.Assertions.assertThat;
/*******************************************************************************
** Durable processing, rollback, suppression and race acceptance on real H2.
@@ -422,6 +426,7 @@ class EmailInboxProcessorTest
/*******************************************************************************
** Test adapter implementation.
*******************************************************************************/
@Override
public void raise(com.kingsrook.qqq.backend.core.alerts.AlertSignal signal)
{
@@ -32,7 +32,6 @@ import java.time.ZoneOffset;
import java.util.ArrayList;
import java.util.Base64;
import java.util.List;
import java.util.Map;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import com.kingsrook.qbits.email.EmailRuntime;
@@ -52,8 +51,18 @@ import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;
import static com.kingsrook.qbits.email.tracking.TrackingTestSupport.*;
import static org.assertj.core.api.Assertions.*;
import static com.kingsrook.qbits.email.tracking.TrackingTestSupport.NOW;
import static com.kingsrook.qbits.email.tracking.TrackingTestSupport.SECRET;
import static com.kingsrook.qbits.email.tracking.TrackingTestSupport.TOPIC;
import static com.kingsrook.qbits.email.tracking.TrackingTestSupport.certificateTime;
import static com.kingsrook.qbits.email.tracking.TrackingTestSupport.message;
import static com.kingsrook.qbits.email.tracking.TrackingTestSupport.provider;
import static com.kingsrook.qbits.email.tracking.TrackingTestSupport.resource;
import static com.kingsrook.qbits.email.tracking.TrackingTestSupport.response;
import static com.kingsrook.qbits.email.tracking.TrackingTestSupport.signMailgun;
import static com.kingsrook.qbits.email.tracking.TrackingTestSupport.signSns;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
/*******************************************************************************
** Real random-port HTTP exercises the exact route provider and database boundary.
@@ -20,6 +20,9 @@
package com.kingsrook.qbits.email.tracking;
import javax.crypto.Mac;
import javax.crypto.spec.SecretKeySpec;
import javax.net.ssl.SSLSession;
import java.net.URI;
import java.net.http.HttpHeaders;
import java.net.http.HttpRequest;
@@ -36,9 +39,6 @@ import java.util.Base64;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import javax.crypto.Mac;
import javax.crypto.spec.SecretKeySpec;
import javax.net.ssl.SSLSession;
import com.kingsrook.qbits.email.H2Support;
import com.kingsrook.qbits.email.metadata.EmailTables;
import com.kingsrook.qbits.email.support.SystemWrites;
@@ -31,8 +31,16 @@ import org.json.JSONObject;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;
import static com.kingsrook.qbits.email.tracking.TrackingTestSupport.*;
import static org.assertj.core.api.Assertions.*;
import static com.kingsrook.qbits.email.tracking.TrackingTestSupport.NOW;
import static com.kingsrook.qbits.email.tracking.TrackingTestSupport.SECRET;
import static com.kingsrook.qbits.email.tracking.TrackingTestSupport.TOPIC;
import static com.kingsrook.qbits.email.tracking.TrackingTestSupport.certificateTime;
import static com.kingsrook.qbits.email.tracking.TrackingTestSupport.resource;
import static com.kingsrook.qbits.email.tracking.TrackingTestSupport.response;
import static com.kingsrook.qbits.email.tracking.TrackingTestSupport.signMailgun;
import static com.kingsrook.qbits.email.tracking.TrackingTestSupport.signSns;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
/*******************************************************************************
** Crypto boundaries use genuine signatures and a public test-only RSA fixture.
@@ -21,6 +21,7 @@
package com.kingsrook.qbits.email.transport;
import javax.net.ssl.SSLSession;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.net.InetSocketAddress;
@@ -44,7 +45,6 @@ import java.util.Properties;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
import javax.net.ssl.SSLSession;
import com.fasterxml.jackson.databind.JsonNode;
import com.kingsrook.qbits.email.EmailQBitConfig;
import com.kingsrook.qbits.email.EmailRuntime;
@@ -147,9 +147,9 @@ class EmailHttpTransportsTest
*******************************************************************************/
private static EmailEnvelope envelope()
{
return (new EmailEnvelope().withMessageId("42").withFrom("Sender <sender@example.com>")
return (new EmailEnvelope().withMessageId("42").withRfcMessageId("<0123456789abcdef0123456789abcdef@example.com>").withFrom("Sender <sender@example.com>")
.withTo(List.of("Person <to@example.com>")).withCc(List.of("cc@example.com")).withBcc(List.of("bcc@example.com"))
.withReplyTo("Replies <reply@example.com>").withSubject("Hello \u00e9").withTextBody("Plain \u00e9").withHtmlBody("<b>Hello \u00e9</b>")
.withReplyTo("Replies <reply@example.com>").withSubject("Hello é").withTextBody("Plain é").withHtmlBody("<b>Hello é</b>")
.withHeaders(Map.of("List-Unsubscribe", "<https://example.com/unsubscribe>",
"List-Unsubscribe-Post", "List-Unsubscribe=One-Click", "X-Custom", "kept")));
}
@@ -283,6 +283,11 @@ class EmailHttpTransportsTest
assertThat(json.path("MessageStream").asText()).isEqualTo(tracking ? "broadcasts" : "outbound");
assertThat(json.path("TrackOpens").asBoolean()).isEqualTo(tracking);
assertThat(json.path("TrackLinks").asText()).isEqualTo(tracking ? "HtmlAndText" : "None");
assertThat(json.path("Headers")).anySatisfy(header ->
{
assertThat(header.path("Name").asText()).isEqualTo("Message-ID");
assertThat(header.path("Value").asText()).isEqualTo(envelope().getRfcMessageId());
});
assertThat(json.path("Headers").toString()).contains("List-Unsubscribe", "List-Unsubscribe-Post", "X-QQQ-Email-Id", "42", "Auto-Submitted");
assertThat(json.toString()).doesNotContain(SECRET);
}
@@ -303,6 +308,7 @@ class EmailHttpTransportsTest
return (response(202, "", Map.of("x-message-id", List.of("sg-id"))));
});
Map<String, String> headers = new java.util.HashMap<>(envelope().getHeaders());
headers.put("message-id", "<override@example.com>");
for(String reserved : List.of("X-SG-ID", "x-sg-eid", "Received", "DKIM-Signature", "Content-Type",
"Content-Transfer-Encoding", "To", "From", "Subject", "Reply-To", "CC", "BcC"))
{
@@ -329,6 +335,8 @@ class EmailHttpTransportsTest
assertThat(json.at("/content/1/type").asText()).isEqualTo("text/html");
assertThat(json.at("/content/1/value").asText()).isEqualTo(envelope().getHtmlBody());
assertThat(json.at("/custom_args/qqq_email_id").asText()).isEqualTo("42");
assertThat(json.at("/headers/Message-ID").asText()).isEqualTo(envelope().getRfcMessageId());
assertThat(json.path("headers").has("message-id")).isFalse();
assertThat(json.path("headers").toString()).doesNotContain("must-be-removed").contains("List-Unsubscribe", "List-Unsubscribe-Post");
assertThat(json.at("/tracking_settings/open_tracking/enable").asBoolean()).isEqualTo(tracking);
assertThat(json.at("/tracking_settings/click_tracking/enable").asBoolean()).isEqualTo(tracking);
@@ -371,6 +379,8 @@ class EmailHttpTransportsTest
assertThat(mime).doesNotContain("bcc@example.com", "Bcc:", SECRET);
MimeMessage message = new MimeMessage(Session.getInstance(new Properties()), new ByteArrayInputStream(mime.getBytes(StandardCharsets.UTF_8)));
assertThat(message.getSubject()).isEqualTo(envelope().getSubject());
assertThat(message.getMessageID()).isEqualTo(envelope().getRfcMessageId());
assertThat(multipart).contains("name=\"h:Message-Id\"\r\n\r\n" + envelope().getRfcMessageId() + "\r\n");
assertThat(message.getHeader("X-QQQ-Email-Id", null)).isEqualTo("42");
assertThat(message.getHeader("List-Unsubscribe-Post", null)).isEqualTo("List-Unsubscribe=One-Click");
assertThat(message.getRecipients(jakarta.mail.Message.RecipientType.CC)[0].toString()).isEqualTo("cc@example.com");
@@ -47,8 +47,15 @@ import jakarta.mail.internet.InternetAddress;
import jakarta.mail.internet.MimeMessage;
import org.junit.jupiter.api.Test;
import software.amazon.awssdk.services.sesv2.SesV2Client;
import software.amazon.awssdk.services.sesv2.model.*;
import static org.assertj.core.api.Assertions.*;
import software.amazon.awssdk.services.sesv2.model.Message;
import software.amazon.awssdk.services.sesv2.model.MessageRejectedException;
import software.amazon.awssdk.services.sesv2.model.SendEmailRequest;
import software.amazon.awssdk.services.sesv2.model.SendEmailResponse;
import software.amazon.awssdk.services.sesv2.model.SesV2Exception;
import software.amazon.awssdk.services.sesv2.model.TooManyRequestsException;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.assertj.core.api.Assertions.fail;
/*******************************************************************************
** Email Transports Test.
@@ -61,7 +68,7 @@ class EmailTransportsTest
EmailEnvelope envelope()
{
return (new EmailEnvelope()
.withMessageId("123")
.withMessageId("123").withRfcMessageId("<0123456789abcdef0123456789abcdef@example.com>")
.withFrom("sender@example.com")
.withTo(List.of("a@example.com"))
.withBcc(List.of("hidden@example.com"))
@@ -135,10 +142,26 @@ class EmailTransportsTest
Session.getInstance(new Properties()),
new ByteArrayInputStream(captured.get().content().raw().data().asByteArray()));
assertThat(mime.getHeader("Bcc")).isNull();
assertThat(mime.getMessageID()).isEqualTo(envelope().getRfcMessageId());
assertThat(mime.getHeader("X-QQQ-Email-Id")).containsExactly("123");
assertThat(mime.getHeader("X-Test")).containsExactly("yes");
assertThat(mime.getHeader("Auto-Submitted")).containsExactly("auto-generated");
}
/*******************************************************************************
** Repeated JavaMail saves cannot replace the identity or accept a custom override.
*******************************************************************************/
@Test
void mimeIdentitySurvivesSaveChanges() throws Exception
{
var envelope = envelope().withHeaders(Map.of("message-id", "<override@example.com>"));
var message = EmailMime.build(Session.getInstance(new Properties()), envelope);
message.saveChanges();
message.saveChanges();
assertThat(message.getHeader("Message-ID")).containsExactly(envelope.getRfcMessageId());
assertThat(EmailMime.build(Session.getInstance(new Properties()), envelope().withRfcMessageId(null)).getMessageID()).isNotBlank();
}
/*******************************************************************************
** Ses Classifies Throttling Rejection And Server Failure.
*******************************************************************************/
@@ -150,7 +173,7 @@ class EmailTransportsTest
TooManyRequestsException.builder().statusCode(429).message("secret").build(),
MessageRejectedException.builder().statusCode(400).message("secret").build(),
SesV2Exception.builder().statusCode(503).message("secret").build()))
{
{
var transport =
new SesEmailTransport(
builder ->
@@ -313,7 +336,7 @@ class EmailTransportsTest
for(String newline : List.of("\r", "\n"))
{
String bad = "a@example.com" + newline + "Bcc: hidden@example.com";
for(EmailEnvelope envelope : List.of(envelope().withSubject(bad), envelope().withFrom(bad), envelope().withReplyTo(bad),
for(EmailEnvelope envelope : List.of(envelope().withRfcMessageId(bad), envelope().withSubject(bad), envelope().withFrom(bad), envelope().withReplyTo(bad),
envelope().withTo(List.of(bad)), envelope().withCc(List.of(bad)), envelope().withBcc(List.of(bad)),
envelope().withHeaders(Map.of("X-Test", bad)), envelope().withFrom("broken <")))
{