diff --git a/.outpost.yaml.dev b/.outpost.yaml.dev index f4fbf159..53fefddb 100644 --- a/.outpost.yaml.dev +++ b/.outpost.yaml.dev @@ -70,10 +70,9 @@ portal: # ID Generation idgen: type: "nanoid" - event_prefix: "evt_" - destination_prefix: "des_" - delivery_prefix: "dlv_" - delivery_event_prefix: "dev_" + attempt_prefix: "atm" + destination_prefix: "des" + event_prefix: "evt" # Concurrency publish_max_concurrency: 1 diff --git a/Makefile b/Makefile index a8638aad..cbb890fd 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -TEST?=./... +TEST?=./internal/... RUN?= # Build targets diff --git a/cmd/e2e/destwebhook_test.go b/cmd/e2e/destwebhook_test.go index 7d224f00..66c0e22f 100644 --- a/cmd/e2e/destwebhook_test.go +++ b/cmd/e2e/destwebhook_test.go @@ -1467,6 +1467,27 @@ func (suite *basicSuite) TestDeliveryRetry() { // Wait for retry to be scheduled and attempted (poll for at least 2 delivery attempts) suite.waitForMockServerEvents(t, destinationID, 2, 5*time.Second) + // Wait for attempts to be logged, then verify attempt_number increments on automated retry + suite.waitForAttempts(t, "/tenants/"+tenantID+"/attempts", 2, 5*time.Second) + + atmResponse, err := suite.client.Do(suite.AuthRequest(httpclient.Request{ + Method: httpclient.MethodGET, + Path: "/tenants/" + tenantID + "/attempts?dir=asc", + })) + require.NoError(t, err) + require.Equal(t, http.StatusOK, atmResponse.StatusCode) + + atmBody := atmResponse.Body.(map[string]interface{}) + atmModels := atmBody["models"].([]interface{}) + require.GreaterOrEqual(t, len(atmModels), 2, "should have at least 2 attempts from automated retry") + + // Sorted asc by time: attempt_number should increment (0, 1, 2, ...) + for i, m := range atmModels { + attempt := m.(map[string]interface{}) + require.Equal(t, float64(i), attempt["attempt_number"], + "attempt %d should have attempt_number=%d (automated retry increments)", i, i) + } + // Cleanup resp, err = suite.client.Do(suite.AuthRequest(httpclient.Request{ Method: httpclient.MethodDELETE, diff --git a/cmd/e2e/log_test.go b/cmd/e2e/log_test.go index c6c1729a..5cc04ddd 100644 --- a/cmd/e2e/log_test.go +++ b/cmd/e2e/log_test.go @@ -18,14 +18,14 @@ func parseTime(s string) time.Time { return t } -// TestLogAPI tests the Log API endpoints (deliveries, events). +// TestLogAPI tests the Log API endpoints (attempts, events). // // Setup: // 1. Create a tenant and destination // 2. Publish 10 events with small delays for distinct timestamps // // Test Groups: -// - deliveries: list, filter, expand +// - attempts: list, filter, expand // - events: list, filter, retrieve // - sort_order: sort by time ascending/descending // - pagination: paginate through results @@ -111,17 +111,17 @@ func (suite *basicSuite) TestLogAPI() { suite.Require().Equal(http.StatusAccepted, resp.StatusCode, "failed to publish event %d", i) } - // Wait for all deliveries (30s timeout for slow CI environments) - suite.waitForDeliveries(suite.T(), "/tenants/"+tenantID+"/deliveries", 10, 10*time.Second) + // Wait for all attempts (30s timeout for slow CI environments) + suite.waitForAttempts(suite.T(), "/tenants/"+tenantID+"/attempts", 10, 10*time.Second) // ========================================================================= - // Deliveries Tests + // Attempts Tests // ========================================================================= - suite.Run("deliveries", func() { + suite.Run("attempts", func() { suite.Run("list all", func() { resp, err := suite.client.Do(suite.AuthRequest(httpclient.Request{ Method: httpclient.MethodGET, - Path: "/tenants/" + tenantID + "/deliveries", + Path: "/tenants/" + tenantID + "/attempts", })) suite.Require().NoError(err) suite.Require().Equal(http.StatusOK, resp.StatusCode) @@ -137,12 +137,13 @@ func (suite *basicSuite) TestLogAPI() { suite.Equal(destinationID, first["destination"]) suite.NotEmpty(first["status"]) suite.NotEmpty(first["delivered_at"]) + suite.Equal(float64(0), first["attempt_number"], "attempt_number should be present and equal to 0 for first attempt") }) suite.Run("filter by destination_id", func() { resp, err := suite.client.Do(suite.AuthRequest(httpclient.Request{ Method: httpclient.MethodGET, - Path: "/tenants/" + tenantID + "/deliveries?destination_id=" + destinationID, + Path: "/tenants/" + tenantID + "/attempts?destination_id=" + destinationID, })) suite.Require().NoError(err) suite.Require().Equal(http.StatusOK, resp.StatusCode) @@ -155,7 +156,7 @@ func (suite *basicSuite) TestLogAPI() { suite.Run("filter by event_id", func() { resp, err := suite.client.Do(suite.AuthRequest(httpclient.Request{ Method: httpclient.MethodGET, - Path: "/tenants/" + tenantID + "/deliveries?event_id=" + eventIDs[0], + Path: "/tenants/" + tenantID + "/attempts?event_id=" + eventIDs[0], })) suite.Require().NoError(err) suite.Require().Equal(http.StatusOK, resp.StatusCode) @@ -168,7 +169,7 @@ func (suite *basicSuite) TestLogAPI() { suite.Run("include=event returns event object without data", func() { resp, err := suite.client.Do(suite.AuthRequest(httpclient.Request{ Method: httpclient.MethodGET, - Path: "/tenants/" + tenantID + "/deliveries?include=event&limit=1", + Path: "/tenants/" + tenantID + "/attempts?include=event&limit=1", })) suite.Require().NoError(err) suite.Require().Equal(http.StatusOK, resp.StatusCode) @@ -177,8 +178,8 @@ func (suite *basicSuite) TestLogAPI() { models := body["models"].([]interface{}) suite.Require().Len(models, 1) - delivery := models[0].(map[string]interface{}) - event := delivery["event"].(map[string]interface{}) + attempt := models[0].(map[string]interface{}) + event := attempt["event"].(map[string]interface{}) suite.NotEmpty(event["id"]) suite.NotEmpty(event["topic"]) suite.NotEmpty(event["time"]) @@ -188,7 +189,7 @@ func (suite *basicSuite) TestLogAPI() { suite.Run("include=event.data returns event object with data", func() { resp, err := suite.client.Do(suite.AuthRequest(httpclient.Request{ Method: httpclient.MethodGET, - Path: "/tenants/" + tenantID + "/deliveries?include=event.data&limit=1", + Path: "/tenants/" + tenantID + "/attempts?include=event.data&limit=1", })) suite.Require().NoError(err) suite.Require().Equal(http.StatusOK, resp.StatusCode) @@ -197,8 +198,8 @@ func (suite *basicSuite) TestLogAPI() { models := body["models"].([]interface{}) suite.Require().Len(models, 1) - delivery := models[0].(map[string]interface{}) - event := delivery["event"].(map[string]interface{}) + attempt := models[0].(map[string]interface{}) + event := attempt["event"].(map[string]interface{}) suite.NotEmpty(event["id"]) suite.NotNil(event["data"]) // include=event.data SHOULD include data }) @@ -206,7 +207,7 @@ func (suite *basicSuite) TestLogAPI() { suite.Run("include=response_data returns response data", func() { resp, err := suite.client.Do(suite.AuthRequest(httpclient.Request{ Method: httpclient.MethodGET, - Path: "/tenants/" + tenantID + "/deliveries?include=response_data&limit=1", + Path: "/tenants/" + tenantID + "/attempts?include=response_data&limit=1", })) suite.Require().NoError(err) suite.Require().Equal(http.StatusOK, resp.StatusCode) @@ -215,8 +216,8 @@ func (suite *basicSuite) TestLogAPI() { models := body["models"].([]interface{}) suite.Require().Len(models, 1) - delivery := models[0].(map[string]interface{}) - suite.NotNil(delivery["response_data"]) + attempt := models[0].(map[string]interface{}) + suite.NotNil(attempt["response_data"]) }) }) @@ -508,14 +509,14 @@ func (suite *basicSuite) TestLogAPI() { // 2. Configure mock webhook server to FAIL (return 500) // 3. Create a destination pointing to the mock server // 4. Publish an event with eligible_for_retry=false (fails once, no auto-retry) -// 5. Wait for delivery to fail, then fetch the delivery ID +// 5. Wait for attempt to fail, then fetch the attempt ID // 6. Update mock server to SUCCEED (return 200) // // Test Cases: -// - POST /:tenantID/deliveries/:deliveryID/retry - Successful retry returns 202 Accepted -// - POST /:tenantID/deliveries/:deliveryID/retry (non-existent) - Returns 404 -// - Verify retry created new delivery - Event now has 2+ deliveries -// - POST /:tenantID/deliveries/:deliveryID/retry (disabled destination) - Returns 400 +// - POST /:tenantID/attempts/:attemptID/retry - Successful retry returns 202 Accepted +// - POST /:tenantID/attempts/:attemptID/retry (non-existent) - Returns 404 +// - Verify retry created new attempt - Event now has 2+ attempts +// - POST /:tenantID/attempts/:attemptID/retry (disabled destination) - Returns 400 func (suite *basicSuite) TestRetryAPI() { tenantID := idgen.String() destinationID := idgen.Destination() @@ -548,7 +549,7 @@ func (suite *basicSuite) TestRetryAPI() { "url": fmt.Sprintf("%s/webhook/%s", suite.mockServerBaseURL, destinationID), }, "response": map[string]interface{}{ - "status": 500, // Fail deliveries + "status": 500, // Fail attempts }, }, }, @@ -602,22 +603,25 @@ func (suite *basicSuite) TestRetryAPI() { } suite.RunAPITests(suite.T(), setupTests) - // Wait for delivery to complete (and fail) - suite.waitForDeliveries(suite.T(), "/tenants/"+tenantID+"/deliveries?event_id="+eventID, 1, 5*time.Second) + // Wait for attempt to complete (and fail) + suite.waitForAttempts(suite.T(), "/tenants/"+tenantID+"/attempts?event_id="+eventID, 1, 5*time.Second) - // Get the delivery ID - deliveriesResp, err := suite.client.Do(suite.AuthRequest(httpclient.Request{ + // Get the attempt ID + attemptsResp, err := suite.client.Do(suite.AuthRequest(httpclient.Request{ Method: httpclient.MethodGET, - Path: "/tenants/" + tenantID + "/deliveries?event_id=" + eventID, + Path: "/tenants/" + tenantID + "/attempts?event_id=" + eventID, })) suite.Require().NoError(err) - suite.Require().Equal(http.StatusOK, deliveriesResp.StatusCode) + suite.Require().Equal(http.StatusOK, attemptsResp.StatusCode) - body := deliveriesResp.Body.(map[string]interface{}) + body := attemptsResp.Body.(map[string]interface{}) models := body["models"].([]interface{}) - suite.Require().NotEmpty(models, "should have at least one delivery") - firstDelivery := models[0].(map[string]interface{}) - deliveryID := firstDelivery["id"].(string) + suite.Require().NotEmpty(models, "should have at least one attempt") + firstAttempt := models[0].(map[string]interface{}) + attemptID := firstAttempt["id"].(string) + + // Verify first attempt has attempt_number=0 + suite.Equal(float64(0), firstAttempt["attempt_number"], "first attempt should have attempt_number=0") // Update mock to succeed for retry updateMockTests := []APITest{ @@ -649,12 +653,12 @@ func (suite *basicSuite) TestRetryAPI() { // Test retry endpoint retryTests := []APITest{ - // POST /:tenantID/deliveries/:deliveryID/retry - successful retry + // POST /:tenantID/attempts/:attemptID/retry - successful retry { - Name: "POST /:tenantID/deliveries/:deliveryID/retry - retry delivery", + Name: "POST /:tenantID/attempts/:attemptID/retry - retry attempt", Request: suite.AuthRequest(httpclient.Request{ Method: httpclient.MethodPOST, - Path: "/tenants/" + tenantID + "/deliveries/" + deliveryID + "/retry", + Path: "/tenants/" + tenantID + "/attempts/" + attemptID + "/retry", }), Expected: APITestExpectation{ Match: &httpclient.Response{ @@ -665,12 +669,12 @@ func (suite *basicSuite) TestRetryAPI() { }, }, }, - // POST /:tenantID/deliveries/:deliveryID/retry - non-existent delivery + // POST /:tenantID/attempts/:attemptID/retry - non-existent attempt { - Name: "POST /:tenantID/deliveries/:deliveryID/retry - not found", + Name: "POST /:tenantID/attempts/:attemptID/retry - not found", Request: suite.AuthRequest(httpclient.Request{ Method: httpclient.MethodPOST, - Path: "/tenants/" + tenantID + "/deliveries/" + idgen.Delivery() + "/retry", + Path: "/tenants/" + tenantID + "/attempts/" + idgen.Attempt() + "/retry", }), Expected: APITestExpectation{ Match: &httpclient.Response{ @@ -681,37 +685,36 @@ func (suite *basicSuite) TestRetryAPI() { } suite.RunAPITests(suite.T(), retryTests) - // Wait for retry delivery to complete - suite.waitForDeliveries(suite.T(), "/tenants/"+tenantID+"/deliveries?event_id="+eventID, 2, 5*time.Second) + // Wait for retry attempt to complete + suite.waitForAttempts(suite.T(), "/tenants/"+tenantID+"/attempts?event_id="+eventID, 2, 5*time.Second) - // Verify we have more deliveries after retry - verifyTests := []APITest{ - { - Name: "GET /:tenantID/deliveries?event_id=X - verify retry created new delivery", - Request: suite.AuthRequest(httpclient.Request{ - Method: httpclient.MethodGET, - Path: "/tenants/" + tenantID + "/deliveries?event_id=" + eventID, - }), - Expected: APITestExpectation{ - Validate: map[string]interface{}{ - "type": "object", - "properties": map[string]interface{}{ - "statusCode": map[string]interface{}{"const": 200}, - "body": map[string]interface{}{ - "type": "object", - "properties": map[string]interface{}{ - "models": map[string]interface{}{ - "type": "array", - "minItems": 2, // Original + retry - }, - }, - }, - }, - }, - }, - }, + // Verify retry created a new attempt with incremented attempt_number + verifyResp, err := suite.client.Do(suite.AuthRequest(httpclient.Request{ + Method: httpclient.MethodGET, + Path: "/tenants/" + tenantID + "/attempts?event_id=" + eventID + "&dir=asc", + })) + suite.Require().NoError(err) + suite.Require().Equal(http.StatusOK, verifyResp.StatusCode) + + verifyBody := verifyResp.Body.(map[string]interface{}) + verifyModels := verifyBody["models"].([]interface{}) + suite.Require().Len(verifyModels, 2, "should have original + retry attempt") + + // Both attempts should have attempt_number=0 (manual retry resets to 0) + for _, m := range verifyModels { + atm := m.(map[string]interface{}) + suite.Equal(float64(0), atm["attempt_number"], "attempt should have attempt_number=0") + } + + // Verify we have one manual=true (retry) and one manual=false (original) + manualCount := 0 + for _, m := range verifyModels { + atm := m.(map[string]interface{}) + if manual, ok := atm["manual"].(bool); ok && manual { + manualCount++ + } } - suite.RunAPITests(suite.T(), verifyTests) + suite.Equal(1, manualCount, "should have exactly one manual retry attempt") // Test retry on disabled destination disableTests := []APITest{ @@ -728,10 +731,10 @@ func (suite *basicSuite) TestRetryAPI() { }, }, { - Name: "POST /:tenantID/deliveries/:deliveryID/retry - disabled destination", + Name: "POST /:tenantID/attempts/:attemptID/retry - disabled destination", Request: suite.AuthRequest(httpclient.Request{ Method: httpclient.MethodPOST, - Path: "/tenants/" + tenantID + "/deliveries/" + deliveryID + "/retry", + Path: "/tenants/" + tenantID + "/attempts/" + attemptID + "/retry", }), Expected: APITestExpectation{ Match: &httpclient.Response{ @@ -776,311 +779,24 @@ func (suite *basicSuite) TestRetryAPI() { suite.RunAPITests(suite.T(), cleanupTests) } -// TestLegacyLogAPI tests the deprecated legacy endpoints for backward compatibility. -// All legacy endpoints return "Deprecation: true" header to signal migration. -// -// Setup: -// 1. Create a tenant -// 2. Configure mock webhook server to accept deliveries -// 3. Create a destination pointing to the mock server -// 4. Publish an event and wait for delivery to complete -// -// Test Cases: -// - GET /:tenantID/destinations/:destID/events - Legacy list events (returns {data, count}) -// - GET /:tenantID/destinations/:destID/events/:eventID - Legacy retrieve event -// - GET /:tenantID/events/:eventID/deliveries - Legacy list deliveries (returns bare array, not {data}) -// - POST /:tenantID/destinations/:destID/events/:eventID/retry - Legacy retry endpoint -// -// All responses include: -// - Deprecation: true header -// - X-Deprecated-Message header with migration guidance -func (suite *basicSuite) TestLegacyLogAPI() { - tenantID := idgen.String() - destinationID := idgen.Destination() - eventID := idgen.Event() - - // Setup - setupTests := []APITest{ - { - Name: "PUT /:tenantID - create tenant", - Request: suite.AuthRequest(httpclient.Request{ - Method: httpclient.MethodPUT, - Path: "/tenants/" + tenantID, - }), - Expected: APITestExpectation{ - Match: &httpclient.Response{ - StatusCode: http.StatusCreated, - }, - }, - }, - { - Name: "PUT mockserver/destinations - setup mock", - Request: httpclient.Request{ - Method: httpclient.MethodPUT, - BaseURL: suite.mockServerBaseURL, - Path: "/destinations", - Body: map[string]interface{}{ - "id": destinationID, - "type": "webhook", - "config": map[string]interface{}{ - "url": fmt.Sprintf("%s/webhook/%s", suite.mockServerBaseURL, destinationID), - }, - }, - }, - Expected: APITestExpectation{ - Match: &httpclient.Response{ - StatusCode: http.StatusOK, - }, - }, - }, - { - Name: "POST /:tenantID/destinations - create destination", - Request: suite.AuthRequest(httpclient.Request{ - Method: httpclient.MethodPOST, - Path: "/tenants/" + tenantID + "/destinations", - Body: map[string]interface{}{ - "id": destinationID, - "type": "webhook", - "topics": "*", - "config": map[string]interface{}{ - "url": fmt.Sprintf("%s/webhook/%s", suite.mockServerBaseURL, destinationID), - }, - }, - }), - Expected: APITestExpectation{ - Match: &httpclient.Response{ - StatusCode: http.StatusCreated, - }, - }, - }, - { - Name: "POST /publish - publish event", - Request: suite.AuthRequest(httpclient.Request{ - Method: httpclient.MethodPOST, - Path: "/publish", - Body: map[string]interface{}{ - "id": eventID, - "tenant_id": tenantID, - "topic": "user.created", - "eligible_for_retry": true, - "data": map[string]interface{}{ - "user_id": "789", - }, - }, - }), - Expected: APITestExpectation{ - Match: &httpclient.Response{ - StatusCode: http.StatusAccepted, - }, - }, - }, - } - suite.RunAPITests(suite.T(), setupTests) - - // Wait for delivery - suite.waitForDeliveries(suite.T(), "/tenants/"+tenantID+"/deliveries", 1, 5*time.Second) - - // Test legacy endpoints - all should return deprecation headers - legacyTests := []APITest{ - // GET /:tenantID/destinations/:destinationID/events - legacy list events by destination - { - Name: "GET /:tenantID/destinations/:destinationID/events - legacy endpoint", - Request: suite.AuthRequest(httpclient.Request{ - Method: httpclient.MethodGET, - Path: "/tenants/" + tenantID + "/destinations/" + destinationID + "/events", - }), - Expected: APITestExpectation{ - Validate: map[string]interface{}{ - "type": "object", - "properties": map[string]interface{}{ - "statusCode": map[string]interface{}{"const": 200}, - "headers": map[string]interface{}{ - "type": "object", - "properties": map[string]interface{}{ - "Deprecation": map[string]interface{}{ - "type": "array", - "items": map[string]interface{}{ - "const": "true", - }, - }, - }, - }, - "body": map[string]interface{}{ - "type": "object", - "required": []interface{}{"data", "count"}, - "properties": map[string]interface{}{ - "data": map[string]interface{}{ - "type": "array", - "minItems": 1, - }, - "count": map[string]interface{}{"type": "number"}, - }, - }, - }, - }, - }, - }, - // GET /:tenantID/destinations/:destinationID/events/:eventID - legacy retrieve event - { - Name: "GET /:tenantID/destinations/:destinationID/events/:eventID - legacy endpoint", - Request: suite.AuthRequest(httpclient.Request{ - Method: httpclient.MethodGET, - Path: "/tenants/" + tenantID + "/destinations/" + destinationID + "/events/" + eventID, - }), - Expected: APITestExpectation{ - Validate: map[string]interface{}{ - "type": "object", - "properties": map[string]interface{}{ - "statusCode": map[string]interface{}{"const": 200}, - "headers": map[string]interface{}{ - "type": "object", - "properties": map[string]interface{}{ - "Deprecation": map[string]interface{}{ - "type": "array", - "items": map[string]interface{}{ - "const": "true", - }, - }, - }, - }, - "body": map[string]interface{}{ - "type": "object", - "required": []interface{}{"id", "topic"}, - "properties": map[string]interface{}{ - "id": map[string]interface{}{"const": eventID}, - "topic": map[string]interface{}{"const": "user.created"}, - }, - }, - }, - }, - }, - }, - // GET /:tenantID/events/:eventID/deliveries - legacy list deliveries by event - { - Name: "GET /:tenantID/events/:eventID/deliveries - legacy endpoint (returns bare array)", - Request: suite.AuthRequest(httpclient.Request{ - Method: httpclient.MethodGET, - Path: "/tenants/" + tenantID + "/events/" + eventID + "/deliveries", - }), - Expected: APITestExpectation{ - Validate: map[string]interface{}{ - "type": "object", - "properties": map[string]interface{}{ - "statusCode": map[string]interface{}{"const": 200}, - "headers": map[string]interface{}{ - "type": "object", - "properties": map[string]interface{}{ - "Deprecation": map[string]interface{}{ - "type": "array", - "items": map[string]interface{}{ - "const": "true", - }, - }, - }, - }, - // Legacy endpoint returns bare array, not {data: [...]} - "body": map[string]interface{}{ - "type": "array", - "minItems": 1, - "items": map[string]interface{}{ - "type": "object", - "required": []interface{}{"id", "status", "delivered_at"}, - "properties": map[string]interface{}{ - "id": map[string]interface{}{"type": "string"}, - "status": map[string]interface{}{"type": "string"}, - "delivered_at": map[string]interface{}{"type": "string"}, - }, - }, - }, - }, - }, - }, - }, - // POST /:tenantID/destinations/:destinationID/events/:eventID/retry - legacy retry - { - Name: "POST /:tenantID/destinations/:destinationID/events/:eventID/retry - legacy endpoint", - Request: suite.AuthRequest(httpclient.Request{ - Method: httpclient.MethodPOST, - Path: "/tenants/" + tenantID + "/destinations/" + destinationID + "/events/" + eventID + "/retry", - }), - Expected: APITestExpectation{ - Validate: map[string]interface{}{ - "type": "object", - "properties": map[string]interface{}{ - "statusCode": map[string]interface{}{"const": 202}, - "headers": map[string]interface{}{ - "type": "object", - "properties": map[string]interface{}{ - "Deprecation": map[string]interface{}{ - "type": "array", - "items": map[string]interface{}{ - "const": "true", - }, - }, - }, - }, - "body": map[string]interface{}{ - "type": "object", - "properties": map[string]interface{}{ - "success": map[string]interface{}{"const": true}, - }, - }, - }, - }, - }, - }, - } - suite.RunAPITests(suite.T(), legacyTests) - - // Cleanup - cleanupTests := []APITest{ - { - Name: "DELETE mockserver/destinations/:destinationID", - Request: httpclient.Request{ - Method: httpclient.MethodDELETE, - BaseURL: suite.mockServerBaseURL, - Path: "/destinations/" + destinationID, - }, - Expected: APITestExpectation{ - Match: &httpclient.Response{ - StatusCode: http.StatusOK, - }, - }, - }, - { - Name: "DELETE /:tenantID", - Request: suite.AuthRequest(httpclient.Request{ - Method: httpclient.MethodDELETE, - Path: "/tenants/" + tenantID, - }), - Expected: APITestExpectation{ - Match: &httpclient.Response{ - StatusCode: http.StatusOK, - }, - }, - }, - } - suite.RunAPITests(suite.T(), cleanupTests) -} - -// TestAdminLogEndpoints tests the admin-only /events and /deliveries endpoints. +// TestAdminLogEndpoints tests the admin-only /events and /attempts endpoints. // // These endpoints allow cross-tenant queries with optional tenant_id filter. // // Setup: // 1. Create two tenants with destinations // 2. Publish events to each tenant -// 3. Wait for deliveries to complete +// 3. Wait for attempts to complete // // Test Cases: // - GET /events without auth returns 401 -// - GET /deliveries without auth returns 401 +// - GET /attempts without auth returns 401 // - GET /events with JWT returns 401 (admin-only) -// - GET /deliveries with JWT returns 401 (admin-only) +// - GET /attempts with JWT returns 401 (admin-only) // - GET /events with admin key returns all events (cross-tenant) -// - GET /deliveries with admin key returns all deliveries (cross-tenant) +// - GET /attempts with admin key returns all attempts (cross-tenant) // - GET /events?tenant_id=X filters to single tenant -// - GET /deliveries?tenant_id=X filters to single tenant +// - GET /attempts?tenant_id=X filters to single tenant func (suite *basicSuite) TestAdminLogEndpoints() { tenant1ID := idgen.String() tenant2ID := idgen.String() @@ -1218,9 +934,9 @@ func (suite *basicSuite) TestAdminLogEndpoints() { } suite.RunAPITests(suite.T(), setupTests) - // Wait for deliveries for both tenants - suite.waitForDeliveries(suite.T(), "/tenants/"+tenant1ID+"/deliveries", 1, 5*time.Second) - suite.waitForDeliveries(suite.T(), "/tenants/"+tenant2ID+"/deliveries", 1, 5*time.Second) + // Wait for attempts for both tenants + suite.waitForAttempts(suite.T(), "/tenants/"+tenant1ID+"/attempts", 1, 5*time.Second) + suite.waitForAttempts(suite.T(), "/tenants/"+tenant2ID+"/attempts", 1, 5*time.Second) // Get JWT token for tenant1 to test that JWT auth is rejected on admin endpoints tokenResp, err := suite.client.Do(suite.AuthRequest(httpclient.Request{ @@ -1246,10 +962,10 @@ func (suite *basicSuite) TestAdminLogEndpoints() { suite.Equal(http.StatusUnauthorized, resp.StatusCode) }) - suite.Run("GET /deliveries without auth returns 401", func() { + suite.Run("GET /attempts without auth returns 401", func() { resp, err := suite.client.Do(httpclient.Request{ Method: httpclient.MethodGET, - Path: "/deliveries", + Path: "/attempts", }) suite.Require().NoError(err) suite.Equal(http.StatusUnauthorized, resp.StatusCode) @@ -1264,10 +980,10 @@ func (suite *basicSuite) TestAdminLogEndpoints() { suite.Equal(http.StatusUnauthorized, resp.StatusCode) }) - suite.Run("GET /deliveries with JWT returns 401 (admin-only)", func() { + suite.Run("GET /attempts with JWT returns 401 (admin-only)", func() { resp, err := suite.client.Do(suite.AuthJWTRequest(httpclient.Request{ Method: httpclient.MethodGET, - Path: "/deliveries", + Path: "/attempts", }, jwtToken)) suite.Require().NoError(err) suite.Equal(http.StatusUnauthorized, resp.StatusCode) @@ -1303,31 +1019,31 @@ func (suite *basicSuite) TestAdminLogEndpoints() { suite.True(eventsSeen[event2ID], "should include tenant2 event") }) - suite.Run("GET /deliveries returns deliveries from all tenants", func() { + suite.Run("GET /attempts returns attempts from all tenants", func() { resp, err := suite.client.Do(suite.AuthRequest(httpclient.Request{ Method: httpclient.MethodGET, - Path: "/deliveries?include=event", + Path: "/attempts?include=event", })) suite.Require().NoError(err) suite.Require().Equal(http.StatusOK, resp.StatusCode) body := resp.Body.(map[string]interface{}) models := body["models"].([]interface{}) - // Should have at least 2 deliveries (one from each tenant we created) + // Should have at least 2 attempts (one from each tenant we created) suite.GreaterOrEqual(len(models), 2) - // Verify we have deliveries from both tenants by checking event IDs + // Verify we have attempts from both tenants by checking event IDs eventsSeen := map[string]bool{} for _, item := range models { - delivery := item.(map[string]interface{}) - if event, ok := delivery["event"].(map[string]interface{}); ok { + attempt := item.(map[string]interface{}) + if event, ok := attempt["event"].(map[string]interface{}); ok { if id, ok := event["id"].(string); ok { eventsSeen[id] = true } } } - suite.True(eventsSeen[event1ID], "should include tenant1 delivery") - suite.True(eventsSeen[event2ID], "should include tenant2 delivery") + suite.True(eventsSeen[event1ID], "should include tenant1 attempt") + suite.True(eventsSeen[event2ID], "should include tenant2 attempt") }) }) @@ -1352,10 +1068,10 @@ func (suite *basicSuite) TestAdminLogEndpoints() { suite.Equal(event1ID, event["id"]) }) - suite.Run("GET /deliveries?tenant_id=X filters to single tenant", func() { + suite.Run("GET /attempts?tenant_id=X filters to single tenant", func() { resp, err := suite.client.Do(suite.AuthRequest(httpclient.Request{ Method: httpclient.MethodGET, - Path: "/deliveries?tenant_id=" + tenant2ID + "&include=event", + Path: "/attempts?tenant_id=" + tenant2ID + "&include=event", })) suite.Require().NoError(err) suite.Require().Equal(http.StatusOK, resp.StatusCode) @@ -1364,9 +1080,9 @@ func (suite *basicSuite) TestAdminLogEndpoints() { models := body["models"].([]interface{}) suite.Len(models, 1) - // Verify only tenant2 delivery by event ID - delivery := models[0].(map[string]interface{}) - event := delivery["event"].(map[string]interface{}) + // Verify only tenant2 attempt by event ID + attempt := models[0].(map[string]interface{}) + event := attempt["event"].(map[string]interface{}) suite.Equal(event2ID, event["id"]) }) }) diff --git a/cmd/e2e/suites_test.go b/cmd/e2e/suites_test.go index baf5bc3a..f2531e19 100644 --- a/cmd/e2e/suites_test.go +++ b/cmd/e2e/suites_test.go @@ -41,8 +41,8 @@ func waitForHealthy(t *testing.T, port int, timeout time.Duration) { t.Fatalf("timed out waiting for health check at %s", healthURL) } -// waitForDeliveries polls until at least minCount deliveries exist for the given path. -func (s *e2eSuite) waitForDeliveries(t *testing.T, path string, minCount int, timeout time.Duration) { +// waitForAttempts polls until at least minCount attempts exist for the given path. +func (s *e2eSuite) waitForAttempts(t *testing.T, path string, minCount int, timeout time.Duration) { t.Helper() deadline := time.Now().Add(timeout) var lastCount int @@ -72,9 +72,9 @@ func (s *e2eSuite) waitForDeliveries(t *testing.T, path string, minCount int, ti time.Sleep(100 * time.Millisecond) } if lastErr != nil { - t.Fatalf("timed out waiting for %d deliveries at %s: last error: %v", minCount, path, lastErr) + t.Fatalf("timed out waiting for %d attempts at %s: last error: %v", minCount, path, lastErr) } - t.Fatalf("timed out waiting for %d deliveries at %s: got %d (status %d)", minCount, path, lastCount, lastStatus) + t.Fatalf("timed out waiting for %d attempts at %s: got %d (status %d)", minCount, path, lastCount, lastStatus) } // waitForDestinationDisabled polls until the destination has disabled_at set (non-null). @@ -544,3 +544,166 @@ func TestE2E_Regression_AutoDisableWithoutCallbackURL(t *testing.T) { // Cleanup mock server _ = mockServerInfra } + +// TestE2E_Regression_RetryRaceCondition verifies that retries are not lost when +// the retry scheduler queries logstore before the event has been persisted. +// +// Test configuration creates a timing window where retry fires before log persistence: +// - LogBatchThresholdSeconds = 5 (slow persistence) +// - RetryIntervalSeconds = 1 (fast retry) +// - RetryVisibilityTimeoutSeconds = 2 (quick reprocessing when event not found) +// +// Expected behavior: retry remains in queue until event is available, then succeeds. +func TestE2E_Regression_RetryRaceCondition(t *testing.T) { + t.Parallel() + if testing.Short() { + t.Skip("skipping e2e test") + } + + // Setup infrastructure + testinfraCleanup := testinfra.Start(t) + defer testinfraCleanup() + gin.SetMode(gin.TestMode) + mockServerBaseURL := testinfra.GetMockServer(t) + + // Configure with slow log persistence and fast retry + cfg := configs.Basic(t, configs.BasicOpts{ + LogStorage: configs.LogStorageTypeClickHouse, + }) + + // SLOW log persistence: batch won't flush for 5 seconds + cfg.LogBatchThresholdSeconds = 5 + cfg.LogBatchSize = 10000 // High batch size to prevent early flush + + // FAST retry: retry fires after ~1 second + cfg.RetryIntervalSeconds = 1 + cfg.RetryPollBackoffMs = 50 + cfg.RetryMaxLimit = 5 + cfg.RetryVisibilityTimeoutSeconds = 2 // Short VT so retry happens quickly after event not found + + require.NoError(t, cfg.Validate(config.Flags{})) + + // Start application + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + appDone := make(chan struct{}) + go func() { + defer close(appDone) + application := app.New(&cfg) + if err := application.Run(ctx); err != nil { + log.Println("Application stopped:", err) + } + }() + defer func() { + cancel() + <-appDone + }() + + // Wait for services to start + waitForHealthy(t, cfg.APIPort, 5*time.Second) + + // Setup test client + client := httpclient.New(fmt.Sprintf("http://localhost:%d/api/v1", cfg.APIPort), cfg.APIKey) + mockServerInfra := testinfra.NewMockServerInfra(mockServerBaseURL) + + // Test data + tenantID := fmt.Sprintf("tenant_race_%d", time.Now().UnixNano()) + destinationID := fmt.Sprintf("dest_race_%d", time.Now().UnixNano()) + secret := "testsecret1234567890abcdefghijklmnop" + + // Create tenant + resp, err := client.Do(httpclient.Request{ + Method: httpclient.MethodPUT, + Path: "/tenants/" + tenantID, + Headers: map[string]string{"Authorization": "Bearer " + cfg.APIKey}, + }) + require.NoError(t, err) + require.Equal(t, 201, resp.StatusCode, "failed to create tenant") + + // Configure mock server destination + resp, err = client.Do(httpclient.Request{ + Method: httpclient.MethodPUT, + BaseURL: mockServerBaseURL, + Path: "/destinations", + Body: map[string]interface{}{ + "id": destinationID, + "type": "webhook", + "config": map[string]interface{}{ + "url": fmt.Sprintf("%s/webhook/%s", mockServerBaseURL, destinationID), + }, + "credentials": map[string]interface{}{ + "secret": secret, + }, + }, + }) + require.NoError(t, err) + require.Equal(t, 200, resp.StatusCode, "failed to configure mock server") + + // Create destination + resp, err = client.Do(httpclient.Request{ + Method: httpclient.MethodPOST, + Path: "/tenants/" + tenantID + "/destinations", + Headers: map[string]string{"Authorization": "Bearer " + cfg.APIKey}, + Body: map[string]interface{}{ + "id": destinationID, + "type": "webhook", + "topics": "*", + "config": map[string]interface{}{ + "url": fmt.Sprintf("%s/webhook/%s", mockServerBaseURL, destinationID), + }, + "credentials": map[string]interface{}{ + "secret": secret, + }, + }, + }) + require.NoError(t, err) + require.Equal(t, 201, resp.StatusCode, "failed to create destination") + + // Publish event that will always fail (should_err: true) + // We want to verify that retries happen (mock server is hit multiple times) + resp, err = client.Do(httpclient.Request{ + Method: httpclient.MethodPOST, + Path: "/publish", + Headers: map[string]string{"Authorization": "Bearer " + cfg.APIKey}, + Body: map[string]interface{}{ + "tenant_id": tenantID, + "topic": "user.created", + "eligible_for_retry": true, + "metadata": map[string]interface{}{ + "should_err": "true", // All deliveries fail + }, + "data": map[string]interface{}{ + "test": "race-condition-test", + }, + }, + }) + require.NoError(t, err) + require.Equal(t, 202, resp.StatusCode, "failed to publish event") + + // Wait for retries to complete + // - t=0: Event published, first delivery fails + // - t=1s: Retry fires, event not in logstore yet, message returns to queue + // - t=3s: Message visible again after 2s VT, retry fires again + // - t=5s: Log batch flushes, event now in logstore + // - t=5s+: Retry finds event, delivery succeeds + time.Sleep(10 * time.Second) + + // Verify mock server received multiple delivery attempts + resp, err = client.Do(httpclient.Request{ + Method: httpclient.MethodGET, + BaseURL: mockServerBaseURL, + Path: "/destinations/" + destinationID + "/events", + }) + require.NoError(t, err) + require.Equal(t, 200, resp.StatusCode) + + events, ok := resp.Body.([]interface{}) + require.True(t, ok, "expected events array") + + // Should have at least 2 attempts: initial failure + successful retry + require.GreaterOrEqual(t, len(events), 2, + "expected multiple delivery attempts (initial + retry after event persisted)") + + _ = mockServerInfra +} diff --git a/cmd/publish/publish_http.go b/cmd/publish/publish_http.go index 71d13137..40a70745 100644 --- a/cmd/publish/publish_http.go +++ b/cmd/publish/publish_http.go @@ -16,14 +16,11 @@ const ( func publishHTTP(body map[string]interface{}) error { log.Printf("[x] Publishing HTTP") - // make HTTP POST request to the URL specified in the body - jsonData, err := json.Marshal(body) if err != nil { return fmt.Errorf("failed to marshal body to JSON: %w", err) } - // Make HTTP POST request req, err := http.NewRequest("POST", ServerURL, bytes.NewBuffer(jsonData)) if err != nil { return fmt.Errorf("failed to create HTTP request: %w", err) @@ -37,7 +34,6 @@ func publishHTTP(body map[string]interface{}) error { } defer resp.Body.Close() - // Check for non-200 status code if resp.StatusCode != http.StatusOK { return fmt.Errorf("received non-200 response: %d", resp.StatusCode) } diff --git a/docs/apis/openapi.yaml b/docs/apis/openapi.yaml index 39760b65..34933f17 100644 --- a/docs/apis/openapi.yaml +++ b/docs/apis/openapi.yaml @@ -475,7 +475,7 @@ components: additionalProperties: type: string nullable: true - description: Static key-value pairs merged into event metadata on every delivery. + description: Static key-value pairs merged into event metadata on every attempt. example: { "app-id": "my-app", "region": "us-east-1" } metadata: type: object @@ -562,7 +562,7 @@ components: additionalProperties: type: string nullable: true - description: Static key-value pairs merged into event metadata on every delivery. + description: Static key-value pairs merged into event metadata on every attempt. example: { "app-id": "my-app", "region": "us-east-1" } metadata: type: object @@ -649,7 +649,7 @@ components: additionalProperties: type: string nullable: true - description: Static key-value pairs merged into event metadata on every delivery. + description: Static key-value pairs merged into event metadata on every attempt. example: { "app-id": "my-app", "region": "us-east-1" } metadata: type: object @@ -726,7 +726,7 @@ components: additionalProperties: type: string nullable: true - description: Static key-value pairs merged into event metadata on every delivery. + description: Static key-value pairs merged into event metadata on every attempt. example: { "app-id": "my-app", "region": "us-east-1" } metadata: type: object @@ -810,7 +810,7 @@ components: additionalProperties: type: string nullable: true - description: Static key-value pairs merged into event metadata on every delivery. + description: Static key-value pairs merged into event metadata on every attempt. example: { "app-id": "my-app", "region": "us-east-1" } metadata: type: object @@ -887,7 +887,7 @@ components: additionalProperties: type: string nullable: true - description: Static key-value pairs merged into event metadata on every delivery. + description: Static key-value pairs merged into event metadata on every attempt. example: { "app-id": "my-app", "region": "us-east-1" } metadata: type: object @@ -973,7 +973,7 @@ components: additionalProperties: type: string nullable: true - description: Static key-value pairs merged into event metadata on every delivery. + description: Static key-value pairs merged into event metadata on every attempt. example: { "app-id": "my-app", "region": "us-east-1" } metadata: type: object @@ -1060,7 +1060,7 @@ components: additionalProperties: type: string nullable: true - description: Static key-value pairs merged into event metadata on every delivery. + description: Static key-value pairs merged into event metadata on every attempt. example: { "app-id": "my-app", "region": "us-east-1" } metadata: type: object @@ -1143,7 +1143,7 @@ components: additionalProperties: type: string nullable: true - description: Static key-value pairs merged into event metadata on every delivery. + description: Static key-value pairs merged into event metadata on every attempt. example: { "app-id": "my-app", "region": "us-east-1" } metadata: type: object @@ -1177,7 +1177,7 @@ components: additionalProperties: type: string nullable: true - description: Static key-value pairs merged into event metadata on every delivery. + description: Static key-value pairs merged into event metadata on every attempt. example: { "app-id": "my-app", "region": "us-east-1" } metadata: type: object @@ -1211,7 +1211,7 @@ components: additionalProperties: type: string nullable: true - description: Static key-value pairs merged into event metadata on every delivery. + description: Static key-value pairs merged into event metadata on every attempt. example: { "app-id": "my-app", "region": "us-east-1" } metadata: type: object @@ -1244,7 +1244,7 @@ components: additionalProperties: type: string nullable: true - description: Static key-value pairs merged into event metadata on every delivery. + description: Static key-value pairs merged into event metadata on every attempt. example: { "app-id": "my-app", "region": "us-east-1" } metadata: type: object @@ -1278,7 +1278,7 @@ components: additionalProperties: type: string nullable: true - description: Static key-value pairs merged into event metadata on every delivery. + description: Static key-value pairs merged into event metadata on every attempt. example: { "app-id": "my-app", "region": "us-east-1" } metadata: type: object @@ -1313,7 +1313,7 @@ components: additionalProperties: type: string nullable: true - description: Static key-value pairs merged into event metadata on every delivery. + description: Static key-value pairs merged into event metadata on every attempt. example: { "app-id": "my-app", "region": "us-east-1" } metadata: type: object @@ -1348,7 +1348,7 @@ components: additionalProperties: type: string nullable: true - description: Static key-value pairs merged into event metadata on every delivery. + description: Static key-value pairs merged into event metadata on every attempt. example: { "app-id": "my-app", "region": "us-east-1" } metadata: type: object @@ -1382,7 +1382,7 @@ components: additionalProperties: type: string nullable: true - description: Static key-value pairs merged into event metadata on every delivery. + description: Static key-value pairs merged into event metadata on every attempt. example: { "app-id": "my-app", "region": "us-east-1" } metadata: type: object @@ -1449,7 +1449,7 @@ components: additionalProperties: type: string nullable: true - description: Static key-value pairs merged into event metadata on every delivery. + description: Static key-value pairs merged into event metadata on every attempt. example: { "app-id": "my-app", "region": "us-east-1" } metadata: type: object @@ -1475,7 +1475,7 @@ components: additionalProperties: type: string nullable: true - description: Static key-value pairs merged into event metadata on every delivery. + description: Static key-value pairs merged into event metadata on every attempt. example: { "app-id": "my-app", "region": "us-east-1" } metadata: type: object @@ -1501,7 +1501,7 @@ components: additionalProperties: type: string nullable: true - description: Static key-value pairs merged into event metadata on every delivery. + description: Static key-value pairs merged into event metadata on every attempt. example: { "app-id": "my-app", "region": "us-east-1" } metadata: type: object @@ -1526,7 +1526,7 @@ components: additionalProperties: type: string nullable: true - description: Static key-value pairs merged into event metadata on every delivery. + description: Static key-value pairs merged into event metadata on every attempt. example: { "app-id": "my-app", "region": "us-east-1" } metadata: type: object @@ -1552,7 +1552,7 @@ components: additionalProperties: type: string nullable: true - description: Static key-value pairs merged into event metadata on every delivery. + description: Static key-value pairs merged into event metadata on every attempt. example: { "app-id": "my-app", "region": "us-east-1" } metadata: type: object @@ -1578,7 +1578,7 @@ components: additionalProperties: type: string nullable: true - description: Static key-value pairs merged into event metadata on every delivery. + description: Static key-value pairs merged into event metadata on every attempt. example: { "app-id": "my-app", "region": "us-east-1" } metadata: type: object @@ -1605,7 +1605,7 @@ components: additionalProperties: type: string nullable: true - description: Static key-value pairs merged into event metadata on every delivery. + description: Static key-value pairs merged into event metadata on every attempt. example: { "app-id": "my-app", "region": "us-east-1" } metadata: type: object @@ -1631,7 +1631,7 @@ components: additionalProperties: type: string nullable: true - description: Static key-value pairs merged into event metadata on every delivery. + description: Static key-value pairs merged into event metadata on every attempt. example: { "app-id": "my-app", "region": "us-east-1" } metadata: type: object @@ -1769,24 +1769,24 @@ components: type: string example: { "content-type": "application/json" } - # Delivery schemas for deliveries-first API - Delivery: + # Attempt schemas for attempts-first API + Attempt: type: object - description: A delivery represents a single delivery attempt of an event to a destination. + description: An attempt represents a single delivery attempt of an event to a destination. properties: id: type: string - description: Unique identifier for this delivery. - example: "del_123" + description: Unique identifier for this attempt. + example: "atm_123" status: type: string enum: [success, failed] - description: The delivery status. + description: The attempt status. example: "success" delivered_at: type: string format: date-time - description: Time the delivery was attempted. + description: Time the attempt was made. example: "2024-01-01T00:00:05Z" code: type: string @@ -1794,16 +1794,16 @@ components: example: "200" response_data: type: object - description: Response data from the delivery attempt. Only included when include=response_data. + description: Response data from the attempt. Only included when include=response_data. additionalProperties: true example: { "status_code": 200, "body": '{"status":"ok"}', "headers": { "content-type": "application/json" } } - attempt: + attempt_number: type: integer description: The attempt number (1 for first attempt, 2+ for retries). example: 1 manual: type: boolean - description: Whether this delivery was manually triggered (e.g., a retry initiated by a user). + description: Whether this attempt was manually triggered (e.g., a retry initiated by a user). example: false event: oneOf: @@ -1815,7 +1815,7 @@ components: description: The associated event. Returns event ID by default, or included event object when include=event or include=event.data. destination: type: string - description: The destination ID this delivery was sent to. + description: The destination ID this attempt was sent to. example: "des_456" EventSummary: type: object @@ -1872,15 +1872,15 @@ components: additionalProperties: true description: The event payload data. example: { "user_id": "userid", "status": "active" } - DeliveryPaginatedResult: + AttemptPaginatedResult: type: object - description: Paginated list of deliveries. + description: Paginated list of attempts. properties: models: type: array items: - $ref: "#/components/schemas/Delivery" - description: Array of delivery objects. + $ref: "#/components/schemas/Attempt" + description: Array of attempt objects. pagination: $ref: "#/components/schemas/SeekPagination" @@ -2028,14 +2028,14 @@ tags: description: Operations for retrieving destination type schemas. - name: Topics description: Operations for retrieving available event topics. - - name: Deliveries + - name: Attempts description: | - Deliveries represent individual delivery attempts of events to destinations. The deliveries API provides a delivery-centric view of event processing. + Attempts represent individual delivery attempts of events to destinations. The attempts API provides an attempt-centric view of event processing. - Each delivery contains: - - `id`: Unique delivery identifier + Each attempt contains: + - `id`: Unique attempt identifier - `status`: success or failed - - `delivered_at`: Timestamp of the delivery attempt + - `delivered_at`: Timestamp of the attempt - `code`: HTTP status code or error code - `attempt`: Attempt number (1 for first attempt, 2+ for retries) - `event`: Associated event (ID or included object) @@ -2044,7 +2044,7 @@ tags: Use the `include` query parameter to include related data: - `include=event`: Include event summary (id, topic, time, eligible_for_retry, metadata) - `include=event.data`: Include full event with payload data - - `include=response_data`: Include response body and headers from the delivery attempt + - `include=response_data`: Include response body and headers from the attempt - name: Events description: Operations related to event history. @@ -2456,15 +2456,15 @@ paths: schema: $ref: "#/components/schemas/APIErrorResponse" - /deliveries: + /attempts: get: - tags: [Deliveries] - summary: List Deliveries (Admin) + tags: [Attempts] + summary: List Attempts (Admin) description: | - Retrieves a paginated list of deliveries across all tenants. This is an admin-only endpoint that requires the Admin API Key. + Retrieves a paginated list of attempts across all tenants. This is an admin-only endpoint that requires the Admin API Key. - When `tenant_id` is not provided, returns deliveries from all tenants. When `tenant_id` is provided, returns only deliveries for that tenant. - operationId: adminListDeliveries + When `tenant_id` is not provided, returns attempts from all tenants. When `tenant_id` is provided, returns only attempts for that tenant. + operationId: adminListAttempts security: - AdminApiKey: [] parameters: @@ -2473,26 +2473,26 @@ paths: required: false schema: type: string - description: Filter deliveries by tenant ID. If not provided, returns deliveries from all tenants. + description: Filter attempts by tenant ID. If not provided, returns attempts from all tenants. - name: event_id in: query required: false schema: type: string - description: Filter deliveries by event ID. + description: Filter attempts by event ID. - name: destination_id in: query required: false schema: type: string - description: Filter deliveries by destination ID. + description: Filter attempts by destination ID. - name: status in: query required: false schema: type: string enum: [success, failed] - description: Filter deliveries by status. + description: Filter attempts by status. - name: topic in: query required: false @@ -2502,21 +2502,21 @@ paths: - type: array items: type: string - description: Filter deliveries by event topic(s). Can be specified multiple times or comma-separated. + description: Filter attempts by event topic(s). Can be specified multiple times or comma-separated. - name: time[gte] in: query required: false schema: type: string format: date-time - description: Filter deliveries by event time >= value (RFC3339 or YYYY-MM-DD format). + description: Filter attempts by event time >= value (RFC3339 or YYYY-MM-DD format). - name: time[lte] in: query required: false schema: type: string format: date-time - description: Filter deliveries by event time <= value (RFC3339 or YYYY-MM-DD format). + description: Filter attempts by event time <= value (RFC3339 or YYYY-MM-DD format). - name: limit in: query required: false @@ -2570,27 +2570,27 @@ paths: description: Sort direction. responses: "200": - description: A paginated list of deliveries. + description: A paginated list of attempts. content: application/json: schema: - $ref: "#/components/schemas/DeliveryPaginatedResult" + $ref: "#/components/schemas/AttemptPaginatedResult" examples: - AdminDeliveriesListExample: + AdminAttemptsListExample: value: models: - - id: "del_123" + - id: "atm_123" status: "success" delivered_at: "2024-01-01T00:00:05Z" code: "200" - attempt: 1 + attempt_number: 1 event: "evt_123" destination: "des_456" - - id: "del_124" + - id: "att_124" status: "failed" delivered_at: "2024-01-02T10:00:01Z" code: "503" - attempt: 2 + attempt_number: 2 event: "evt_789" destination: "des_789" pagination: @@ -2599,7 +2599,7 @@ paths: limit: 100 next: "MTcwNDA2NzIwMA==" prev: null - AdminDeliveriesWithIncludeExample: + AdminAttemptsWithIncludeExample: summary: Response with include=event value: models: @@ -2607,7 +2607,7 @@ paths: status: "success" delivered_at: "2024-01-01T00:00:05Z" code: "200" - attempt: 1 + attempt_number: 1 event: id: "evt_123" topic: "user.created" @@ -3016,6 +3016,253 @@ paths: description: Tenant or Destination not found. # Add other error responses + # Destination-scoped Attempts + /tenants/{tenant_id}/destinations/{destination_id}/attempts: + parameters: + - name: tenant_id + in: path + required: true + schema: + type: string + description: The ID of the tenant. Required when using AdminApiKey authentication. + - name: destination_id + in: path + required: true + schema: + type: string + description: The ID of the destination. + get: + tags: [Destinations] + summary: List Destination Attempts + description: Retrieves a paginated list of attempts scoped to a specific destination. + operationId: listTenantDestinationAttempts + parameters: + - name: event_id + in: query + required: false + schema: + type: string + description: Filter attempts by event ID. + - name: status + in: query + required: false + schema: + type: string + enum: [success, failed] + description: Filter attempts by status. + - name: topic + in: query + required: false + schema: + oneOf: + - type: string + - type: array + items: + type: string + description: Filter attempts by event topic(s). Can be specified multiple times or comma-separated. + - name: time[gte] + in: query + required: false + schema: + type: string + format: date-time + description: Filter attempts by event time >= value (RFC3339 or YYYY-MM-DD format). + - name: time[lte] + in: query + required: false + schema: + type: string + format: date-time + description: Filter attempts by event time <= value (RFC3339 or YYYY-MM-DD format). + - name: limit + in: query + required: false + schema: + type: integer + default: 100 + minimum: 1 + maximum: 1000 + description: Number of items per page (default 100, max 1000). + - name: next + in: query + required: false + schema: + type: string + description: Cursor for next page of results. + - name: prev + in: query + required: false + schema: + type: string + description: Cursor for previous page of results. + - name: include + in: query + required: false + schema: + oneOf: + - type: string + - type: array + items: + type: string + description: | + Fields to include in the response. Can be specified multiple times or comma-separated. + - `event`: Include event summary (id, topic, time, eligible_for_retry, metadata) + - `event.data`: Include full event with payload data + - `response_data`: Include response body and headers + - name: order_by + in: query + required: false + schema: + type: string + enum: [time] + default: time + description: Field to sort by. + - name: dir + in: query + required: false + schema: + type: string + enum: [asc, desc] + default: desc + description: Sort direction. + responses: + "200": + description: A paginated list of attempts for the destination. + content: + application/json: + schema: + $ref: "#/components/schemas/AttemptPaginatedResult" + examples: + DestinationAttemptsListExample: + value: + models: + - id: "atm_123" + status: "success" + delivered_at: "2024-01-01T00:00:05Z" + code: "200" + attempt_number: 1 + event: "evt_123" + destination: "des_456" + - id: "atm_124" + status: "failed" + delivered_at: "2024-01-02T10:00:01Z" + code: "503" + attempt_number: 2 + event: "evt_789" + destination: "des_456" + pagination: + order_by: "time" + dir: "desc" + limit: 100 + next: "MTcwNDA2NzIwMA==" + prev: null + "404": + description: Tenant or Destination not found. + "422": + description: Validation error (invalid query parameters). + content: + application/json: + schema: + $ref: "#/components/schemas/APIErrorResponse" + + /tenants/{tenant_id}/destinations/{destination_id}/attempts/{attempt_id}: + parameters: + - name: tenant_id + in: path + required: true + schema: + type: string + description: The ID of the tenant. Required when using AdminApiKey authentication. + - name: destination_id + in: path + required: true + schema: + type: string + description: The ID of the destination. + - name: attempt_id + in: path + required: true + schema: + type: string + description: The ID of the attempt. + get: + tags: [Destinations] + summary: Get Destination Attempt + description: Retrieves details for a specific attempt scoped to a destination. + operationId: getTenantDestinationAttempt + parameters: + - name: include + in: query + required: false + schema: + oneOf: + - type: string + - type: array + items: + type: string + description: | + Fields to include in the response. Can be specified multiple times or comma-separated. + - `event`: Include event summary + - `event.data`: Include full event with payload data + - `response_data`: Include response body and headers + responses: + "200": + description: Attempt details. + content: + application/json: + schema: + $ref: "#/components/schemas/Attempt" + examples: + DestinationAttemptExample: + value: + id: "atm_123" + status: "success" + delivered_at: "2024-01-01T00:00:05Z" + code: "200" + attempt_number: 1 + event: "evt_123" + destination: "des_456" + "404": + description: Tenant, Destination, or Attempt not found. + + /tenants/{tenant_id}/destinations/{destination_id}/attempts/{attempt_id}/retry: + parameters: + - name: tenant_id + in: path + required: true + schema: + type: string + description: The ID of the tenant. Required when using AdminApiKey authentication. + - name: destination_id + in: path + required: true + schema: + type: string + description: The ID of the destination. + - name: attempt_id + in: path + required: true + schema: + type: string + description: The ID of the attempt to retry. + post: + tags: [Destinations] + summary: Retry Destination Attempt + description: | + Triggers a retry for an attempt scoped to a destination. Only the latest attempt for an event+destination pair can be retried. + The destination must exist and be enabled. + operationId: retryTenantDestinationAttempt + responses: + "202": + description: Retry accepted for processing. + "404": + description: Tenant, Destination, or Attempt not found. + "409": + description: | + Attempt not eligible for retry. This can happen when: + - The attempt is not the latest for this event+destination pair + - The destination is disabled or deleted + # Publish (Admin Only) /publish: post: @@ -3307,8 +3554,8 @@ paths: "404": description: Tenant not found. - # Deliveries (Tenant Specific - Admin or JWT) - /tenants/{tenant_id}/deliveries: + # Attempts (Tenant Specific - Admin or JWT) + /tenants/{tenant_id}/attempts: parameters: - name: tenant_id in: path @@ -3317,30 +3564,30 @@ paths: type: string description: The ID of the tenant. Required when using AdminApiKey authentication. get: - tags: [Deliveries] - summary: List Deliveries - description: Retrieves a paginated list of deliveries for the tenant, with filtering and sorting options. - operationId: listTenantDeliveries + tags: [Attempts] + summary: List Attempts + description: Retrieves a paginated list of attempts for the tenant, with filtering and sorting options. + operationId: listTenantAttempts parameters: - name: destination_id in: query required: false schema: type: string - description: Filter deliveries by destination ID. + description: Filter attempts by destination ID. - name: event_id in: query required: false schema: type: string - description: Filter deliveries by event ID. + description: Filter attempts by event ID. - name: status in: query required: false schema: type: string enum: [success, failed] - description: Filter deliveries by status. + description: Filter attempts by status. - name: topic in: query required: false @@ -3350,21 +3597,21 @@ paths: - type: array items: type: string - description: Filter deliveries by event topic(s). Can be specified multiple times or comma-separated. + description: Filter attempts by event topic(s). Can be specified multiple times or comma-separated. - name: time[gte] in: query required: false schema: type: string format: date-time - description: Filter deliveries by event time >= value (RFC3339 or YYYY-MM-DD format). + description: Filter attempts by event time >= value (RFC3339 or YYYY-MM-DD format). - name: time[lte] in: query required: false schema: type: string format: date-time - description: Filter deliveries by event time <= value (RFC3339 or YYYY-MM-DD format). + description: Filter attempts by event time <= value (RFC3339 or YYYY-MM-DD format). - name: limit in: query required: false @@ -3418,27 +3665,27 @@ paths: description: Sort direction. responses: "200": - description: A paginated list of deliveries. + description: A paginated list of attempts. content: application/json: schema: - $ref: "#/components/schemas/DeliveryPaginatedResult" + $ref: "#/components/schemas/AttemptPaginatedResult" examples: - DeliveriesListExample: + AttemptsListExample: value: models: - - id: "del_123" + - id: "atm_123" status: "success" delivered_at: "2024-01-01T00:00:05Z" code: "200" - attempt: 1 + attempt_number: 1 event: "evt_123" destination: "des_456" - - id: "del_124" + - id: "att_124" status: "failed" delivered_at: "2024-01-02T10:00:01Z" code: "503" - attempt: 2 + attempt_number: 2 event: "evt_789" destination: "des_456" pagination: @@ -3447,15 +3694,15 @@ paths: limit: 100 next: "MTcwNDA2NzIwMA==" prev: null - DeliveriesWithIncludeExample: + AttemptsWithIncludeExample: summary: Response with include=event value: models: - - id: "del_123" + - id: "atm_123" status: "success" delivered_at: "2024-01-01T00:00:05Z" code: "200" - attempt: 1 + attempt_number: 1 event: id: "evt_123" topic: "user.created" @@ -3478,7 +3725,7 @@ paths: schema: $ref: "#/components/schemas/APIErrorResponse" - /tenants/{tenant_id}/deliveries/{delivery_id}: + /tenants/{tenant_id}/attempts/{attempt_id}: parameters: - name: tenant_id in: path @@ -3486,17 +3733,17 @@ paths: schema: type: string description: The ID of the tenant. Required when using AdminApiKey authentication. - - name: delivery_id + - name: attempt_id in: path required: true schema: type: string - description: The ID of the delivery. + description: The ID of the attempt. get: - tags: [Deliveries] - summary: Get Delivery - description: Retrieves details for a specific delivery. - operationId: getTenantDelivery + tags: [Attempts] + summary: Get Attempt + description: Retrieves details for a specific attempt. + operationId: getTenantAttempt parameters: - name: include in: query @@ -3514,25 +3761,25 @@ paths: - `response_data`: Include response body and headers responses: "200": - description: Delivery details. + description: Attempt details. content: application/json: schema: - $ref: "#/components/schemas/Delivery" + $ref: "#/components/schemas/Attempt" examples: - DeliveryExample: + AttemptExample: value: - id: "del_123" + id: "atm_123" status: "success" delivered_at: "2024-01-01T00:00:05Z" code: "200" - attempt: 1 + attempt_number: 1 event: "evt_123" destination: "des_456" - DeliveryWithIncludeExample: + AttemptWithIncludeExample: summary: Response with include=event.data,response_data value: - id: "del_123" + id: "atm_123" status: "success" delivered_at: "2024-01-01T00:00:05Z" code: "200" @@ -3540,7 +3787,7 @@ paths: status_code: 200 body: '{"status":"ok"}' headers: { "content-type": "application/json" } - attempt: 1 + attempt_number: 1 event: id: "evt_123" topic: "user.created" @@ -3550,9 +3797,9 @@ paths: data: { "user_id": "userid", "status": "active" } destination: "des_456" "404": - description: Tenant or Delivery not found. + description: Tenant or Attempt not found. - /tenants/{tenant_id}/deliveries/{delivery_id}/retry: + /tenants/{tenant_id}/attempts/{attempt_id}/retry: parameters: - name: tenant_id in: path @@ -3560,28 +3807,28 @@ paths: schema: type: string description: The ID of the tenant. Required when using AdminApiKey authentication. - - name: delivery_id + - name: attempt_id in: path required: true schema: type: string - description: The ID of the delivery to retry. + description: The ID of the attempt to retry. post: - tags: [Deliveries] - summary: Retry Delivery + tags: [Attempts] + summary: Retry Attempt description: | - Triggers a retry for a delivery. Only the latest delivery for an event+destination pair can be retried. + Triggers a retry for an attempt. Only the latest attempt for an event+destination pair can be retried. The destination must exist and be enabled. - operationId: retryTenantDelivery + operationId: retryTenantAttempt responses: "202": description: Retry accepted for processing. "404": - description: Tenant or Delivery not found. + description: Tenant or Attempt not found. "409": description: | - Delivery not eligible for retry. This can happen when: - - The delivery is not the latest for this event+destination pair + Attempt not eligible for retry. This can happen when: + - The attempt is not the latest for this event+destination pair - The destination is disabled or deleted # Events (Tenant Specific - Admin or JWT) @@ -3746,7 +3993,7 @@ paths: "404": description: Tenant or Event not found. - /tenants/{tenant_id}/events/{event_id}/deliveries: + /tenants/{tenant_id}/events/{event_id}/attempts: parameters: - name: tenant_id in: path @@ -3762,12 +4009,12 @@ paths: description: The ID of the event. get: tags: [Events] - summary: List Event Delivery Attempts - description: Retrieves a list of delivery attempts for a specific event, including response details. - operationId: listTenantEventDeliveries + summary: List Event Attempts + description: Retrieves a list of attempts for a specific event, including response details. + operationId: listTenantEventAttempts responses: "200": - description: A list of delivery attempts. + description: A list of attempts. content: application/json: schema: @@ -3775,7 +4022,7 @@ paths: items: $ref: "#/components/schemas/DeliveryAttempt" examples: - DeliveriesListExample: + AttemptsListExample: value: - delivered_at: "2024-01-01T00:00:05Z" status: "success" @@ -3790,208 +4037,6 @@ paths: "404": description: Tenant or Event not found. - /tenants/{tenant_id}/destinations/{destination_id}/events: - parameters: - - name: tenant_id - in: path - required: true - schema: - type: string - description: The ID of the tenant. Required when using AdminApiKey authentication. - - name: destination_id - in: path - required: true - schema: - type: string - description: The ID of the destination. - get: - tags: [Events] - summary: List Events by Destination - deprecated: true - description: | - **Deprecated**: Use `GET /tenants/{tenant_id}/deliveries?destination_id={destination_id}` instead. - - Retrieves events associated with a specific destination for the tenant. - operationId: listTenantEventsByDestination - parameters: - - name: status - in: query - required: false - schema: - type: string - enum: [success, failed] - description: Filter events by delivery status. - - name: next - in: query - required: false - schema: - type: string - description: Cursor for next page of results. - - name: prev - in: query - required: false - schema: - type: string - description: Cursor for previous page of results. - - name: limit - in: query - required: false - schema: - type: integer - default: 100 - minimum: 1 - maximum: 1000 - description: Number of items per page (default 100, max 1000). - - name: time[gte] - in: query - required: false - schema: - type: string - format: date-time - description: Filter events with time >= value (RFC3339 or YYYY-MM-DD format). - - name: time[lte] - in: query - required: false - schema: - type: string - format: date-time - description: Filter events with time <= value (RFC3339 or YYYY-MM-DD format). - - name: order_by - in: query - required: false - schema: - type: string - enum: [time] - default: time - description: Field to sort by. - - name: dir - in: query - required: false - schema: - type: string - enum: [asc, desc] - default: desc - description: Sort direction. - responses: - "200": - description: A paginated list of events for the destination. - content: - application/json: - schema: - $ref: "#/components/schemas/EventPaginatedResult" - examples: - EventsListExample: - value: - models: - - id: "evt_123" - destination_id: "des_456" - topic: "user.created" - time: "2024-01-01T00:00:00Z" - successful_at: "2024-01-01T00:00:05Z" - metadata: { "source": "crm" } - data: { "user_id": "userid", "status": "active" } - - id: "evt_789" - destination_id: "des_456" - topic: "order.shipped" - time: "2024-01-02T10:00:00Z" - successful_at: null - metadata: { "source": "oms" } - data: { "order_id": "orderid", "tracking": "1Z..." } - pagination: - order_by: "time" - dir: "desc" - limit: 100 - next: null - prev: null - "404": - description: Tenant or Destination not found. - - /tenants/{tenant_id}/destinations/{destination_id}/events/{event_id}: - parameters: - - name: tenant_id - in: path - required: true - schema: - type: string - description: The ID of the tenant. Required when using AdminApiKey authentication. - - name: destination_id - in: path - required: true - schema: - type: string - description: The ID of the destination. - - name: event_id - in: path - required: true - schema: - type: string - description: The ID of the event. - get: - tags: [Events] - summary: Get Event by Destination - deprecated: true - description: | - **Deprecated**: Use `GET /tenants/{tenant_id}/deliveries/{delivery_id}?include=event.data` instead. - - Retrieves a specific event associated with a specific destination for the tenant. - operationId: getTenantEventByDestination - responses: - "200": - description: Event details. - content: - application/json: - schema: - $ref: "#/components/schemas/Event" - examples: - EventExample: # Same as /{tenant_id}/events/{event_id} example - value: - id: "evt_123" - destination_id: "des_456" - topic: "user.created" - time: "2024-01-01T00:00:00Z" - successful_at: "2024-01-01T00:00:05Z" - metadata: { "source": "crm" } - data: { "user_id": "userid", "status": "active" } - "404": - description: Tenant, Destination or Event not found. - - /tenants/{tenant_id}/destinations/{destination_id}/events/{event_id}/retry: - parameters: - - name: tenant_id - in: path - required: true - schema: - type: string - description: The ID of the tenant. Required when using AdminApiKey authentication. - - name: destination_id - in: path - required: true - schema: - type: string - description: The ID of the destination. - - name: event_id - in: path - required: true - schema: - type: string - description: The ID of the event to retry. - post: - tags: [Events] - summary: Retry Event Delivery - deprecated: true - description: | - **Deprecated**: Use `POST /tenants/{tenant_id}/deliveries/{delivery_id}/retry` instead. - - Triggers a retry for a failed event delivery. - operationId: retryTenantEvent - responses: - "202": - description: Retry accepted for processing. - "404": - description: Tenant, Destination or Event not found. - "409": # Conflict might be appropriate if event is not retryable - description: Event not eligible for retry. - # Tenant Agnostic Routes (JWT Auth Only) - Mirroring tenant-specific routes where AllowTenantFromJWT=true # Note: Portal routes (/portal, /token) still require AdminApiKey even when tenant is inferred from JWT, diff --git a/docs/pages/features/event-delivery.mdx b/docs/pages/features/event-delivery.mdx index a0ebff20..e18c1059 100644 --- a/docs/pages/features/event-delivery.mdx +++ b/docs/pages/features/event-delivery.mdx @@ -12,7 +12,7 @@ The retry interval uses an exponential backoff algorithm with a base of `2`. ## Manual Retries -Manual retries can be triggered for any given event via the [Event API](/docs/api/events#retry-event-delivery) or user portal. +Manual retries can be triggered for any given attempt via the [Attempts API](/docs/api/attempts#retry-attempt) or user portal. ## Disabled destinations diff --git a/docs/pages/guides/building-your-own-ui.mdx b/docs/pages/guides/building-your-own-ui.mdx index 0ab3076a..2edbcc6a 100644 --- a/docs/pages/guides/building-your-own-ui.mdx +++ b/docs/pages/guides/building-your-own-ui.mdx @@ -372,6 +372,6 @@ return ( ); ``` -For each event, you can retrieve all its associated delivery attempts using the [List Event Deliveries Attempts API](/docs/api/event-deliveries-attempts#list-event-deliveries-attempts). +For each event, you can retrieve all its associated delivery attempts using the [List Event Attempts API](/docs/api/events#list-event-attempts). You can find the source code of the `Events.tsx` component of the User Portal here: [Events.tsx](https://github.com/hookdeck/outpost/blob/main/internal/portal/src/scenes/Destination/Events/Events.tsx) diff --git a/docs/pages/guides/migrate-to-outpost.mdx b/docs/pages/guides/migrate-to-outpost.mdx index ad966ea2..cdd2b4a8 100644 --- a/docs/pages/guides/migrate-to-outpost.mdx +++ b/docs/pages/guides/migrate-to-outpost.mdx @@ -129,7 +129,7 @@ To migrate your historical data to Outpost, you need to map your existing data s The Outpost schema contains two tables related to events: 1. **events** - The events that Outpost has received to publish. -2. **deliveries** - The delivery attempts of events to destinations. +2. **attempts** - The delivery attempts of events to destinations. The following diagram shows the Outpost schema. You can connect to the database instance within your Outpost installation to inspect the schema further. diff --git a/docs/pages/references/configuration.mdx b/docs/pages/references/configuration.mdx index ce799b38..528fa31a 100644 --- a/docs/pages/references/configuration.mdx +++ b/docs/pages/references/configuration.mdx @@ -74,10 +74,9 @@ Global configurations are provided through env variables or a YAML file. ConfigM | `GCP_PUBSUB_SERVICE_ACCOUNT_CREDENTIALS` | JSON string or path to a file containing GCP service account credentials for Pub/Sub. Required if GCP Pub/Sub is the chosen MQ provider and not running in an environment with implicit credentials (e.g., GCE, GKE). | `nil` | Conditional | | `GIN_MODE` | Sets the Gin framework mode (e.g., 'debug', 'release', 'test'). See Gin documentation for details. | `release` | No | | `HTTP_USER_AGENT` | Custom HTTP User-Agent string for outgoing webhook deliveries. If unset, a default (OrganizationName/Version) is used. | `nil` | No | -| `IDGEN_DELIVERY_EVENT_PREFIX` | Prefix for delivery event IDs (e.g., 'dev_' produces 'dev_123'). Default: empty (no prefix) | `nil` | No | -| `IDGEN_DELIVERY_PREFIX` | Prefix for delivery IDs (e.g., 'dlv_' produces 'dlv_123'). Default: empty (no prefix) | `nil` | No | -| `IDGEN_DESTINATION_PREFIX` | Prefix for destination IDs (e.g., 'dst_' produces 'dst_123'). Default: empty (no prefix) | `nil` | No | -| `IDGEN_EVENT_PREFIX` | Prefix for event IDs (e.g., 'evt_' produces 'evt_123'). Default: empty (no prefix) | `nil` | No | +| `IDGEN_ATTEMPT_PREFIX` | Prefix for attempt IDs, prepended with underscore (e.g., 'atm_123'). Default: empty (no prefix) | `nil` | No | +| `IDGEN_DESTINATION_PREFIX` | Prefix for destination IDs, prepended with underscore (e.g., 'dst_123'). Default: empty (no prefix) | `nil` | No | +| `IDGEN_EVENT_PREFIX` | Prefix for event IDs, prepended with underscore (e.g., 'evt_123'). Default: empty (no prefix) | `nil` | No | | `IDGEN_TYPE` | ID generation type for all entities: uuidv4, uuidv7, nanoid. Default: uuidv4 | `uuidv4` | No | | `LOG_BATCH_SIZE` | Maximum number of log entries to batch together before writing to storage. | `1000` | No | | `LOG_BATCH_THRESHOLD_SECONDS` | Maximum time in seconds to buffer logs before flushing them to storage, if batch size is not reached. | `10` | No | @@ -134,6 +133,7 @@ Global configurations are provided through env variables or a YAML file. ConfigM | `RETRY_INTERVAL_SECONDS` | Interval in seconds for exponential backoff retry strategy (base 2). Ignored if retry_schedule is provided. | `30` | No | | `RETRY_POLL_BACKOFF_MS` | Backoff time in milliseconds when the retry monitor finds no messages to process. When a retry message is found, the monitor immediately polls for the next message without delay. Lower values provide faster retry processing but increase Redis load. For serverless Redis providers (Upstash, ElastiCache Serverless), consider increasing to 5000-10000ms to reduce costs. Default: 100 | `100` | No | | `RETRY_SCHEDULE` | Comma-separated list of retry delays in seconds. If provided, overrides retry_interval_seconds and retry_max_limit. Schedule length defines the max number of retries. Example: '5,60,600,3600,7200' for 5 retries at 5s, 1m, 10m, 1h, 2h. | `[]` | No | +| `RETRY_VISIBILITY_TIMEOUT_SECONDS` | Time in seconds a retry message is hidden after being received before becoming visible again for reprocessing. This applies when event data is temporarily unavailable (e.g., race condition with log persistence). Default: 30 | `30` | No | | `SERVICE` | Specifies the service type to run. Valid values: 'api', 'log', 'delivery', or empty/all for singular mode (runs all services). | `nil` | No | | `TELEMETRY_BATCH_INTERVAL` | Maximum time in seconds to wait before sending a batch of telemetry events if batch size is not reached. | `5` | No | | `TELEMETRY_BATCH_SIZE` | Maximum number of telemetry events to batch before sending. | `100` | No | @@ -270,16 +270,13 @@ gin_mode: "release" http_user_agent: "" idgen: - # Prefix for delivery event IDs (e.g., 'dev_' produces 'dev_123'). Default: empty (no prefix) - delivery_event_prefix: "" + # Prefix for attempt IDs, prepended with underscore (e.g., 'atm_123'). Default: empty (no prefix) + attempt_prefix: "" - # Prefix for delivery IDs (e.g., 'dlv_' produces 'dlv_123'). Default: empty (no prefix) - delivery_prefix: "" - - # Prefix for destination IDs (e.g., 'dst_' produces 'dst_123'). Default: empty (no prefix) + # Prefix for destination IDs, prepended with underscore (e.g., 'dst_123'). Default: empty (no prefix) destination_prefix: "" - # Prefix for event IDs (e.g., 'evt_' produces 'evt_123'). Default: empty (no prefix) + # Prefix for event IDs, prepended with underscore (e.g., 'evt_123'). Default: empty (no prefix) event_prefix: "" # ID generation type for all entities: uuidv4, uuidv7, nanoid. Default: uuidv4 @@ -611,6 +608,9 @@ retry_poll_backoff_ms: 100 # Comma-separated list of retry delays in seconds. If provided, overrides retry_interval_seconds and retry_max_limit. Schedule length defines the max number of retries. Example: '5,60,600,3600,7200' for 5 retries at 5s, 1m, 10m, 1h, 2h. retry_schedule: [] +# Time in seconds a retry message is hidden after being received before becoming visible again for reprocessing. This applies when event data is temporarily unavailable (e.g., race condition with log persistence). Default: 30 +retry_visibility_timeout_seconds: 30 + # Specifies the service type to run. Valid values: 'api', 'log', 'delivery', or empty/all for singular mode (runs all services). service: "" diff --git a/go.mod b/go.mod index c2de85c1..48572390 100644 --- a/go.mod +++ b/go.mod @@ -51,7 +51,6 @@ require ( github.com/testcontainers/testcontainers-go/modules/localstack v0.36.0 github.com/testcontainers/testcontainers-go/modules/postgres v0.36.0 github.com/testcontainers/testcontainers-go/modules/rabbitmq v0.36.0 - github.com/testcontainers/testcontainers-go/modules/redis v0.36.0 github.com/uptrace/opentelemetry-go-extra/otelzap v0.3.1 github.com/urfave/cli/v3 v3.4.1 go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin v0.53.0 diff --git a/go.sum b/go.sum index 6c635117..af9e1f8a 100644 --- a/go.sum +++ b/go.sum @@ -906,8 +906,6 @@ github.com/go-playground/validator/v10 v10.22.0 h1:k6HsTZ0sTnROkhS//R0O+55JgM8C4 github.com/go-playground/validator/v10 v10.22.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg= github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= -github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI= -github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= @@ -1288,8 +1286,6 @@ github.com/testcontainers/testcontainers-go/modules/postgres v0.36.0 h1:xTGNNsOD github.com/testcontainers/testcontainers-go/modules/postgres v0.36.0/go.mod h1:WKS3MGq1lzbVibIRnL08TOaf5bKWPxJe5frzyQfV4oY= github.com/testcontainers/testcontainers-go/modules/rabbitmq v0.36.0 h1:gobSVNvTsiJTcGTlVJMpeUfAcz85HAMMwo8xEVQZItE= github.com/testcontainers/testcontainers-go/modules/rabbitmq v0.36.0/go.mod h1:rLtFlrLEWcU/Ud52FiGk57QvUqoAHvR380hZo+tkBaI= -github.com/testcontainers/testcontainers-go/modules/redis v0.36.0 h1:Z+6APQ0DjQP8Kj5Fu+lkAlH2v7f5QkAQyyjnf1Kq8sw= -github.com/testcontainers/testcontainers-go/modules/redis v0.36.0/go.mod h1:LV66RJhSMikZrxJRc6O0nKcRqykmjQSyX82S93haE2w= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tklauser/go-sysconf v0.3.14 h1:g5vzr9iPFFz24v2KZXs/pvpvh8/V9Fw6vQK5ZZb78yU= github.com/tklauser/go-sysconf v0.3.14/go.mod h1:1ym4lWMLUOhuBOPGtRcJm7tEGX4SCYNEEEtghGG/8uY= diff --git a/internal/alert/evaluator.go b/internal/alert/evaluator.go index 60562b5b..92386e96 100644 --- a/internal/alert/evaluator.go +++ b/internal/alert/evaluator.go @@ -39,7 +39,6 @@ func NewAlertEvaluator(thresholds []int, autoDisableFailureCount int) AlertEvalu }) } - // Sort by failure count sort.Slice(finalThresholds, func(i, j int) bool { return finalThresholds[i].failures < finalThresholds[j].failures }) // Check if we need to add 100 diff --git a/internal/alert/monitor.go b/internal/alert/monitor.go index 6694e06c..39ff54f4 100644 --- a/internal/alert/monitor.go +++ b/internal/alert/monitor.go @@ -83,7 +83,7 @@ func WithDeploymentID(deploymentID string) AlertOption { // DeliveryAttempt represents a single delivery attempt type DeliveryAttempt struct { Success bool - DeliveryEvent *models.DeliveryEvent + DeliveryTask *models.DeliveryTask Destination *AlertDestination Timestamp time.Time DeliveryResponse map[string]interface{} @@ -148,7 +148,6 @@ func (m *alertMonitor) HandleAttempt(ctx context.Context, attempt DeliveryAttemp return fmt.Errorf("failed to get alert state: %w", err) } - // Check if we should send an alert level, shouldAlert := m.evaluator.ShouldAlert(count) if !shouldAlert { return nil @@ -156,10 +155,10 @@ func (m *alertMonitor) HandleAttempt(ctx context.Context, attempt DeliveryAttemp alert := NewConsecutiveFailureAlert(ConsecutiveFailureData{ Event: AlertedEvent{ - ID: attempt.DeliveryEvent.Event.ID, - Topic: attempt.DeliveryEvent.Event.Topic, - Metadata: attempt.DeliveryEvent.Event.Metadata, - Data: attempt.DeliveryEvent.Event.Data, + ID: attempt.DeliveryTask.Event.ID, + Topic: attempt.DeliveryTask.Event.Topic, + Metadata: attempt.DeliveryTask.Event.Metadata, + Data: attempt.DeliveryTask.Event.Data, }, MaxConsecutiveFailures: m.autoDisableFailureCount, ConsecutiveFailures: count, @@ -175,7 +174,7 @@ func (m *alertMonitor) HandleAttempt(ctx context.Context, attempt DeliveryAttemp } m.logger.Ctx(ctx).Audit("destination disabled", - zap.String("event_id", attempt.DeliveryEvent.Event.ID), + zap.String("event_id", attempt.DeliveryTask.Event.ID), zap.String("tenant_id", attempt.Destination.TenantID), zap.String("destination_id", attempt.Destination.ID), zap.String("destination_type", attempt.Destination.Type), @@ -187,7 +186,7 @@ func (m *alertMonitor) HandleAttempt(ctx context.Context, attempt DeliveryAttemp if err := m.notifier.Notify(ctx, alert); err != nil { m.logger.Ctx(ctx).Error("failed to send alert", zap.Error(err), - zap.String("event_id", attempt.DeliveryEvent.Event.ID), + zap.String("event_id", attempt.DeliveryTask.Event.ID), zap.String("tenant_id", attempt.Destination.TenantID), zap.String("destination_id", attempt.Destination.ID), zap.String("destination_type", attempt.Destination.Type), @@ -196,7 +195,7 @@ func (m *alertMonitor) HandleAttempt(ctx context.Context, attempt DeliveryAttemp } m.logger.Ctx(ctx).Audit("alert sent", - zap.String("event_id", attempt.DeliveryEvent.Event.ID), + zap.String("event_id", attempt.DeliveryTask.Event.ID), zap.String("tenant_id", attempt.Destination.TenantID), zap.String("destination_id", attempt.Destination.ID), zap.String("destination_type", attempt.Destination.Type), diff --git a/internal/alert/monitor_test.go b/internal/alert/monitor_test.go index 10f6003e..16372475 100644 --- a/internal/alert/monitor_test.go +++ b/internal/alert/monitor_test.go @@ -53,11 +53,11 @@ func TestAlertMonitor_ConsecutiveFailures_MaxFailures(t *testing.T) { dest := &alert.AlertDestination{ID: "dest_1", TenantID: "tenant_1"} event := &models.Event{Topic: "test.event"} - deliveryEvent := &models.DeliveryEvent{Event: *event} + task := &models.DeliveryTask{Event: *event} attempt := alert.DeliveryAttempt{ - Success: false, - DeliveryEvent: deliveryEvent, - Destination: dest, + Success: false, + DeliveryTask: task, + Destination: dest, DeliveryResponse: map[string]interface{}{ "status": "500", "data": map[string]any{"error": "test error"}, @@ -120,11 +120,11 @@ func TestAlertMonitor_ConsecutiveFailures_Reset(t *testing.T) { dest := &alert.AlertDestination{ID: "dest_1", TenantID: "tenant_1"} event := &models.Event{Topic: "test.event"} - deliveryEvent := &models.DeliveryEvent{Event: *event} + task := &models.DeliveryTask{Event: *event} failedAttempt := alert.DeliveryAttempt{ - Success: false, - DeliveryEvent: deliveryEvent, - Destination: dest, + Success: false, + DeliveryTask: task, + Destination: dest, DeliveryResponse: map[string]interface{}{ "status": "500", "data": map[string]any{"error": "test error"}, @@ -193,11 +193,11 @@ func TestAlertMonitor_ConsecutiveFailures_AboveThreshold(t *testing.T) { dest := &alert.AlertDestination{ID: "dest_above", TenantID: "tenant_above"} event := &models.Event{Topic: "test.event"} - deliveryEvent := &models.DeliveryEvent{Event: *event} + task := &models.DeliveryTask{Event: *event} attempt := alert.DeliveryAttempt{ - Success: false, - DeliveryEvent: deliveryEvent, - Destination: dest, + Success: false, + DeliveryTask: task, + Destination: dest, DeliveryResponse: map[string]interface{}{ "status": "500", }, diff --git a/internal/alert/notifier.go b/internal/alert/notifier.go index c0c8775f..ef9589bc 100644 --- a/internal/alert/notifier.go +++ b/internal/alert/notifier.go @@ -110,32 +110,27 @@ func NewHTTPAlertNotifier(callbackURL string, opts ...NotifierOption) AlertNotif } func (n *httpAlertNotifier) Notify(ctx context.Context, alert Alert) error { - // Marshal alert to JSON body, err := alert.MarshalJSON() if err != nil { return fmt.Errorf("failed to marshal alert: %w", err) } - // Create request req, err := http.NewRequestWithContext(ctx, http.MethodPost, n.callbackURL, bytes.NewReader(body)) if err != nil { return fmt.Errorf("failed to create request: %w", err) } - // Set headers req.Header.Set("Content-Type", "application/json") if n.bearerToken != "" { req.Header.Set("Authorization", "Bearer "+n.bearerToken) } - // Send request resp, err := n.client.Do(req) if err != nil { return fmt.Errorf("failed to send alert: %w", err) } defer resp.Body.Close() - // Check response status if resp.StatusCode >= 400 { return fmt.Errorf("alert callback failed with status %d", resp.StatusCode) } diff --git a/internal/alert/store.go b/internal/alert/store.go index 9aaf2424..3bc3a0a4 100644 --- a/internal/alert/store.go +++ b/internal/alert/store.go @@ -42,7 +42,6 @@ func (s *redisAlertStore) IncrementConsecutiveFailureCount(ctx context.Context, incrCmd := pipe.Incr(ctx, key) pipe.Expire(ctx, key, 24*time.Hour) - // Execute the transaction _, err := pipe.Exec(ctx) if err != nil { return 0, fmt.Errorf("failed to execute consecutive failure count transaction: %w", err) diff --git a/internal/apirouter/legacy_handlers.go b/internal/apirouter/legacy_handlers.go deleted file mode 100644 index 18195f5e..00000000 --- a/internal/apirouter/legacy_handlers.go +++ /dev/null @@ -1,265 +0,0 @@ -package apirouter - -import ( - "errors" - "net/http" - "strconv" - - "github.com/gin-gonic/gin" - "github.com/hookdeck/outpost/internal/deliverymq" - "github.com/hookdeck/outpost/internal/logging" - "github.com/hookdeck/outpost/internal/logstore" - "github.com/hookdeck/outpost/internal/models" - "go.uber.org/zap" -) - -var ( - ErrDestinationDisabled = errors.New("destination is disabled") -) - -// LegacyHandlers provides backward-compatible endpoints for the old API. -// These handlers are deprecated and will be removed in a future version. -type LegacyHandlers struct { - logger *logging.Logger - entityStore models.EntityStore - logStore logstore.LogStore - deliveryMQ *deliverymq.DeliveryMQ -} - -func NewLegacyHandlers( - logger *logging.Logger, - entityStore models.EntityStore, - logStore logstore.LogStore, - deliveryMQ *deliverymq.DeliveryMQ, -) *LegacyHandlers { - return &LegacyHandlers{ - logger: logger, - entityStore: entityStore, - logStore: logStore, - deliveryMQ: deliveryMQ, - } -} - -// setDeprecationHeader adds deprecation warning headers to the response. -func setDeprecationHeader(c *gin.Context, newEndpoint string) { - c.Header("Deprecation", "true") - c.Header("X-Deprecated-Message", "This endpoint is deprecated. Use "+newEndpoint+" instead.") -} - -// RetryByEventDestination handles the legacy retry endpoint: -// POST /:tenantID/destinations/:destinationID/events/:eventID/retry -// -// This shim finds the latest delivery for the event+destination pair and retries it. -// Deprecated: Use POST /:tenantID/deliveries/:deliveryID/retry instead. -func (h *LegacyHandlers) RetryByEventDestination(c *gin.Context) { - setDeprecationHeader(c, "POST /:tenantID/deliveries/:deliveryID/retry") - - tenant := mustTenantFromContext(c) - if tenant == nil { - return - } - destinationID := c.Param("destinationID") - eventID := c.Param("eventID") - - // 1. Check destination exists and is enabled - destination, err := h.entityStore.RetrieveDestination(c.Request.Context(), tenant.ID, destinationID) - if err != nil { - AbortWithError(c, http.StatusInternalServerError, NewErrInternalServer(err)) - return - } - if destination == nil { - AbortWithError(c, http.StatusNotFound, NewErrNotFound("destination")) - return - } - if destination.DisabledAt != nil { - AbortWithError(c, http.StatusBadRequest, NewErrBadRequest(ErrDestinationDisabled)) - return - } - - // 2. Retrieve event - event, err := h.logStore.RetrieveEvent(c.Request.Context(), logstore.RetrieveEventRequest{ - TenantID: tenant.ID, - EventID: eventID, - }) - if err != nil { - AbortWithError(c, http.StatusInternalServerError, NewErrInternalServer(err)) - return - } - if event == nil { - AbortWithError(c, http.StatusNotFound, NewErrNotFound("event")) - return - } - - // 3. Create and publish retry delivery event - deliveryEvent := models.NewManualDeliveryEvent(*event, destination.ID) - - if err := h.deliveryMQ.Publish(c.Request.Context(), deliveryEvent); err != nil { - AbortWithError(c, http.StatusInternalServerError, NewErrInternalServer(err)) - return - } - - h.logger.Ctx(c.Request.Context()).Audit("manual retry initiated (legacy)", - zap.String("event_id", event.ID), - zap.String("destination_id", destination.ID)) - - c.JSON(http.StatusAccepted, gin.H{ - "success": true, - }) -} - -// ListEventsByDestination handles the legacy endpoint: -// GET /:tenantID/destinations/:destinationID/events -// -// This shim queries deliveries filtered by destination and returns unique events. -// Deprecated: Use GET /:tenantID/deliveries?destination_id=X&include=event instead. -func (h *LegacyHandlers) ListEventsByDestination(c *gin.Context) { - setDeprecationHeader(c, "GET /:tenantID/deliveries?destination_id=X&include=event") - - tenant := mustTenantFromContext(c) - if tenant == nil { - return - } - destinationID := c.Param("destinationID") - - // Parse and validate cursors (next/prev are mutually exclusive) - cursors, errResp := ParseCursors(c) - if errResp != nil { - AbortWithError(c, errResp.Code, *errResp) - return - } - - // Parse pagination params - limit := 100 - if limitStr := c.Query("limit"); limitStr != "" { - if parsed, err := strconv.Atoi(limitStr); err == nil && parsed > 0 { - limit = parsed - } - } - - // Query deliveries for this destination with pagination - response, err := h.logStore.ListDeliveryEvent(c.Request.Context(), logstore.ListDeliveryEventRequest{ - TenantID: tenant.ID, - DestinationIDs: []string{destinationID}, - Limit: limit, - Next: cursors.Next, - Prev: cursors.Prev, - SortOrder: "desc", - }) - if err != nil { - AbortWithError(c, http.StatusInternalServerError, NewErrInternalServer(err)) - return - } - - // Extract unique events (by event ID, keep first occurrence) - seen := make(map[string]bool) - events := []models.Event{} - for _, de := range response.Data { - if !seen[de.Event.ID] { - seen[de.Event.ID] = true - events = append(events, de.Event) - } - } - - // Return empty array (not null) if no events - if len(events) == 0 { - c.JSON(http.StatusOK, gin.H{ - "data": []models.Event{}, - "next": "", - "prev": "", - "count": 0, - }) - return - } - - c.JSON(http.StatusOK, gin.H{ - "data": events, - "next": response.Next, - "prev": response.Prev, - "count": len(events), - }) -} - -// RetrieveEventByDestination handles the legacy endpoint: -// GET /:tenantID/destinations/:destinationID/events/:eventID -// -// Deprecated: Use GET /:tenantID/events/:eventID instead. -func (h *LegacyHandlers) RetrieveEventByDestination(c *gin.Context) { - setDeprecationHeader(c, "GET /:tenantID/events/:eventID") - - tenant := mustTenantFromContext(c) - if tenant == nil { - return - } - eventID := c.Param("eventID") - // destinationID is available but not strictly needed for retrieval - - event, err := h.logStore.RetrieveEvent(c.Request.Context(), logstore.RetrieveEventRequest{ - TenantID: tenant.ID, - EventID: eventID, - }) - if err != nil { - AbortWithError(c, http.StatusInternalServerError, NewErrInternalServer(err)) - return - } - if event == nil { - AbortWithError(c, http.StatusNotFound, NewErrNotFound("event")) - return - } - - c.JSON(http.StatusOK, event) -} - -// LegacyDeliveryResponse matches the old delivery response format. -type LegacyDeliveryResponse struct { - ID string `json:"id"` - DeliveredAt string `json:"delivered_at"` - Status string `json:"status"` - Code string `json:"code"` - ResponseData map[string]interface{} `json:"response_data"` -} - -// ListDeliveriesByEvent handles the legacy endpoint: -// GET /:tenantID/events/:eventID/deliveries -// -// Deprecated: Use GET /:tenantID/deliveries?event_id=X instead. -func (h *LegacyHandlers) ListDeliveriesByEvent(c *gin.Context) { - setDeprecationHeader(c, "GET /:tenantID/deliveries?event_id=X") - - tenant := mustTenantFromContext(c) - if tenant == nil { - return - } - eventID := c.Param("eventID") - - // Query deliveries for this event - response, err := h.logStore.ListDeliveryEvent(c.Request.Context(), logstore.ListDeliveryEventRequest{ - TenantID: tenant.ID, - EventID: eventID, - Limit: 100, - SortOrder: "desc", - }) - if err != nil { - AbortWithError(c, http.StatusInternalServerError, NewErrInternalServer(err)) - return - } - - // Return empty array (not null) if no deliveries - if len(response.Data) == 0 { - c.JSON(http.StatusOK, []LegacyDeliveryResponse{}) - return - } - - // Transform to legacy delivery response format (bare array) - deliveries := make([]LegacyDeliveryResponse, len(response.Data)) - for i, de := range response.Data { - deliveries[i] = LegacyDeliveryResponse{ - ID: de.Delivery.ID, - DeliveredAt: de.Delivery.Time.UTC().Format("2006-01-02T15:04:05Z07:00"), - Status: de.Delivery.Status, - Code: de.Delivery.Code, - ResponseData: de.Delivery.ResponseData, - } - } - - c.JSON(http.StatusOK, deliveries) -} diff --git a/internal/apirouter/legacy_handlers_test.go b/internal/apirouter/legacy_handlers_test.go deleted file mode 100644 index 883eac63..00000000 --- a/internal/apirouter/legacy_handlers_test.go +++ /dev/null @@ -1,348 +0,0 @@ -package apirouter_test - -import ( - "context" - "encoding/json" - "fmt" - "net/http" - "net/http/httptest" - "testing" - "time" - - "github.com/hookdeck/outpost/internal/idgen" - "github.com/hookdeck/outpost/internal/models" - "github.com/hookdeck/outpost/internal/util/testutil" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestLegacyRetryByEventDestination(t *testing.T) { - t.Parallel() - - result := setupTestRouterFull(t, "", "") - - // Create a tenant and destination - tenantID := idgen.String() - destinationID := idgen.Destination() - require.NoError(t, result.entityStore.UpsertTenant(context.Background(), models.Tenant{ - ID: tenantID, - CreatedAt: time.Now(), - })) - require.NoError(t, result.entityStore.UpsertDestination(context.Background(), models.Destination{ - ID: destinationID, - TenantID: tenantID, - Type: "webhook", - Topics: []string{"*"}, - CreatedAt: time.Now(), - })) - - // Seed a delivery event - eventID := idgen.Event() - deliveryID := idgen.Delivery() - eventTime := time.Now().Add(-1 * time.Hour).Truncate(time.Millisecond) - deliveryTime := eventTime.Add(100 * time.Millisecond) - - event := testutil.EventFactory.AnyPointer( - testutil.EventFactory.WithID(eventID), - testutil.EventFactory.WithTenantID(tenantID), - testutil.EventFactory.WithDestinationID(destinationID), - testutil.EventFactory.WithTopic("order.created"), - testutil.EventFactory.WithTime(eventTime), - ) - - delivery := testutil.DeliveryFactory.AnyPointer( - testutil.DeliveryFactory.WithID(deliveryID), - testutil.DeliveryFactory.WithEventID(eventID), - testutil.DeliveryFactory.WithDestinationID(destinationID), - testutil.DeliveryFactory.WithStatus("failed"), - testutil.DeliveryFactory.WithTime(deliveryTime), - ) - - de := &models.DeliveryEvent{ - ID: fmt.Sprintf("%s_%s", eventID, deliveryID), - DestinationID: destinationID, - Event: *event, - Delivery: delivery, - } - - require.NoError(t, result.logStore.InsertManyDeliveryEvent(context.Background(), []*models.DeliveryEvent{de})) - - t.Run("should retry via legacy endpoint and return deprecation header", func(t *testing.T) { - w := httptest.NewRecorder() - req, _ := http.NewRequest("POST", baseAPIPath+"/tenants/"+tenantID+"/destinations/"+destinationID+"/events/"+eventID+"/retry", nil) - result.router.ServeHTTP(w, req) - - assert.Equal(t, http.StatusAccepted, w.Code) - assert.Equal(t, "true", w.Header().Get("Deprecation")) - assert.Contains(t, w.Header().Get("X-Deprecated-Message"), "POST /:tenantID/deliveries/:deliveryID/retry") - - var response map[string]interface{} - require.NoError(t, json.Unmarshal(w.Body.Bytes(), &response)) - assert.Equal(t, true, response["success"]) - }) - - t.Run("should return 404 for non-existent event", func(t *testing.T) { - w := httptest.NewRecorder() - req, _ := http.NewRequest("POST", baseAPIPath+"/tenants/"+tenantID+"/destinations/"+destinationID+"/events/nonexistent/retry", nil) - result.router.ServeHTTP(w, req) - - assert.Equal(t, http.StatusNotFound, w.Code) - }) - - t.Run("should return 404 for non-existent destination", func(t *testing.T) { - w := httptest.NewRecorder() - req, _ := http.NewRequest("POST", baseAPIPath+"/tenants/"+tenantID+"/destinations/nonexistent/events/"+eventID+"/retry", nil) - result.router.ServeHTTP(w, req) - - assert.Equal(t, http.StatusNotFound, w.Code) - }) - - t.Run("should return 400 when destination is disabled", func(t *testing.T) { - // Create a disabled destination - disabledDestinationID := idgen.Destination() - disabledAt := time.Now() - require.NoError(t, result.entityStore.UpsertDestination(context.Background(), models.Destination{ - ID: disabledDestinationID, - TenantID: tenantID, - Type: "webhook", - Topics: []string{"*"}, - CreatedAt: time.Now(), - DisabledAt: &disabledAt, - })) - - w := httptest.NewRecorder() - req, _ := http.NewRequest("POST", baseAPIPath+"/tenants/"+tenantID+"/destinations/"+disabledDestinationID+"/events/"+eventID+"/retry", nil) - result.router.ServeHTTP(w, req) - - assert.Equal(t, http.StatusBadRequest, w.Code) - }) -} - -func TestLegacyListEventsByDestination(t *testing.T) { - t.Parallel() - - result := setupTestRouterFull(t, "", "") - - // Create a tenant and destination - tenantID := idgen.String() - destinationID := idgen.Destination() - require.NoError(t, result.entityStore.UpsertTenant(context.Background(), models.Tenant{ - ID: tenantID, - CreatedAt: time.Now(), - })) - require.NoError(t, result.entityStore.UpsertDestination(context.Background(), models.Destination{ - ID: destinationID, - TenantID: tenantID, - Type: "webhook", - Topics: []string{"*"}, - CreatedAt: time.Now(), - })) - - // Seed delivery events - eventID := idgen.Event() - deliveryID := idgen.Delivery() - eventTime := time.Now().Add(-1 * time.Hour).Truncate(time.Millisecond) - deliveryTime := eventTime.Add(100 * time.Millisecond) - - event := testutil.EventFactory.AnyPointer( - testutil.EventFactory.WithID(eventID), - testutil.EventFactory.WithTenantID(tenantID), - testutil.EventFactory.WithDestinationID(destinationID), - testutil.EventFactory.WithTopic("order.created"), - testutil.EventFactory.WithTime(eventTime), - ) - - delivery := testutil.DeliveryFactory.AnyPointer( - testutil.DeliveryFactory.WithID(deliveryID), - testutil.DeliveryFactory.WithEventID(eventID), - testutil.DeliveryFactory.WithDestinationID(destinationID), - testutil.DeliveryFactory.WithStatus("success"), - testutil.DeliveryFactory.WithTime(deliveryTime), - ) - - de := &models.DeliveryEvent{ - ID: fmt.Sprintf("%s_%s", eventID, deliveryID), - DestinationID: destinationID, - Event: *event, - Delivery: delivery, - } - - require.NoError(t, result.logStore.InsertManyDeliveryEvent(context.Background(), []*models.DeliveryEvent{de})) - - t.Run("should list events for destination with deprecation header", func(t *testing.T) { - w := httptest.NewRecorder() - req, _ := http.NewRequest("GET", baseAPIPath+"/tenants/"+tenantID+"/destinations/"+destinationID+"/events", nil) - result.router.ServeHTTP(w, req) - - assert.Equal(t, http.StatusOK, w.Code) - assert.Equal(t, "true", w.Header().Get("Deprecation")) - - var response map[string]interface{} - require.NoError(t, json.Unmarshal(w.Body.Bytes(), &response)) - - events := response["data"].([]interface{}) - assert.Len(t, events, 1) - - firstEvent := events[0].(map[string]interface{}) - assert.Equal(t, eventID, firstEvent["id"]) - assert.Equal(t, "order.created", firstEvent["topic"]) - }) -} - -func TestLegacyRetrieveEventByDestination(t *testing.T) { - t.Parallel() - - result := setupTestRouterFull(t, "", "") - - // Create a tenant and destination - tenantID := idgen.String() - destinationID := idgen.Destination() - require.NoError(t, result.entityStore.UpsertTenant(context.Background(), models.Tenant{ - ID: tenantID, - CreatedAt: time.Now(), - })) - require.NoError(t, result.entityStore.UpsertDestination(context.Background(), models.Destination{ - ID: destinationID, - TenantID: tenantID, - Type: "webhook", - Topics: []string{"*"}, - CreatedAt: time.Now(), - })) - - // Seed a delivery event - eventID := idgen.Event() - deliveryID := idgen.Delivery() - eventTime := time.Now().Add(-1 * time.Hour).Truncate(time.Millisecond) - deliveryTime := eventTime.Add(100 * time.Millisecond) - - event := testutil.EventFactory.AnyPointer( - testutil.EventFactory.WithID(eventID), - testutil.EventFactory.WithTenantID(tenantID), - testutil.EventFactory.WithDestinationID(destinationID), - testutil.EventFactory.WithTopic("order.created"), - testutil.EventFactory.WithTime(eventTime), - ) - - delivery := testutil.DeliveryFactory.AnyPointer( - testutil.DeliveryFactory.WithID(deliveryID), - testutil.DeliveryFactory.WithEventID(eventID), - testutil.DeliveryFactory.WithDestinationID(destinationID), - testutil.DeliveryFactory.WithStatus("success"), - testutil.DeliveryFactory.WithTime(deliveryTime), - ) - - de := &models.DeliveryEvent{ - ID: fmt.Sprintf("%s_%s", eventID, deliveryID), - DestinationID: destinationID, - Event: *event, - Delivery: delivery, - } - - require.NoError(t, result.logStore.InsertManyDeliveryEvent(context.Background(), []*models.DeliveryEvent{de})) - - t.Run("should retrieve event by destination with deprecation header", func(t *testing.T) { - w := httptest.NewRecorder() - req, _ := http.NewRequest("GET", baseAPIPath+"/tenants/"+tenantID+"/destinations/"+destinationID+"/events/"+eventID, nil) - result.router.ServeHTTP(w, req) - - assert.Equal(t, http.StatusOK, w.Code) - assert.Equal(t, "true", w.Header().Get("Deprecation")) - - var response map[string]interface{} - require.NoError(t, json.Unmarshal(w.Body.Bytes(), &response)) - - assert.Equal(t, eventID, response["id"]) - assert.Equal(t, "order.created", response["topic"]) - }) - - t.Run("should return 404 for non-existent event", func(t *testing.T) { - w := httptest.NewRecorder() - req, _ := http.NewRequest("GET", baseAPIPath+"/tenants/"+tenantID+"/destinations/"+destinationID+"/events/nonexistent", nil) - result.router.ServeHTTP(w, req) - - assert.Equal(t, http.StatusNotFound, w.Code) - }) -} - -func TestLegacyListDeliveriesByEvent(t *testing.T) { - t.Parallel() - - result := setupTestRouterFull(t, "", "") - - // Create a tenant and destination - tenantID := idgen.String() - destinationID := idgen.Destination() - require.NoError(t, result.entityStore.UpsertTenant(context.Background(), models.Tenant{ - ID: tenantID, - CreatedAt: time.Now(), - })) - require.NoError(t, result.entityStore.UpsertDestination(context.Background(), models.Destination{ - ID: destinationID, - TenantID: tenantID, - Type: "webhook", - Topics: []string{"*"}, - CreatedAt: time.Now(), - })) - - // Seed a delivery event - eventID := idgen.Event() - deliveryID := idgen.Delivery() - eventTime := time.Now().Add(-1 * time.Hour).Truncate(time.Millisecond) - deliveryTime := eventTime.Add(100 * time.Millisecond) - - event := testutil.EventFactory.AnyPointer( - testutil.EventFactory.WithID(eventID), - testutil.EventFactory.WithTenantID(tenantID), - testutil.EventFactory.WithDestinationID(destinationID), - testutil.EventFactory.WithTopic("order.created"), - testutil.EventFactory.WithTime(eventTime), - ) - - delivery := testutil.DeliveryFactory.AnyPointer( - testutil.DeliveryFactory.WithID(deliveryID), - testutil.DeliveryFactory.WithEventID(eventID), - testutil.DeliveryFactory.WithDestinationID(destinationID), - testutil.DeliveryFactory.WithStatus("success"), - testutil.DeliveryFactory.WithTime(deliveryTime), - ) - - de := &models.DeliveryEvent{ - ID: fmt.Sprintf("%s_%s", eventID, deliveryID), - DestinationID: destinationID, - Event: *event, - Delivery: delivery, - } - - require.NoError(t, result.logStore.InsertManyDeliveryEvent(context.Background(), []*models.DeliveryEvent{de})) - - t.Run("should list deliveries for event with deprecation header", func(t *testing.T) { - w := httptest.NewRecorder() - req, _ := http.NewRequest("GET", baseAPIPath+"/tenants/"+tenantID+"/events/"+eventID+"/deliveries", nil) - result.router.ServeHTTP(w, req) - - assert.Equal(t, http.StatusOK, w.Code) - assert.Equal(t, "true", w.Header().Get("Deprecation")) - - // Old format returns bare array, not {data: [...]} - var deliveries []map[string]interface{} - require.NoError(t, json.Unmarshal(w.Body.Bytes(), &deliveries)) - - assert.Len(t, deliveries, 1) - assert.Equal(t, deliveryID, deliveries[0]["id"]) - assert.Equal(t, "success", deliveries[0]["status"]) - }) - - t.Run("should return empty list for non-existent event", func(t *testing.T) { - w := httptest.NewRecorder() - req, _ := http.NewRequest("GET", baseAPIPath+"/tenants/"+tenantID+"/events/nonexistent/deliveries", nil) - result.router.ServeHTTP(w, req) - - assert.Equal(t, http.StatusOK, w.Code) - - // Old format returns bare array - var deliveries []map[string]interface{} - require.NoError(t, json.Unmarshal(w.Body.Bytes(), &deliveries)) - - assert.Len(t, deliveries, 0) - }) -} diff --git a/internal/apirouter/log_handlers.go b/internal/apirouter/log_handlers.go index 7e2ee0b2..7f22339c 100644 --- a/internal/apirouter/log_handlers.go +++ b/internal/apirouter/log_handlers.go @@ -11,7 +11,6 @@ import ( "github.com/hookdeck/outpost/internal/cursor" "github.com/hookdeck/outpost/internal/logging" "github.com/hookdeck/outpost/internal/logstore" - "github.com/hookdeck/outpost/internal/models" ) type LogHandlers struct { @@ -87,14 +86,14 @@ func parseIncludeOptions(c *gin.Context) IncludeOptions { // API Response types -// APIDelivery is the API response for a delivery -type APIDelivery struct { +// APIAttempt is the API response for an attempt +type APIAttempt struct { ID string `json:"id"` Status string `json:"status"` DeliveredAt time.Time `json:"delivered_at"` Code string `json:"code,omitempty"` ResponseData map[string]interface{} `json:"response_data,omitempty"` - Attempt int `json:"attempt"` + AttemptNumber int `json:"attempt_number"` Manual bool `json:"manual"` // Expandable fields - string (ID) or object depending on expand @@ -131,9 +130,9 @@ type APIEvent struct { Data map[string]interface{} `json:"data,omitempty"` } -// DeliveryPaginatedResult is the paginated response for listing deliveries. -type DeliveryPaginatedResult struct { - Models []APIDelivery `json:"models"` +// AttemptPaginatedResult is the paginated response for listing attempts. +type AttemptPaginatedResult struct { + Models []APIAttempt `json:"models"` Pagination SeekPagination `json:"pagination"` } @@ -143,43 +142,47 @@ type EventPaginatedResult struct { Pagination SeekPagination `json:"pagination"` } -// toAPIDelivery converts a DeliveryEvent to APIDelivery with expand options -func toAPIDelivery(de *models.DeliveryEvent, opts IncludeOptions) APIDelivery { - api := APIDelivery{ - Attempt: de.Attempt, - Manual: de.Manual, - Destination: de.DestinationID, +// toAPIAttempt converts an AttemptRecord to APIAttempt with expand options +func toAPIAttempt(ar *logstore.AttemptRecord, opts IncludeOptions) APIAttempt { + api := APIAttempt{ + AttemptNumber: ar.Attempt.AttemptNumber, + Manual: ar.Attempt.Manual, + Destination: ar.Attempt.DestinationID, } - if de.Delivery != nil { - api.ID = de.Delivery.ID - api.Status = de.Delivery.Status - api.DeliveredAt = de.Delivery.Time - api.Code = de.Delivery.Code + if ar.Attempt != nil { + api.ID = ar.Attempt.ID + api.Status = ar.Attempt.Status + api.DeliveredAt = ar.Attempt.Time + api.Code = ar.Attempt.Code if opts.ResponseData { - api.ResponseData = de.Delivery.ResponseData + api.ResponseData = ar.Attempt.ResponseData } } - if opts.EventData { - api.Event = APIEventFull{ - ID: de.Event.ID, - Topic: de.Event.Topic, - Time: de.Event.Time, - EligibleForRetry: de.Event.EligibleForRetry, - Metadata: de.Event.Metadata, - Data: de.Event.Data, - } - } else if opts.Event { - api.Event = APIEventSummary{ - ID: de.Event.ID, - Topic: de.Event.Topic, - Time: de.Event.Time, - EligibleForRetry: de.Event.EligibleForRetry, - Metadata: de.Event.Metadata, + if ar.Event != nil { + if opts.EventData { + api.Event = APIEventFull{ + ID: ar.Event.ID, + Topic: ar.Event.Topic, + Time: ar.Event.Time, + EligibleForRetry: ar.Event.EligibleForRetry, + Metadata: ar.Event.Metadata, + Data: ar.Event.Data, + } + } else if opts.Event { + api.Event = APIEventSummary{ + ID: ar.Event.ID, + Topic: ar.Event.Topic, + Time: ar.Event.Time, + EligibleForRetry: ar.Event.EligibleForRetry, + Metadata: ar.Event.Metadata, + } + } else { + api.Event = ar.Event.ID } } else { - api.Event = de.Event.ID + api.Event = ar.Attempt.EventID } // TODO: Handle destination expansion @@ -190,17 +193,28 @@ func toAPIDelivery(de *models.DeliveryEvent, opts IncludeOptions) APIDelivery { return api } -// ListDeliveries handles GET /:tenantID/deliveries +// ListAttempts handles GET /:tenantID/attempts // Query params: event_id, destination_id, status, topic[], start, end, limit, next, prev, expand[], sort_order -func (h *LogHandlers) ListDeliveries(c *gin.Context) { +func (h *LogHandlers) ListAttempts(c *gin.Context) { + tenant := mustTenantFromContext(c) + if tenant == nil { + return + } + h.listAttemptsInternal(c, tenant.ID, "") +} + +// ListDestinationAttempts handles GET /:tenantID/destinations/:destinationID/attempts +// Same as ListAttempts but scoped to a specific destination via URL param. +func (h *LogHandlers) ListDestinationAttempts(c *gin.Context) { tenant := mustTenantFromContext(c) if tenant == nil { return } - h.listDeliveriesInternal(c, tenant.ID) + destinationID := c.Param("destinationID") + h.listAttemptsInternal(c, tenant.ID, destinationID) } -func (h *LogHandlers) listDeliveriesInternal(c *gin.Context, tenantID string) { +func (h *LogHandlers) listAttemptsInternal(c *gin.Context, tenantID string, destinationID string) { // Parse and validate cursors (next/prev are mutually exclusive) cursors, errResp := ParseCursors(c) if errResp != nil { @@ -231,7 +245,7 @@ func (h *LogHandlers) listDeliveriesInternal(c *gin.Context, tenantID string) { _ = orderBy // Parse time date filters - deliveryTimeFilter, errResp := ParseDateFilter(c, "time") + attemptTimeFilter, errResp := ParseDateFilter(c, "time") if errResp != nil { AbortWithError(c, errResp.Code, *errResp) return @@ -240,21 +254,23 @@ func (h *LogHandlers) listDeliveriesInternal(c *gin.Context, tenantID string) { limit := parseLimit(c, 100, 1000) var destinationIDs []string - if destID := c.Query("destination_id"); destID != "" { + if destinationID != "" { + destinationIDs = []string{destinationID} + } else if destID := c.Query("destination_id"); destID != "" { destinationIDs = []string{destID} } - req := logstore.ListDeliveryEventRequest{ + req := logstore.ListAttemptRequest{ TenantID: tenantID, EventID: c.Query("event_id"), DestinationIDs: destinationIDs, Status: c.Query("status"), Topics: parseQueryArray(c, "topic"), TimeFilter: logstore.TimeFilter{ - GTE: deliveryTimeFilter.GTE, - LTE: deliveryTimeFilter.LTE, - GT: deliveryTimeFilter.GT, - LT: deliveryTimeFilter.LT, + GTE: attemptTimeFilter.GTE, + LTE: attemptTimeFilter.LTE, + GT: attemptTimeFilter.GT, + LT: attemptTimeFilter.LT, }, Limit: limit, Next: cursors.Next, @@ -262,7 +278,7 @@ func (h *LogHandlers) listDeliveriesInternal(c *gin.Context, tenantID string) { SortOrder: dir, } - response, err := h.logStore.ListDeliveryEvent(c.Request.Context(), req) + response, err := h.logStore.ListAttempt(c.Request.Context(), req) if err != nil { if errors.Is(err, cursor.ErrInvalidCursor) || errors.Is(err, cursor.ErrVersionMismatch) { AbortWithError(c, http.StatusBadRequest, NewErrBadRequest(err)) @@ -274,13 +290,13 @@ func (h *LogHandlers) listDeliveriesInternal(c *gin.Context, tenantID string) { includeOpts := parseIncludeOptions(c) - apiDeliveries := make([]APIDelivery, len(response.Data)) - for i, de := range response.Data { - apiDeliveries[i] = toAPIDelivery(de, includeOpts) + apiAttempts := make([]APIAttempt, len(response.Data)) + for i, ar := range response.Data { + apiAttempts[i] = toAPIAttempt(ar, includeOpts) } - c.JSON(http.StatusOK, DeliveryPaginatedResult{ - Models: apiDeliveries, + c.JSON(http.StatusOK, AttemptPaginatedResult{ + Models: apiAttempts, Pagination: SeekPagination{ OrderBy: orderBy, Dir: dir, @@ -320,30 +336,30 @@ func (h *LogHandlers) RetrieveEvent(c *gin.Context) { }) } -// RetrieveDelivery handles GET /:tenantID/deliveries/:deliveryID -func (h *LogHandlers) RetrieveDelivery(c *gin.Context) { +// RetrieveAttempt handles GET /:tenantID/attempts/:attemptID +func (h *LogHandlers) RetrieveAttempt(c *gin.Context) { tenant := mustTenantFromContext(c) if tenant == nil { return } - deliveryID := c.Param("deliveryID") + attemptID := c.Param("attemptID") - deliveryEvent, err := h.logStore.RetrieveDeliveryEvent(c.Request.Context(), logstore.RetrieveDeliveryEventRequest{ - TenantID: tenant.ID, - DeliveryID: deliveryID, + attemptRecord, err := h.logStore.RetrieveAttempt(c.Request.Context(), logstore.RetrieveAttemptRequest{ + TenantID: tenant.ID, + AttemptID: attemptID, }) if err != nil { AbortWithError(c, http.StatusInternalServerError, NewErrInternalServer(err)) return } - if deliveryEvent == nil { - AbortWithError(c, http.StatusNotFound, NewErrNotFound("delivery")) + if attemptRecord == nil { + AbortWithError(c, http.StatusNotFound, NewErrNotFound("attempt")) return } includeOpts := parseIncludeOptions(c) - c.JSON(http.StatusOK, toAPIDelivery(deliveryEvent, includeOpts)) + c.JSON(http.StatusOK, toAPIAttempt(attemptRecord, includeOpts)) } // AdminListEvents handles GET /events (admin-only, cross-tenant) @@ -352,10 +368,10 @@ func (h *LogHandlers) AdminListEvents(c *gin.Context) { h.listEventsInternal(c, c.Query("tenant_id")) } -// AdminListDeliveries handles GET /deliveries (admin-only, cross-tenant) +// AdminListAttempts handles GET /attempts (admin-only, cross-tenant) // Query params: tenant_id (optional), event_id, destination_id, status, topic[], start, end, limit, next, prev, expand[], sort_order -func (h *LogHandlers) AdminListDeliveries(c *gin.Context) { - h.listDeliveriesInternal(c, c.Query("tenant_id")) +func (h *LogHandlers) AdminListAttempts(c *gin.Context) { + h.listAttemptsInternal(c, c.Query("tenant_id"), "") } // ListEvents handles GET /:tenantID/events diff --git a/internal/apirouter/log_handlers_test.go b/internal/apirouter/log_handlers_test.go index 3bfc69ac..431e9fdb 100644 --- a/internal/apirouter/log_handlers_test.go +++ b/internal/apirouter/log_handlers_test.go @@ -3,7 +3,6 @@ package apirouter_test import ( "context" "encoding/json" - "fmt" "net/http" "net/http/httptest" "testing" @@ -16,7 +15,7 @@ import ( "github.com/stretchr/testify/require" ) -func TestListDeliveries(t *testing.T) { +func TestListAttempts(t *testing.T) { t.Parallel() result := setupTestRouterFull(t, "", "") @@ -36,9 +35,9 @@ func TestListDeliveries(t *testing.T) { CreatedAt: time.Now(), })) - t.Run("should return empty list when no deliveries", func(t *testing.T) { + t.Run("should return empty list when no attempts", func(t *testing.T) { w := httptest.NewRecorder() - req, _ := http.NewRequest("GET", baseAPIPath+"/tenants/"+tenantID+"/deliveries", nil) + req, _ := http.NewRequest("GET", baseAPIPath+"/tenants/"+tenantID+"/attempts", nil) result.router.ServeHTTP(w, req) assert.Equal(t, http.StatusOK, w.Code) @@ -50,12 +49,12 @@ func TestListDeliveries(t *testing.T) { assert.Len(t, data, 0) }) - t.Run("should list deliveries", func(t *testing.T) { - // Seed delivery events + t.Run("should list attempts", func(t *testing.T) { + // Seed attempt events eventID := idgen.Event() - deliveryID := idgen.Delivery() + attemptID := idgen.Attempt() eventTime := time.Now().Add(-1 * time.Hour).Truncate(time.Millisecond) - deliveryTime := eventTime.Add(100 * time.Millisecond) + attemptTime := eventTime.Add(100 * time.Millisecond) event := testutil.EventFactory.AnyPointer( testutil.EventFactory.WithID(eventID), @@ -65,25 +64,18 @@ func TestListDeliveries(t *testing.T) { testutil.EventFactory.WithTime(eventTime), ) - delivery := testutil.DeliveryFactory.AnyPointer( - testutil.DeliveryFactory.WithID(deliveryID), - testutil.DeliveryFactory.WithEventID(eventID), - testutil.DeliveryFactory.WithDestinationID(destinationID), - testutil.DeliveryFactory.WithStatus("success"), - testutil.DeliveryFactory.WithTime(deliveryTime), + attempt := testutil.AttemptFactory.AnyPointer( + testutil.AttemptFactory.WithID(attemptID), + testutil.AttemptFactory.WithEventID(eventID), + testutil.AttemptFactory.WithDestinationID(destinationID), + testutil.AttemptFactory.WithStatus("success"), + testutil.AttemptFactory.WithTime(attemptTime), ) - de := &models.DeliveryEvent{ - ID: fmt.Sprintf("%s_%s", eventID, deliveryID), - DestinationID: destinationID, - Event: *event, - Delivery: delivery, - } - - require.NoError(t, result.logStore.InsertManyDeliveryEvent(context.Background(), []*models.DeliveryEvent{de})) + require.NoError(t, result.logStore.InsertMany(context.Background(), []*models.LogEntry{{Event: event, Attempt: attempt}})) w := httptest.NewRecorder() - req, _ := http.NewRequest("GET", baseAPIPath+"/tenants/"+tenantID+"/deliveries", nil) + req, _ := http.NewRequest("GET", baseAPIPath+"/tenants/"+tenantID+"/attempts", nil) result.router.ServeHTTP(w, req) assert.Equal(t, http.StatusOK, w.Code) @@ -94,16 +86,16 @@ func TestListDeliveries(t *testing.T) { data := response["models"].([]interface{}) assert.Len(t, data, 1) - firstDelivery := data[0].(map[string]interface{}) - assert.Equal(t, deliveryID, firstDelivery["id"]) - assert.Equal(t, "success", firstDelivery["status"]) - assert.Equal(t, eventID, firstDelivery["event"]) // Not included - assert.Equal(t, destinationID, firstDelivery["destination"]) + firstAttempt := data[0].(map[string]interface{}) + assert.Equal(t, attemptID, firstAttempt["id"]) + assert.Equal(t, "success", firstAttempt["status"]) + assert.Equal(t, eventID, firstAttempt["event"]) // Not included + assert.Equal(t, destinationID, firstAttempt["destination"]) }) t.Run("should include event when include=event", func(t *testing.T) { w := httptest.NewRecorder() - req, _ := http.NewRequest("GET", baseAPIPath+"/tenants/"+tenantID+"/deliveries?include=event", nil) + req, _ := http.NewRequest("GET", baseAPIPath+"/tenants/"+tenantID+"/attempts?include=event", nil) result.router.ServeHTTP(w, req) assert.Equal(t, http.StatusOK, w.Code) @@ -114,8 +106,8 @@ func TestListDeliveries(t *testing.T) { data := response["models"].([]interface{}) require.Len(t, data, 1) - firstDelivery := data[0].(map[string]interface{}) - event := firstDelivery["event"].(map[string]interface{}) + firstAttempt := data[0].(map[string]interface{}) + event := firstAttempt["event"].(map[string]interface{}) assert.NotNil(t, event["id"]) assert.Equal(t, "user.created", event["topic"]) // data should not be present without include=event.data @@ -124,7 +116,7 @@ func TestListDeliveries(t *testing.T) { t.Run("should include event.data when include=event.data", func(t *testing.T) { w := httptest.NewRecorder() - req, _ := http.NewRequest("GET", baseAPIPath+"/tenants/"+tenantID+"/deliveries?include=event.data", nil) + req, _ := http.NewRequest("GET", baseAPIPath+"/tenants/"+tenantID+"/attempts?include=event.data", nil) result.router.ServeHTTP(w, req) assert.Equal(t, http.StatusOK, w.Code) @@ -135,15 +127,15 @@ func TestListDeliveries(t *testing.T) { data := response["models"].([]interface{}) require.Len(t, data, 1) - firstDelivery := data[0].(map[string]interface{}) - event := firstDelivery["event"].(map[string]interface{}) + firstAttempt := data[0].(map[string]interface{}) + event := firstAttempt["event"].(map[string]interface{}) assert.NotNil(t, event["id"]) assert.NotNil(t, event["data"]) // data should be present }) t.Run("should filter by destination_id", func(t *testing.T) { w := httptest.NewRecorder() - req, _ := http.NewRequest("GET", baseAPIPath+"/tenants/"+tenantID+"/deliveries?destination_id="+destinationID, nil) + req, _ := http.NewRequest("GET", baseAPIPath+"/tenants/"+tenantID+"/attempts?destination_id="+destinationID, nil) result.router.ServeHTTP(w, req) assert.Equal(t, http.StatusOK, w.Code) @@ -157,7 +149,7 @@ func TestListDeliveries(t *testing.T) { t.Run("should filter by non-existent destination_id", func(t *testing.T) { w := httptest.NewRecorder() - req, _ := http.NewRequest("GET", baseAPIPath+"/tenants/"+tenantID+"/deliveries?destination_id=nonexistent", nil) + req, _ := http.NewRequest("GET", baseAPIPath+"/tenants/"+tenantID+"/attempts?destination_id=nonexistent", nil) result.router.ServeHTTP(w, req) assert.Equal(t, http.StatusOK, w.Code) @@ -171,7 +163,7 @@ func TestListDeliveries(t *testing.T) { t.Run("should return 404 for non-existent tenant", func(t *testing.T) { w := httptest.NewRecorder() - req, _ := http.NewRequest("GET", baseAPIPath+"/tenants/nonexistent/deliveries", nil) + req, _ := http.NewRequest("GET", baseAPIPath+"/tenants/nonexistent/attempts", nil) result.router.ServeHTTP(w, req) assert.Equal(t, http.StatusNotFound, w.Code) @@ -179,7 +171,7 @@ func TestListDeliveries(t *testing.T) { t.Run("should exclude response_data by default", func(t *testing.T) { w := httptest.NewRecorder() - req, _ := http.NewRequest("GET", baseAPIPath+"/tenants/"+tenantID+"/deliveries", nil) + req, _ := http.NewRequest("GET", baseAPIPath+"/tenants/"+tenantID+"/attempts", nil) result.router.ServeHTTP(w, req) assert.Equal(t, http.StatusOK, w.Code) @@ -190,16 +182,16 @@ func TestListDeliveries(t *testing.T) { data := response["models"].([]interface{}) require.Len(t, data, 1) - firstDelivery := data[0].(map[string]interface{}) - assert.Nil(t, firstDelivery["response_data"]) + firstAttempt := data[0].(map[string]interface{}) + assert.Nil(t, firstAttempt["response_data"]) }) t.Run("should include response_data with include=response_data", func(t *testing.T) { - // Seed a delivery with response_data + // Seed an attempt with response_data eventID := idgen.Event() - deliveryID := idgen.Delivery() + attemptID := idgen.Attempt() eventTime := time.Now().Add(-30 * time.Minute).Truncate(time.Millisecond) - deliveryTime := eventTime.Add(100 * time.Millisecond) + attemptTime := eventTime.Add(100 * time.Millisecond) event := testutil.EventFactory.AnyPointer( testutil.EventFactory.WithID(eventID), @@ -209,29 +201,22 @@ func TestListDeliveries(t *testing.T) { testutil.EventFactory.WithTime(eventTime), ) - delivery := testutil.DeliveryFactory.AnyPointer( - testutil.DeliveryFactory.WithID(deliveryID), - testutil.DeliveryFactory.WithEventID(eventID), - testutil.DeliveryFactory.WithDestinationID(destinationID), - testutil.DeliveryFactory.WithStatus("success"), - testutil.DeliveryFactory.WithTime(deliveryTime), + attempt := testutil.AttemptFactory.AnyPointer( + testutil.AttemptFactory.WithID(attemptID), + testutil.AttemptFactory.WithEventID(eventID), + testutil.AttemptFactory.WithDestinationID(destinationID), + testutil.AttemptFactory.WithStatus("success"), + testutil.AttemptFactory.WithTime(attemptTime), ) - delivery.ResponseData = map[string]interface{}{ + attempt.ResponseData = map[string]interface{}{ "body": "OK", "status": float64(200), } - de := &models.DeliveryEvent{ - ID: fmt.Sprintf("%s_%s", eventID, deliveryID), - DestinationID: destinationID, - Event: *event, - Delivery: delivery, - } - - require.NoError(t, result.logStore.InsertManyDeliveryEvent(context.Background(), []*models.DeliveryEvent{de})) + require.NoError(t, result.logStore.InsertMany(context.Background(), []*models.LogEntry{{Event: event, Attempt: attempt}})) w := httptest.NewRecorder() - req, _ := http.NewRequest("GET", baseAPIPath+"/tenants/"+tenantID+"/deliveries?include=response_data", nil) + req, _ := http.NewRequest("GET", baseAPIPath+"/tenants/"+tenantID+"/attempts?include=response_data", nil) result.router.ServeHTTP(w, req) assert.Equal(t, http.StatusOK, w.Code) @@ -240,25 +225,25 @@ func TestListDeliveries(t *testing.T) { require.NoError(t, json.Unmarshal(w.Body.Bytes(), &response)) data := response["models"].([]interface{}) - // Find the delivery we just created - var foundDelivery map[string]interface{} + // Find the attempt we just created + var foundAttempt map[string]interface{} for _, d := range data { - del := d.(map[string]interface{}) - if del["id"] == deliveryID { - foundDelivery = del + atm := d.(map[string]interface{}) + if atm["id"] == attemptID { + foundAttempt = atm break } } - require.NotNil(t, foundDelivery, "delivery not found in response") - require.NotNil(t, foundDelivery["response_data"], "response_data should be included") - respData := foundDelivery["response_data"].(map[string]interface{}) + require.NotNil(t, foundAttempt, "attempt not found in response") + require.NotNil(t, foundAttempt["response_data"], "response_data should be included") + respData := foundAttempt["response_data"].(map[string]interface{}) assert.Equal(t, "OK", respData["body"]) assert.Equal(t, float64(200), respData["status"]) }) t.Run("should support comma-separated include param", func(t *testing.T) { w := httptest.NewRecorder() - req, _ := http.NewRequest("GET", baseAPIPath+"/tenants/"+tenantID+"/deliveries?include=event,response_data", nil) + req, _ := http.NewRequest("GET", baseAPIPath+"/tenants/"+tenantID+"/attempts?include=event,response_data", nil) result.router.ServeHTTP(w, req) assert.Equal(t, http.StatusOK, w.Code) @@ -269,16 +254,16 @@ func TestListDeliveries(t *testing.T) { data := response["models"].([]interface{}) require.GreaterOrEqual(t, len(data), 1) - firstDelivery := data[0].(map[string]interface{}) + firstAttempt := data[0].(map[string]interface{}) // event should be included (object, not string) - event := firstDelivery["event"].(map[string]interface{}) + event := firstAttempt["event"].(map[string]interface{}) assert.NotNil(t, event["id"]) assert.NotNil(t, event["topic"]) }) t.Run("should return validation error for invalid dir", func(t *testing.T) { w := httptest.NewRecorder() - req, _ := http.NewRequest("GET", baseAPIPath+"/tenants/"+tenantID+"/deliveries?dir=invalid", nil) + req, _ := http.NewRequest("GET", baseAPIPath+"/tenants/"+tenantID+"/attempts?dir=invalid", nil) result.router.ServeHTTP(w, req) assert.Equal(t, http.StatusUnprocessableEntity, w.Code) @@ -286,7 +271,7 @@ func TestListDeliveries(t *testing.T) { t.Run("should accept valid dir param", func(t *testing.T) { w := httptest.NewRecorder() - req, _ := http.NewRequest("GET", baseAPIPath+"/tenants/"+tenantID+"/deliveries?dir=asc", nil) + req, _ := http.NewRequest("GET", baseAPIPath+"/tenants/"+tenantID+"/attempts?dir=asc", nil) result.router.ServeHTTP(w, req) assert.Equal(t, http.StatusOK, w.Code) @@ -294,7 +279,7 @@ func TestListDeliveries(t *testing.T) { t.Run("should cap limit at 1000", func(t *testing.T) { w := httptest.NewRecorder() - req, _ := http.NewRequest("GET", baseAPIPath+"/tenants/"+tenantID+"/deliveries?limit=5000", nil) + req, _ := http.NewRequest("GET", baseAPIPath+"/tenants/"+tenantID+"/attempts?limit=5000", nil) result.router.ServeHTTP(w, req) // Should succeed, limit is silently capped @@ -302,7 +287,7 @@ func TestListDeliveries(t *testing.T) { }) } -func TestRetrieveDelivery(t *testing.T) { +func TestRetrieveAttempt(t *testing.T) { t.Parallel() result := setupTestRouterFull(t, "", "") @@ -322,11 +307,11 @@ func TestRetrieveDelivery(t *testing.T) { CreatedAt: time.Now(), })) - // Seed a delivery event + // Seed an attempt event eventID := idgen.Event() - deliveryID := idgen.Delivery() + attemptID := idgen.Attempt() eventTime := time.Now().Add(-1 * time.Hour).Truncate(time.Millisecond) - deliveryTime := eventTime.Add(100 * time.Millisecond) + attemptTime := eventTime.Add(100 * time.Millisecond) event := testutil.EventFactory.AnyPointer( testutil.EventFactory.WithID(eventID), @@ -336,26 +321,19 @@ func TestRetrieveDelivery(t *testing.T) { testutil.EventFactory.WithTime(eventTime), ) - delivery := testutil.DeliveryFactory.AnyPointer( - testutil.DeliveryFactory.WithID(deliveryID), - testutil.DeliveryFactory.WithEventID(eventID), - testutil.DeliveryFactory.WithDestinationID(destinationID), - testutil.DeliveryFactory.WithStatus("failed"), - testutil.DeliveryFactory.WithTime(deliveryTime), + attempt := testutil.AttemptFactory.AnyPointer( + testutil.AttemptFactory.WithID(attemptID), + testutil.AttemptFactory.WithEventID(eventID), + testutil.AttemptFactory.WithDestinationID(destinationID), + testutil.AttemptFactory.WithStatus("failed"), + testutil.AttemptFactory.WithTime(attemptTime), ) - de := &models.DeliveryEvent{ - ID: fmt.Sprintf("%s_%s", eventID, deliveryID), - DestinationID: destinationID, - Event: *event, - Delivery: delivery, - } + require.NoError(t, result.logStore.InsertMany(context.Background(), []*models.LogEntry{{Event: event, Attempt: attempt}})) - require.NoError(t, result.logStore.InsertManyDeliveryEvent(context.Background(), []*models.DeliveryEvent{de})) - - t.Run("should retrieve delivery by ID", func(t *testing.T) { + t.Run("should retrieve attempt by ID", func(t *testing.T) { w := httptest.NewRecorder() - req, _ := http.NewRequest("GET", baseAPIPath+"/tenants/"+tenantID+"/deliveries/"+deliveryID, nil) + req, _ := http.NewRequest("GET", baseAPIPath+"/tenants/"+tenantID+"/attempts/"+attemptID, nil) result.router.ServeHTTP(w, req) assert.Equal(t, http.StatusOK, w.Code) @@ -363,7 +341,7 @@ func TestRetrieveDelivery(t *testing.T) { var response map[string]interface{} require.NoError(t, json.Unmarshal(w.Body.Bytes(), &response)) - assert.Equal(t, deliveryID, response["id"]) + assert.Equal(t, attemptID, response["id"]) assert.Equal(t, "failed", response["status"]) assert.Equal(t, eventID, response["event"]) // Not included assert.Equal(t, destinationID, response["destination"]) @@ -371,7 +349,7 @@ func TestRetrieveDelivery(t *testing.T) { t.Run("should include event when include=event", func(t *testing.T) { w := httptest.NewRecorder() - req, _ := http.NewRequest("GET", baseAPIPath+"/tenants/"+tenantID+"/deliveries/"+deliveryID+"?include=event", nil) + req, _ := http.NewRequest("GET", baseAPIPath+"/tenants/"+tenantID+"/attempts/"+attemptID+"?include=event", nil) result.router.ServeHTTP(w, req) assert.Equal(t, http.StatusOK, w.Code) @@ -388,7 +366,7 @@ func TestRetrieveDelivery(t *testing.T) { t.Run("should include event.data when include=event.data", func(t *testing.T) { w := httptest.NewRecorder() - req, _ := http.NewRequest("GET", baseAPIPath+"/tenants/"+tenantID+"/deliveries/"+deliveryID+"?include=event.data", nil) + req, _ := http.NewRequest("GET", baseAPIPath+"/tenants/"+tenantID+"/attempts/"+attemptID+"?include=event.data", nil) result.router.ServeHTTP(w, req) assert.Equal(t, http.StatusOK, w.Code) @@ -401,9 +379,9 @@ func TestRetrieveDelivery(t *testing.T) { assert.NotNil(t, event["data"]) // data should be present }) - t.Run("should return 404 for non-existent delivery", func(t *testing.T) { + t.Run("should return 404 for non-existent attempt", func(t *testing.T) { w := httptest.NewRecorder() - req, _ := http.NewRequest("GET", baseAPIPath+"/tenants/"+tenantID+"/deliveries/nonexistent", nil) + req, _ := http.NewRequest("GET", baseAPIPath+"/tenants/"+tenantID+"/attempts/nonexistent", nil) result.router.ServeHTTP(w, req) assert.Equal(t, http.StatusNotFound, w.Code) @@ -411,7 +389,7 @@ func TestRetrieveDelivery(t *testing.T) { t.Run("should return 404 for non-existent tenant", func(t *testing.T) { w := httptest.NewRecorder() - req, _ := http.NewRequest("GET", baseAPIPath+"/tenants/nonexistent/deliveries/"+deliveryID, nil) + req, _ := http.NewRequest("GET", baseAPIPath+"/tenants/nonexistent/attempts/"+attemptID, nil) result.router.ServeHTTP(w, req) assert.Equal(t, http.StatusNotFound, w.Code) @@ -438,11 +416,11 @@ func TestRetrieveEvent(t *testing.T) { CreatedAt: time.Now(), })) - // Seed a delivery event + // Seed an attempt event eventID := idgen.Event() - deliveryID := idgen.Delivery() + attemptID := idgen.Attempt() eventTime := time.Now().Add(-1 * time.Hour).Truncate(time.Millisecond) - deliveryTime := eventTime.Add(100 * time.Millisecond) + attemptTime := eventTime.Add(100 * time.Millisecond) event := testutil.EventFactory.AnyPointer( testutil.EventFactory.WithID(eventID), @@ -458,22 +436,15 @@ func TestRetrieveEvent(t *testing.T) { }), ) - delivery := testutil.DeliveryFactory.AnyPointer( - testutil.DeliveryFactory.WithID(deliveryID), - testutil.DeliveryFactory.WithEventID(eventID), - testutil.DeliveryFactory.WithDestinationID(destinationID), - testutil.DeliveryFactory.WithStatus("success"), - testutil.DeliveryFactory.WithTime(deliveryTime), + attempt := testutil.AttemptFactory.AnyPointer( + testutil.AttemptFactory.WithID(attemptID), + testutil.AttemptFactory.WithEventID(eventID), + testutil.AttemptFactory.WithDestinationID(destinationID), + testutil.AttemptFactory.WithStatus("success"), + testutil.AttemptFactory.WithTime(attemptTime), ) - de := &models.DeliveryEvent{ - ID: fmt.Sprintf("%s_%s", eventID, deliveryID), - DestinationID: destinationID, - Event: *event, - Delivery: delivery, - } - - require.NoError(t, result.logStore.InsertManyDeliveryEvent(context.Background(), []*models.DeliveryEvent{de})) + require.NoError(t, result.logStore.InsertMany(context.Background(), []*models.LogEntry{{Event: event, Attempt: attempt}})) t.Run("should retrieve event by ID", func(t *testing.T) { w := httptest.NewRecorder() @@ -545,11 +516,11 @@ func TestListEvents(t *testing.T) { }) t.Run("should list events", func(t *testing.T) { - // Seed delivery events + // Seed attempt events eventID := idgen.Event() - deliveryID := idgen.Delivery() + attemptID := idgen.Attempt() eventTime := time.Now().Add(-1 * time.Hour).Truncate(time.Millisecond) - deliveryTime := eventTime.Add(100 * time.Millisecond) + attemptTime := eventTime.Add(100 * time.Millisecond) event := testutil.EventFactory.AnyPointer( testutil.EventFactory.WithID(eventID), @@ -562,22 +533,15 @@ func TestListEvents(t *testing.T) { }), ) - delivery := testutil.DeliveryFactory.AnyPointer( - testutil.DeliveryFactory.WithID(deliveryID), - testutil.DeliveryFactory.WithEventID(eventID), - testutil.DeliveryFactory.WithDestinationID(destinationID), - testutil.DeliveryFactory.WithStatus("success"), - testutil.DeliveryFactory.WithTime(deliveryTime), + attempt := testutil.AttemptFactory.AnyPointer( + testutil.AttemptFactory.WithID(attemptID), + testutil.AttemptFactory.WithEventID(eventID), + testutil.AttemptFactory.WithDestinationID(destinationID), + testutil.AttemptFactory.WithStatus("success"), + testutil.AttemptFactory.WithTime(attemptTime), ) - de := &models.DeliveryEvent{ - ID: fmt.Sprintf("%s_%s", eventID, deliveryID), - DestinationID: destinationID, - Event: *event, - Delivery: delivery, - } - - require.NoError(t, result.logStore.InsertManyDeliveryEvent(context.Background(), []*models.DeliveryEvent{de})) + require.NoError(t, result.logStore.InsertMany(context.Background(), []*models.LogEntry{{Event: event, Attempt: attempt}})) w := httptest.NewRecorder() req, _ := http.NewRequest("GET", baseAPIPath+"/tenants/"+tenantID+"/events", nil) diff --git a/internal/apirouter/logger_middleware_integration_test.go b/internal/apirouter/logger_middleware_integration_test.go index 2d44ea86..392b1f94 100644 --- a/internal/apirouter/logger_middleware_integration_test.go +++ b/internal/apirouter/logger_middleware_integration_test.go @@ -65,7 +65,7 @@ func (r *mockRegistry) CreatePublisher(ctx context.Context, destination *models. return nil, fmt.Errorf("not implemented") } -func (r *mockRegistry) PublishEvent(ctx context.Context, destination *models.Destination, event *models.Event) (*models.Delivery, error) { +func (r *mockRegistry) PublishEvent(ctx context.Context, destination *models.Destination, event *models.Event) (*models.Attempt, error) { return nil, fmt.Errorf("not implemented") } diff --git a/internal/apirouter/retry_handlers.go b/internal/apirouter/retry_handlers.go index 1fa2bd5a..f52f99d3 100644 --- a/internal/apirouter/retry_handlers.go +++ b/internal/apirouter/retry_handlers.go @@ -32,33 +32,33 @@ func NewRetryHandlers( } } -// RetryDelivery handles POST /:tenantID/deliveries/:deliveryID/retry +// RetryAttempt handles POST /:tenantID/attempts/:attemptID/retry // Constraints: -// - Only the latest delivery for an event+destination pair can be retried +// - Only the latest attempt for an event+destination pair can be retried // - Destination must exist and be enabled -func (h *RetryHandlers) RetryDelivery(c *gin.Context) { +func (h *RetryHandlers) RetryAttempt(c *gin.Context) { tenant := mustTenantFromContext(c) if tenant == nil { return } - deliveryID := c.Param("deliveryID") + attemptID := c.Param("attemptID") - // 1. Look up delivery by ID - deliveryEvent, err := h.logStore.RetrieveDeliveryEvent(c.Request.Context(), logstore.RetrieveDeliveryEventRequest{ - TenantID: tenant.ID, - DeliveryID: deliveryID, + // 1. Look up attempt by ID + attemptRecord, err := h.logStore.RetrieveAttempt(c.Request.Context(), logstore.RetrieveAttemptRequest{ + TenantID: tenant.ID, + AttemptID: attemptID, }) if err != nil { AbortWithError(c, http.StatusInternalServerError, NewErrInternalServer(err)) return } - if deliveryEvent == nil { - AbortWithError(c, http.StatusNotFound, NewErrNotFound("delivery")) + if attemptRecord == nil { + AbortWithError(c, http.StatusNotFound, NewErrNotFound("attempt")) return } // 2. Check destination exists and is enabled - destination, err := h.entityStore.RetrieveDestination(c.Request.Context(), tenant.ID, deliveryEvent.DestinationID) + destination, err := h.entityStore.RetrieveDestination(c.Request.Context(), tenant.ID, attemptRecord.Attempt.DestinationID) if err != nil { AbortWithError(c, http.StatusInternalServerError, NewErrInternalServer(err)) return @@ -78,19 +78,19 @@ func (h *RetryHandlers) RetryDelivery(c *gin.Context) { return } - // 3. Create and publish retry delivery event - retryDeliveryEvent := models.NewManualDeliveryEvent(deliveryEvent.Event, deliveryEvent.DestinationID) + // 3. Create and publish manual delivery task + task := models.NewManualDeliveryTask(*attemptRecord.Event, attemptRecord.Attempt.DestinationID) - if err := h.deliveryMQ.Publish(c.Request.Context(), retryDeliveryEvent); err != nil { + if err := h.deliveryMQ.Publish(c.Request.Context(), task); err != nil { AbortWithError(c, http.StatusInternalServerError, NewErrInternalServer(err)) return } h.logger.Ctx(c.Request.Context()).Audit("manual retry initiated", - zap.String("delivery_id", deliveryID), - zap.String("event_id", deliveryEvent.Event.ID), + zap.String("attempt_id", attemptID), + zap.String("event_id", attemptRecord.Event.ID), zap.String("tenant_id", tenant.ID), - zap.String("destination_id", deliveryEvent.DestinationID), + zap.String("destination_id", attemptRecord.Attempt.DestinationID), zap.String("destination_type", destination.Type)) c.JSON(http.StatusAccepted, gin.H{ diff --git a/internal/apirouter/retry_handlers_test.go b/internal/apirouter/retry_handlers_test.go index 486b1274..74c7c65e 100644 --- a/internal/apirouter/retry_handlers_test.go +++ b/internal/apirouter/retry_handlers_test.go @@ -3,7 +3,6 @@ package apirouter_test import ( "context" "encoding/json" - "fmt" "net/http" "net/http/httptest" "testing" @@ -16,7 +15,7 @@ import ( "github.com/stretchr/testify/require" ) -func TestRetryDelivery(t *testing.T) { +func TestRetryAttempt(t *testing.T) { t.Parallel() result := setupTestRouterFull(t, "", "") @@ -36,11 +35,11 @@ func TestRetryDelivery(t *testing.T) { CreatedAt: time.Now(), })) - // Seed a delivery event + // Seed an attempt event eventID := idgen.Event() - deliveryID := idgen.Delivery() + attemptID := idgen.Attempt() eventTime := time.Now().Add(-1 * time.Hour).Truncate(time.Millisecond) - deliveryTime := eventTime.Add(100 * time.Millisecond) + attemptTime := eventTime.Add(100 * time.Millisecond) event := testutil.EventFactory.AnyPointer( testutil.EventFactory.WithID(eventID), @@ -50,26 +49,27 @@ func TestRetryDelivery(t *testing.T) { testutil.EventFactory.WithTime(eventTime), ) - delivery := testutil.DeliveryFactory.AnyPointer( - testutil.DeliveryFactory.WithID(deliveryID), - testutil.DeliveryFactory.WithEventID(eventID), - testutil.DeliveryFactory.WithDestinationID(destinationID), - testutil.DeliveryFactory.WithStatus("failed"), - testutil.DeliveryFactory.WithTime(deliveryTime), + attempt := testutil.AttemptFactory.AnyPointer( + testutil.AttemptFactory.WithID(attemptID), + testutil.AttemptFactory.WithEventID(eventID), + testutil.AttemptFactory.WithDestinationID(destinationID), + testutil.AttemptFactory.WithStatus("failed"), + testutil.AttemptFactory.WithTime(attemptTime), ) - de := &models.DeliveryEvent{ - ID: fmt.Sprintf("%s_%s", eventID, deliveryID), - DestinationID: destinationID, - Event: *event, - Delivery: delivery, - } + require.NoError(t, result.logStore.InsertMany(context.Background(), []*models.LogEntry{{Event: event, Attempt: attempt}})) - require.NoError(t, result.logStore.InsertManyDeliveryEvent(context.Background(), []*models.DeliveryEvent{de})) + t.Run("should retry attempt successfully with full event data", func(t *testing.T) { + // Subscribe to deliveryMQ to capture published task + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() - t.Run("should retry delivery successfully", func(t *testing.T) { + subscription, err := result.deliveryMQ.Subscribe(ctx) + require.NoError(t, err) + + // Trigger manual retry w := httptest.NewRecorder() - req, _ := http.NewRequest("POST", baseAPIPath+"/tenants/"+tenantID+"/deliveries/"+deliveryID+"/retry", nil) + req, _ := http.NewRequest("POST", baseAPIPath+"/tenants/"+tenantID+"/attempts/"+attemptID+"/retry", nil) result.router.ServeHTTP(w, req) assert.Equal(t, http.StatusAccepted, w.Code) @@ -77,11 +77,29 @@ func TestRetryDelivery(t *testing.T) { var response map[string]interface{} require.NoError(t, json.Unmarshal(w.Body.Bytes(), &response)) assert.Equal(t, true, response["success"]) + + // Verify published task has full event data + msg, err := subscription.Receive(ctx) + require.NoError(t, err) + + var task models.DeliveryTask + require.NoError(t, json.Unmarshal(msg.Body, &task)) + + assert.Equal(t, eventID, task.Event.ID) + assert.Equal(t, tenantID, task.Event.TenantID) + assert.Equal(t, destinationID, task.Event.DestinationID) + assert.Equal(t, "order.created", task.Event.Topic) + assert.False(t, task.Event.Time.IsZero(), "event time should be set") + assert.Equal(t, eventTime.UTC(), task.Event.Time.UTC()) + assert.Equal(t, event.Data, task.Event.Data, "event data should match original") + assert.True(t, task.Manual, "should be marked as manual retry") + + msg.Ack() }) - t.Run("should return 404 for non-existent delivery", func(t *testing.T) { + t.Run("should return 404 for non-existent attempt", func(t *testing.T) { w := httptest.NewRecorder() - req, _ := http.NewRequest("POST", baseAPIPath+"/tenants/"+tenantID+"/deliveries/nonexistent/retry", nil) + req, _ := http.NewRequest("POST", baseAPIPath+"/tenants/"+tenantID+"/attempts/nonexistent/retry", nil) result.router.ServeHTTP(w, req) assert.Equal(t, http.StatusNotFound, w.Code) @@ -89,7 +107,7 @@ func TestRetryDelivery(t *testing.T) { t.Run("should return 404 for non-existent tenant", func(t *testing.T) { w := httptest.NewRecorder() - req, _ := http.NewRequest("POST", baseAPIPath+"/tenants/nonexistent/deliveries/"+deliveryID+"/retry", nil) + req, _ := http.NewRequest("POST", baseAPIPath+"/tenants/nonexistent/attempts/"+attemptID+"/retry", nil) result.router.ServeHTTP(w, req) assert.Equal(t, http.StatusNotFound, w.Code) @@ -108,9 +126,9 @@ func TestRetryDelivery(t *testing.T) { DisabledAt: &disabledAt, })) - // Create a delivery for the disabled destination + // Create an attempt for the disabled destination disabledEventID := idgen.Event() - disabledDeliveryID := idgen.Delivery() + disabledAttemptID := idgen.Attempt() disabledEvent := testutil.EventFactory.AnyPointer( testutil.EventFactory.WithID(disabledEventID), @@ -120,25 +138,18 @@ func TestRetryDelivery(t *testing.T) { testutil.EventFactory.WithTime(eventTime), ) - disabledDelivery := testutil.DeliveryFactory.AnyPointer( - testutil.DeliveryFactory.WithID(disabledDeliveryID), - testutil.DeliveryFactory.WithEventID(disabledEventID), - testutil.DeliveryFactory.WithDestinationID(disabledDestinationID), - testutil.DeliveryFactory.WithStatus("failed"), - testutil.DeliveryFactory.WithTime(deliveryTime), + disabledAttempt := testutil.AttemptFactory.AnyPointer( + testutil.AttemptFactory.WithID(disabledAttemptID), + testutil.AttemptFactory.WithEventID(disabledEventID), + testutil.AttemptFactory.WithDestinationID(disabledDestinationID), + testutil.AttemptFactory.WithStatus("failed"), + testutil.AttemptFactory.WithTime(attemptTime), ) - disabledDE := &models.DeliveryEvent{ - ID: fmt.Sprintf("%s_%s", disabledEventID, disabledDeliveryID), - DestinationID: disabledDestinationID, - Event: *disabledEvent, - Delivery: disabledDelivery, - } - - require.NoError(t, result.logStore.InsertManyDeliveryEvent(context.Background(), []*models.DeliveryEvent{disabledDE})) + require.NoError(t, result.logStore.InsertMany(context.Background(), []*models.LogEntry{{Event: disabledEvent, Attempt: disabledAttempt}})) w := httptest.NewRecorder() - req, _ := http.NewRequest("POST", baseAPIPath+"/tenants/"+tenantID+"/deliveries/"+disabledDeliveryID+"/retry", nil) + req, _ := http.NewRequest("POST", baseAPIPath+"/tenants/"+tenantID+"/attempts/"+disabledAttemptID+"/retry", nil) result.router.ServeHTTP(w, req) assert.Equal(t, http.StatusBadRequest, w.Code) diff --git a/internal/apirouter/router.go b/internal/apirouter/router.go index 5ab20f77..f5938d67 100644 --- a/internal/apirouter/router.go +++ b/internal/apirouter/router.go @@ -145,7 +145,6 @@ func NewRouter( logHandlers := NewLogHandlers(logger, logStore) retryHandlers := NewRetryHandlers(logger, entityStore, logStore, deliveryMQ) topicHandlers := NewTopicHandlers(logger, cfg.Topics) - legacyHandlers := NewLegacyHandlers(logger, entityStore, logStore, deliveryMQ) // Non-tenant routes (no :tenantID in path) nonTenantRoutes := []RouteDefinition{ @@ -172,8 +171,8 @@ func NewRouter( }, { Method: http.MethodGet, - Path: "/deliveries", - Handler: logHandlers.AdminListDeliveries, + Path: "/attempts", + Handler: logHandlers.AdminListAttempts, AuthScope: AuthScopeAdmin, Mode: RouteModeAlways, }, @@ -333,11 +332,11 @@ func NewRouter( }, }, - // Event routes + // Destination-scoped attempt routes { Method: http.MethodGet, - Path: "/:tenantID/events", - Handler: logHandlers.ListEvents, + Path: "/:tenantID/destinations/:destinationID/attempts", + Handler: logHandlers.ListDestinationAttempts, AuthScope: AuthScopeAdminOrTenant, Mode: RouteModeAlways, Middlewares: []gin.HandlerFunc{ @@ -346,8 +345,8 @@ func NewRouter( }, { Method: http.MethodGet, - Path: "/:tenantID/events/:eventID", - Handler: logHandlers.RetrieveEvent, + Path: "/:tenantID/destinations/:destinationID/attempts/:attemptID", + Handler: logHandlers.RetrieveAttempt, AuthScope: AuthScopeAdminOrTenant, Mode: RouteModeAlways, Middlewares: []gin.HandlerFunc{ @@ -355,9 +354,9 @@ func NewRouter( }, }, { - Method: http.MethodGet, - Path: "/:tenantID/events/:eventID/deliveries", - Handler: legacyHandlers.ListDeliveriesByEvent, + Method: http.MethodPost, + Path: "/:tenantID/destinations/:destinationID/attempts/:attemptID/retry", + Handler: retryHandlers.RetryAttempt, AuthScope: AuthScopeAdminOrTenant, Mode: RouteModeAlways, Middlewares: []gin.HandlerFunc{ @@ -365,11 +364,11 @@ func NewRouter( }, }, - // Delivery routes + // Event routes { Method: http.MethodGet, - Path: "/:tenantID/deliveries", - Handler: logHandlers.ListDeliveries, + Path: "/:tenantID/events", + Handler: logHandlers.ListEvents, AuthScope: AuthScopeAdminOrTenant, Mode: RouteModeAlways, Middlewares: []gin.HandlerFunc{ @@ -378,32 +377,20 @@ func NewRouter( }, { Method: http.MethodGet, - Path: "/:tenantID/deliveries/:deliveryID", - Handler: logHandlers.RetrieveDelivery, - AuthScope: AuthScopeAdminOrTenant, - Mode: RouteModeAlways, - Middlewares: []gin.HandlerFunc{ - RequireTenantMiddleware(entityStore), - }, - }, - { - Method: http.MethodPost, - Path: "/:tenantID/deliveries/:deliveryID/retry", - Handler: retryHandlers.RetryDelivery, + Path: "/:tenantID/events/:eventID", + Handler: logHandlers.RetrieveEvent, AuthScope: AuthScopeAdminOrTenant, Mode: RouteModeAlways, Middlewares: []gin.HandlerFunc{ RequireTenantMiddleware(entityStore), }, }, - } - // Legacy routes (deprecated, for backward compatibility) - legacyRoutes := []RouteDefinition{ + // Attempt routes { Method: http.MethodGet, - Path: "/:tenantID/destinations/:destinationID/events", - Handler: legacyHandlers.ListEventsByDestination, + Path: "/:tenantID/attempts", + Handler: logHandlers.ListAttempts, AuthScope: AuthScopeAdminOrTenant, Mode: RouteModeAlways, Middlewares: []gin.HandlerFunc{ @@ -412,8 +399,8 @@ func NewRouter( }, { Method: http.MethodGet, - Path: "/:tenantID/destinations/:destinationID/events/:eventID", - Handler: legacyHandlers.RetrieveEventByDestination, + Path: "/:tenantID/attempts/:attemptID", + Handler: logHandlers.RetrieveAttempt, AuthScope: AuthScopeAdminOrTenant, Mode: RouteModeAlways, Middlewares: []gin.HandlerFunc{ @@ -422,8 +409,8 @@ func NewRouter( }, { Method: http.MethodPost, - Path: "/:tenantID/destinations/:destinationID/events/:eventID/retry", - Handler: legacyHandlers.RetryByEventDestination, + Path: "/:tenantID/attempts/:attemptID/retry", + Handler: retryHandlers.RetryAttempt, AuthScope: AuthScopeAdminOrTenant, Mode: RouteModeAlways, Middlewares: []gin.HandlerFunc{ @@ -441,7 +428,6 @@ func NewRouter( tenantScopedRoutes = append(tenantScopedRoutes, portalRoutes...) tenantScopedRoutes = append(tenantScopedRoutes, tenantAgnosticRoutes...) tenantScopedRoutes = append(tenantScopedRoutes, tenantSpecificRoutes...) - tenantScopedRoutes = append(tenantScopedRoutes, legacyRoutes...) // Register tenant-scoped routes under /tenants prefix tenantsGroup := apiRouter.Group("/tenants") diff --git a/internal/apirouter/router_test.go b/internal/apirouter/router_test.go index 4cb35a10..d26840c0 100644 --- a/internal/apirouter/router_test.go +++ b/internal/apirouter/router_test.go @@ -34,6 +34,7 @@ type testRouterResult struct { redisClient redis.Client entityStore models.EntityStore logStore logstore.LogStore + deliveryMQ *deliverymq.DeliveryMQ } func setupTestRouter(t *testing.T, apiKey, jwtSecret string, funcs ...func(t *testing.T) clickhouse.DB) (http.Handler, *logging.Logger, redis.Client) { @@ -73,6 +74,7 @@ func setupTestRouterFull(t *testing.T, apiKey, jwtSecret string, funcs ...func(t redisClient: redisClient, entityStore: entityStore, logStore: logStore, + deliveryMQ: deliveryMQ, } } diff --git a/internal/app/app.go b/internal/app/app.go index 7d0ced4f..8263ba92 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -121,21 +121,17 @@ func (a *App) PostRun(ctx context.Context) { } func (a *App) run(ctx context.Context) error { - // Set up cancellation context ctx, cancel := context.WithCancel(ctx) defer cancel() - // Handle sigterm and await termChan signal termChan := make(chan os.Signal, 1) signal.Notify(termChan, syscall.SIGINT, syscall.SIGTERM) - // Run workers in goroutine errChan := make(chan error, 1) go func() { errChan <- a.supervisor.Run(ctx) }() - // Wait for either termination signal or worker failure var exitErr error select { case <-termChan: @@ -175,15 +171,13 @@ func (a *App) configureIDGenerators() error { zap.String("type", a.config.IDGen.Type), zap.String("event_prefix", a.config.IDGen.EventPrefix), zap.String("destination_prefix", a.config.IDGen.DestinationPrefix), - zap.String("delivery_prefix", a.config.IDGen.DeliveryPrefix), - zap.String("delivery_event_prefix", a.config.IDGen.DeliveryEventPrefix)) + zap.String("attempt_prefix", a.config.IDGen.AttemptPrefix)) if err := idgen.Configure(idgen.IDGenConfig{ - Type: a.config.IDGen.Type, - EventPrefix: a.config.IDGen.EventPrefix, - DestinationPrefix: a.config.IDGen.DestinationPrefix, - DeliveryPrefix: a.config.IDGen.DeliveryPrefix, - DeliveryEventPrefix: a.config.IDGen.DeliveryEventPrefix, + Type: a.config.IDGen.Type, + EventPrefix: a.config.IDGen.EventPrefix, + DestinationPrefix: a.config.IDGen.DestinationPrefix, + AttemptPrefix: a.config.IDGen.AttemptPrefix, }); err != nil { a.logger.Error("failed to configure ID generators", zap.Error(err)) return err @@ -204,7 +198,6 @@ func (a *App) initializeRedis(ctx context.Context) error { } a.redisClient = redisClient - // Run Redis schema migrations if err := runRedisMigrations(ctx, redisClient, a.logger, a.config.DeploymentID); err != nil { a.logger.Error("Redis migration failed", zap.Error(err)) return err diff --git a/internal/app/installation.go b/internal/app/installation.go index 9019e7ea..3c43a097 100644 --- a/internal/app/installation.go +++ b/internal/app/installation.go @@ -21,12 +21,10 @@ func getInstallation(ctx context.Context, redisClient redis.Cmdable, telemetryCo // First attempt: try to get existing installation ID installationID, err := redisClient.HGet(ctx, outpostrcKey, installationKey).Result() if err == nil { - // Installation ID already exists return installationID, nil } if err != redis.Nil { - // Unexpected error return "", err } @@ -41,7 +39,6 @@ func getInstallation(ctx context.Context, redisClient redis.Cmdable, telemetryCo } if wasSet { - // We successfully set the installation ID return newInstallationID, nil } diff --git a/internal/app/migration.go b/internal/app/migration.go index 9938b659..eaa7b543 100644 --- a/internal/app/migration.go +++ b/internal/app/migration.go @@ -51,7 +51,6 @@ func runMigration(ctx context.Context, cfg *config.Config, logger *logging.Logge } if err == nil { - // Migration succeeded if versionJumped > 0 { logger.Info("migrations applied", zap.Int("version", version), @@ -88,7 +87,6 @@ func runMigration(ctx context.Context, cfg *config.Config, logger *logging.Logge case <-ctx.Done(): return ctx.Err() case <-time.After(retryDelay): - // Continue to next attempt } } else { // Exhausted all retries diff --git a/internal/app/redis_migration.go b/internal/app/redis_migration.go index 97aa7ddd..70d52782 100644 --- a/internal/app/redis_migration.go +++ b/internal/app/redis_migration.go @@ -35,7 +35,6 @@ func runRedisMigrations(ctx context.Context, redisClient redis.Cmdable, logger * return nil } - // Check if this is a lock-related error isLockError := isRedisLockError(err) lastErr = err @@ -56,7 +55,6 @@ func runRedisMigrations(ctx context.Context, redisClient redis.Cmdable, logger * case <-ctx.Done(): return ctx.Err() case <-time.After(retryDelay): - // Continue to next attempt } } else { logger.Error("redis migration failed after retries", @@ -70,7 +68,6 @@ func runRedisMigrations(ctx context.Context, redisClient redis.Cmdable, logger * // executeRedisMigrations creates the runner and executes migrations func executeRedisMigrations(ctx context.Context, redisClient redis.Cmdable, logger *logging.Logger, deploymentID string) error { - // Create runner client, ok := redisClient.(redis.Client) if !ok { // Wrap Cmdable to implement Client interface diff --git a/internal/config/config.go b/internal/config/config.go index 48371ba1..35fed1d5 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -76,10 +76,11 @@ type Config struct { LogMaxConcurrency int `yaml:"log_max_concurrency" env:"LOG_MAX_CONCURRENCY" desc:"Maximum number of log writing operations to process concurrently." required:"N"` // Delivery Retry - RetrySchedule []int `yaml:"retry_schedule" env:"RETRY_SCHEDULE" envSeparator:"," desc:"Comma-separated list of retry delays in seconds. If provided, overrides retry_interval_seconds and retry_max_limit. Schedule length defines the max number of retries. Example: '5,60,600,3600,7200' for 5 retries at 5s, 1m, 10m, 1h, 2h." required:"N"` - RetryIntervalSeconds int `yaml:"retry_interval_seconds" env:"RETRY_INTERVAL_SECONDS" desc:"Interval in seconds for exponential backoff retry strategy (base 2). Ignored if retry_schedule is provided." required:"N"` - RetryMaxLimit int `yaml:"retry_max_limit" env:"MAX_RETRY_LIMIT" desc:"Maximum number of retry attempts for a single event delivery before giving up. Ignored if retry_schedule is provided." required:"N"` - RetryPollBackoffMs int `yaml:"retry_poll_backoff_ms" env:"RETRY_POLL_BACKOFF_MS" desc:"Backoff time in milliseconds when the retry monitor finds no messages to process. When a retry message is found, the monitor immediately polls for the next message without delay. Lower values provide faster retry processing but increase Redis load. For serverless Redis providers (Upstash, ElastiCache Serverless), consider increasing to 5000-10000ms to reduce costs. Default: 100" required:"N"` + RetrySchedule []int `yaml:"retry_schedule" env:"RETRY_SCHEDULE" envSeparator:"," desc:"Comma-separated list of retry delays in seconds. If provided, overrides retry_interval_seconds and retry_max_limit. Schedule length defines the max number of retries. Example: '5,60,600,3600,7200' for 5 retries at 5s, 1m, 10m, 1h, 2h." required:"N"` + RetryIntervalSeconds int `yaml:"retry_interval_seconds" env:"RETRY_INTERVAL_SECONDS" desc:"Interval in seconds for exponential backoff retry strategy (base 2). Ignored if retry_schedule is provided." required:"N"` + RetryMaxLimit int `yaml:"retry_max_limit" env:"MAX_RETRY_LIMIT" desc:"Maximum number of retry attempts for a single event delivery before giving up. Ignored if retry_schedule is provided." required:"N"` + RetryPollBackoffMs int `yaml:"retry_poll_backoff_ms" env:"RETRY_POLL_BACKOFF_MS" desc:"Backoff time in milliseconds when the retry monitor finds no messages to process. When a retry message is found, the monitor immediately polls for the next message without delay. Lower values provide faster retry processing but increase Redis load. For serverless Redis providers (Upstash, ElastiCache Serverless), consider increasing to 5000-10000ms to reduce costs. Default: 100" required:"N"` + RetryVisibilityTimeoutSeconds int `yaml:"retry_visibility_timeout_seconds" env:"RETRY_VISIBILITY_TIMEOUT_SECONDS" desc:"Time in seconds a retry message is hidden after being received before becoming visible again for reprocessing. This applies when event data is temporarily unavailable (e.g., race condition with log persistence). Default: 30" required:"N"` // Event Delivery MaxDestinationsPerTenant int `yaml:"max_destinations_per_tenant" env:"MAX_DESTINATIONS_PER_TENANT" desc:"Maximum number of destinations allowed per tenant/organization." required:"N"` @@ -165,6 +166,7 @@ func (c *Config) InitDefaults() { c.RetryIntervalSeconds = 30 c.RetryMaxLimit = 10 c.RetryPollBackoffMs = 100 + c.RetryVisibilityTimeoutSeconds = 30 c.MaxDestinationsPerTenant = 20 c.DeliveryTimeoutSeconds = 5 c.PublishIdempotencyKeyTTL = 3600 // 1 hour diff --git a/internal/config/id_gen.go b/internal/config/id_gen.go index 3837e3b9..a35c001d 100644 --- a/internal/config/id_gen.go +++ b/internal/config/id_gen.go @@ -2,9 +2,8 @@ package config // IDGenConfig is the configuration for ID generation type IDGenConfig struct { - Type string `yaml:"type" env:"IDGEN_TYPE" desc:"ID generation type for all entities: uuidv4, uuidv7, nanoid. Default: uuidv4" required:"N"` - EventPrefix string `yaml:"event_prefix" env:"IDGEN_EVENT_PREFIX" desc:"Prefix for event IDs (e.g., 'evt_' produces 'evt_123'). Default: empty (no prefix)" required:"N"` - DestinationPrefix string `yaml:"destination_prefix" env:"IDGEN_DESTINATION_PREFIX" desc:"Prefix for destination IDs (e.g., 'dst_' produces 'dst_123'). Default: empty (no prefix)" required:"N"` - DeliveryPrefix string `yaml:"delivery_prefix" env:"IDGEN_DELIVERY_PREFIX" desc:"Prefix for delivery IDs (e.g., 'dlv_' produces 'dlv_123'). Default: empty (no prefix)" required:"N"` - DeliveryEventPrefix string `yaml:"delivery_event_prefix" env:"IDGEN_DELIVERY_EVENT_PREFIX" desc:"Prefix for delivery event IDs (e.g., 'dev_' produces 'dev_123'). Default: empty (no prefix)" required:"N"` + Type string `yaml:"type" env:"IDGEN_TYPE" desc:"ID generation type for all entities: uuidv4, uuidv7, nanoid. Default: uuidv4" required:"N"` + AttemptPrefix string `yaml:"attempt_prefix" env:"IDGEN_ATTEMPT_PREFIX" desc:"Prefix for attempt IDs, prepended with underscore (e.g., 'atm_123'). Default: empty (no prefix)" required:"N"` + DestinationPrefix string `yaml:"destination_prefix" env:"IDGEN_DESTINATION_PREFIX" desc:"Prefix for destination IDs, prepended with underscore (e.g., 'dst_123'). Default: empty (no prefix)" required:"N"` + EventPrefix string `yaml:"event_prefix" env:"IDGEN_EVENT_PREFIX" desc:"Prefix for event IDs, prepended with underscore (e.g., 'evt_123'). Default: empty (no prefix)" required:"N"` } diff --git a/internal/deliverymq/deliverymq.go b/internal/deliverymq/deliverymq.go index eb3b2a7a..57dbb805 100644 --- a/internal/deliverymq/deliverymq.go +++ b/internal/deliverymq/deliverymq.go @@ -44,8 +44,8 @@ func (q *DeliveryMQ) Init(ctx context.Context) (func(), error) { return q.queue.Init(ctx) } -func (q *DeliveryMQ) Publish(ctx context.Context, event models.DeliveryEvent) error { - return q.queue.Publish(ctx, &event) +func (q *DeliveryMQ) Publish(ctx context.Context, task models.DeliveryTask) error { + return q.queue.Publish(ctx, &task) } func (q *DeliveryMQ) Subscribe(ctx context.Context) (mqs.Subscription, error) { diff --git a/internal/deliverymq/messagehandler.go b/internal/deliverymq/messagehandler.go index f9625bd9..cf91c06d 100644 --- a/internal/deliverymq/messagehandler.go +++ b/internal/deliverymq/messagehandler.go @@ -12,7 +12,6 @@ import ( "github.com/hookdeck/outpost/internal/destregistry" "github.com/hookdeck/outpost/internal/idempotence" "github.com/hookdeck/outpost/internal/logging" - "github.com/hookdeck/outpost/internal/logstore" "github.com/hookdeck/outpost/internal/models" "github.com/hookdeck/outpost/internal/mqs" "github.com/hookdeck/outpost/internal/scheduler" @@ -20,8 +19,8 @@ import ( "go.uber.org/zap" ) -func idempotencyKeyFromDeliveryEvent(deliveryEvent models.DeliveryEvent) string { - return "idempotency:deliverymq:" + deliveryEvent.ID +func idempotencyKeyFromDeliveryTask(task models.DeliveryTask) string { + return "idempotency:deliverymq:" + task.IdempotencyKey() } var ( @@ -41,15 +40,15 @@ func (e *PreDeliveryError) Unwrap() error { return e.err } -type DeliveryError struct { +type AttemptError struct { err error } -func (e *DeliveryError) Error() string { - return fmt.Sprintf("delivery error: %v", e.err) +func (e *AttemptError) Error() string { + return fmt.Sprintf("attempt error: %v", e.err) } -func (e *DeliveryError) Unwrap() error { +func (e *AttemptError) Unwrap() error { return e.err } @@ -70,7 +69,6 @@ type messageHandler struct { logger *logging.Logger logMQ LogPublisher entityStore DestinationGetter - logStore EventGetter retryScheduler RetryScheduler retryBackoff backoff.Backoff retryMaxLimit int @@ -80,11 +78,11 @@ type messageHandler struct { } type Publisher interface { - PublishEvent(ctx context.Context, destination *models.Destination, event *models.Event) (*models.Delivery, error) + PublishEvent(ctx context.Context, destination *models.Destination, event *models.Event) (*models.Attempt, error) } type LogPublisher interface { - Publish(ctx context.Context, deliveryEvent models.DeliveryEvent) error + Publish(ctx context.Context, entry models.LogEntry) error } type RetryScheduler interface { @@ -96,12 +94,8 @@ type DestinationGetter interface { RetrieveDestination(ctx context.Context, tenantID, destID string) (*models.Destination, error) } -type EventGetter interface { - RetrieveEvent(ctx context.Context, request logstore.RetrieveEventRequest) (*models.Event, error) -} - type DeliveryTracer interface { - Deliver(ctx context.Context, deliveryEvent *models.DeliveryEvent, destination *models.Destination) (context.Context, trace.Span) + Deliver(ctx context.Context, task *models.DeliveryTask, destination *models.Destination) (context.Context, trace.Span) } type AlertMonitor interface { @@ -112,7 +106,6 @@ func NewMessageHandler( logger *logging.Logger, logMQ LogPublisher, entityStore DestinationGetter, - logStore EventGetter, publisher Publisher, eventTracer DeliveryTracer, retryScheduler RetryScheduler, @@ -126,7 +119,6 @@ func NewMessageHandler( logger: logger, logMQ: logMQ, entityStore: entityStore, - logStore: logStore, publisher: publisher, retryScheduler: retryScheduler, retryBackoff: retryBackoff, @@ -137,34 +129,25 @@ func NewMessageHandler( } func (h *messageHandler) Handle(ctx context.Context, msg *mqs.Message) error { - deliveryEvent := models.DeliveryEvent{} + task := models.DeliveryTask{} - // Parse message - if err := deliveryEvent.FromMessage(msg); err != nil { + if err := task.FromMessage(msg); err != nil { return h.handleError(msg, &PreDeliveryError{err: err}) } - h.logger.Ctx(ctx).Info("processing delivery event", - zap.String("delivery_event_id", deliveryEvent.ID), - zap.String("event_id", deliveryEvent.Event.ID), - zap.String("tenant_id", deliveryEvent.Event.TenantID), - zap.String("destination_id", deliveryEvent.DestinationID), - zap.Int("attempt", deliveryEvent.Attempt)) - - // Ensure event data - if err := h.ensureDeliveryEvent(ctx, &deliveryEvent); err != nil { - return h.handleError(msg, &PreDeliveryError{err: err}) - } + h.logger.Ctx(ctx).Info("processing delivery task", + zap.String("event_id", task.Event.ID), + zap.String("tenant_id", task.Event.TenantID), + zap.String("destination_id", task.DestinationID), + zap.Int("attempt", task.Attempt)) - // Get destination - destination, err := h.ensurePublishableDestination(ctx, deliveryEvent) + destination, err := h.ensurePublishableDestination(ctx, task) if err != nil { return h.handleError(msg, &PreDeliveryError{err: err}) } - // Handle delivery - err = h.idempotence.Exec(ctx, idempotencyKeyFromDeliveryEvent(deliveryEvent), func(ctx context.Context) error { - return h.doHandle(ctx, deliveryEvent, destination) + err = h.idempotence.Exec(ctx, idempotencyKeyFromDeliveryTask(task), func(ctx context.Context) error { + return h.doHandle(ctx, task, destination) }) return h.handleError(msg, err) } @@ -187,88 +170,87 @@ func (h *messageHandler) handleError(msg *mqs.Message, err error) error { return err } -func (h *messageHandler) doHandle(ctx context.Context, deliveryEvent models.DeliveryEvent, destination *models.Destination) error { - _, span := h.eventTracer.Deliver(ctx, &deliveryEvent, destination) +func (h *messageHandler) doHandle(ctx context.Context, task models.DeliveryTask, destination *models.Destination) error { + _, span := h.eventTracer.Deliver(ctx, &task, destination) defer span.End() - delivery, err := h.publisher.PublishEvent(ctx, destination, &deliveryEvent.Event) + attempt, err := h.publisher.PublishEvent(ctx, destination, &task.Event) if err != nil { - // If delivery is nil, it means no delivery was made. + // If attempt is nil, it means no attempt was made. // This is an unexpected error and considered a pre-delivery error. - if delivery == nil { + if attempt == nil { return &PreDeliveryError{err: err} } h.logger.Ctx(ctx).Error("failed to publish event", zap.Error(err), - zap.String("delivery_event_id", deliveryEvent.ID), - zap.String("delivery_id", delivery.ID), - zap.String("event_id", deliveryEvent.Event.ID), - zap.String("tenant_id", deliveryEvent.Event.TenantID), + zap.String("attempt_id", attempt.ID), + zap.String("event_id", task.Event.ID), + zap.String("tenant_id", task.Event.TenantID), zap.String("destination_id", destination.ID), zap.String("destination_type", destination.Type)) - deliveryErr := &DeliveryError{err: err} + attemptErr := &AttemptError{err: err} - if h.shouldScheduleRetry(deliveryEvent, err) { - if retryErr := h.scheduleRetry(ctx, deliveryEvent); retryErr != nil { - return h.logDeliveryResult(ctx, &deliveryEvent, destination, delivery, errors.Join(err, retryErr)) + if h.shouldScheduleRetry(task, err) { + if retryErr := h.scheduleRetry(ctx, task); retryErr != nil { + return h.logDeliveryResult(ctx, &task, destination, attempt, errors.Join(err, retryErr)) } } - return h.logDeliveryResult(ctx, &deliveryEvent, destination, delivery, deliveryErr) + return h.logDeliveryResult(ctx, &task, destination, attempt, attemptErr) } // Handle successful delivery - if deliveryEvent.Manual { + if task.Manual { logger := h.logger.Ctx(ctx) - if err := h.retryScheduler.Cancel(ctx, deliveryEvent.GetRetryID()); err != nil { + if err := h.retryScheduler.Cancel(ctx, models.RetryID(task.Event.ID, task.DestinationID)); err != nil { h.logger.Ctx(ctx).Error("failed to cancel scheduled retry", zap.Error(err), - zap.String("delivery_event_id", deliveryEvent.ID), - zap.String("delivery_id", delivery.ID), - zap.String("event_id", deliveryEvent.Event.ID), - zap.String("tenant_id", deliveryEvent.Event.TenantID), + zap.String("attempt_id", attempt.ID), + zap.String("event_id", task.Event.ID), + zap.String("tenant_id", task.Event.TenantID), zap.String("destination_id", destination.ID), zap.String("destination_type", destination.Type), - zap.String("retry_id", deliveryEvent.GetRetryID())) - return h.logDeliveryResult(ctx, &deliveryEvent, destination, delivery, err) + zap.String("retry_id", models.RetryID(task.Event.ID, task.DestinationID))) + return h.logDeliveryResult(ctx, &task, destination, attempt, err) } logger.Audit("scheduled retry canceled", - zap.String("delivery_event_id", deliveryEvent.ID), - zap.String("delivery_id", delivery.ID), - zap.String("event_id", deliveryEvent.Event.ID), - zap.String("tenant_id", deliveryEvent.Event.TenantID), + zap.String("attempt_id", attempt.ID), + zap.String("event_id", task.Event.ID), + zap.String("tenant_id", task.Event.TenantID), zap.String("destination_id", destination.ID), zap.String("destination_type", destination.Type), - zap.String("retry_id", deliveryEvent.GetRetryID())) + zap.String("retry_id", models.RetryID(task.Event.ID, task.DestinationID))) } - return h.logDeliveryResult(ctx, &deliveryEvent, destination, delivery, nil) + return h.logDeliveryResult(ctx, &task, destination, attempt, nil) } -func (h *messageHandler) logDeliveryResult(ctx context.Context, deliveryEvent *models.DeliveryEvent, destination *models.Destination, delivery *models.Delivery, err error) error { +func (h *messageHandler) logDeliveryResult(ctx context.Context, task *models.DeliveryTask, destination *models.Destination, attempt *models.Attempt, err error) error { logger := h.logger.Ctx(ctx) - // Set up delivery record - deliveryEvent.Delivery = delivery + attempt.TenantID = task.Event.TenantID + attempt.AttemptNumber = task.Attempt + attempt.Manual = task.Manual logger.Audit("event delivered", - zap.String("delivery_event_id", deliveryEvent.ID), - zap.String("delivery_id", deliveryEvent.Delivery.ID), - zap.String("event_id", deliveryEvent.Event.ID), - zap.String("tenant_id", deliveryEvent.Event.TenantID), + zap.String("attempt_id", attempt.ID), + zap.String("event_id", task.Event.ID), + zap.String("tenant_id", task.Event.TenantID), zap.String("destination_id", destination.ID), zap.String("destination_type", destination.Type), - zap.String("delivery_status", deliveryEvent.Delivery.Status), - zap.Int("attempt", deliveryEvent.Attempt), - zap.Bool("manual", deliveryEvent.Manual)) + zap.String("attempt_status", attempt.Status), + zap.Int("attempt", task.Attempt), + zap.Bool("manual", task.Manual)) - // Publish delivery log - if logErr := h.logMQ.Publish(ctx, *deliveryEvent); logErr != nil { - logger.Error("failed to publish delivery log", + logEntry := models.LogEntry{ + Event: &task.Event, + Attempt: attempt, + } + if logErr := h.logMQ.Publish(ctx, logEntry); logErr != nil { + logger.Error("failed to publish attempt log", zap.Error(logErr), - zap.String("delivery_event_id", deliveryEvent.ID), - zap.String("delivery_id", deliveryEvent.Delivery.ID), - zap.String("event_id", deliveryEvent.Event.ID), - zap.String("tenant_id", deliveryEvent.Event.TenantID), + zap.String("attempt_id", attempt.ID), + zap.String("event_id", task.Event.ID), + zap.String("tenant_id", task.Event.TenantID), zap.String("destination_id", destination.ID), zap.String("destination_type", destination.Type)) if err != nil { @@ -277,12 +259,11 @@ func (h *messageHandler) logDeliveryResult(ctx context.Context, deliveryEvent *m return &PostDeliveryError{err: logErr} } - // Call alert monitor in goroutine - go h.handleAlertAttempt(ctx, deliveryEvent, destination, err) + go h.handleAlertAttempt(ctx, task, destination, attempt, err) - // If we have a DeliveryError, return it as is - var delErr *DeliveryError - if errors.As(err, &delErr) { + // If we have an AttemptError, return it as is + var atmErr *AttemptError + if errors.As(err, &atmErr) { return err } @@ -300,10 +281,10 @@ func (h *messageHandler) logDeliveryResult(ctx context.Context, deliveryEvent *m return nil } -func (h *messageHandler) handleAlertAttempt(ctx context.Context, deliveryEvent *models.DeliveryEvent, destination *models.Destination, err error) { - attempt := alert.DeliveryAttempt{ - Success: deliveryEvent.Delivery.Status == models.DeliveryStatusSuccess, - DeliveryEvent: deliveryEvent, +func (h *messageHandler) handleAlertAttempt(ctx context.Context, task *models.DeliveryTask, destination *models.Destination, attemptResult *models.Attempt, err error) { + alertAttempt := alert.DeliveryAttempt{ + Success: attemptResult.Status == models.AttemptStatusSuccess, + DeliveryTask: task, Destination: &alert.AlertDestination{ ID: destination.ID, TenantID: destination.TenantID, @@ -313,35 +294,34 @@ func (h *messageHandler) handleAlertAttempt(ctx context.Context, deliveryEvent * CreatedAt: destination.CreatedAt, DisabledAt: destination.DisabledAt, }, - Timestamp: deliveryEvent.Delivery.Time, + Timestamp: attemptResult.Time, } - if !attempt.Success && err != nil { + if !alertAttempt.Success && err != nil { // Extract attempt data if available - var delErr *DeliveryError - if errors.As(err, &delErr) { + var atmErr *AttemptError + if errors.As(err, &atmErr) { var pubErr *destregistry.ErrDestinationPublishAttempt - if errors.As(delErr.err, &pubErr) { - attempt.DeliveryResponse = pubErr.Data + if errors.As(atmErr.err, &pubErr) { + alertAttempt.DeliveryResponse = pubErr.Data } else { - attempt.DeliveryResponse = map[string]interface{}{ - "error": delErr.err.Error(), + alertAttempt.DeliveryResponse = map[string]interface{}{ + "error": atmErr.err.Error(), } } } else { - attempt.DeliveryResponse = map[string]interface{}{ + alertAttempt.DeliveryResponse = map[string]interface{}{ "error": "unexpected", "message": err.Error(), } } } - if monitorErr := h.alertMonitor.HandleAttempt(ctx, attempt); monitorErr != nil { + if monitorErr := h.alertMonitor.HandleAttempt(ctx, alertAttempt); monitorErr != nil { h.logger.Ctx(ctx).Error("failed to handle alert attempt", zap.Error(monitorErr), - zap.String("delivery_event_id", deliveryEvent.ID), - zap.String("delivery_id", deliveryEvent.Delivery.ID), - zap.String("event_id", deliveryEvent.Event.ID), + zap.String("attempt_id", attemptResult.ID), + zap.String("event_id", task.Event.ID), zap.String("tenant_id", destination.TenantID), zap.String("destination_id", destination.ID), zap.String("destination_type", destination.Type)) @@ -349,26 +329,25 @@ func (h *messageHandler) handleAlertAttempt(ctx context.Context, deliveryEvent * } h.logger.Ctx(ctx).Info("alert attempt handled", - zap.String("delivery_event_id", deliveryEvent.ID), - zap.String("delivery_id", deliveryEvent.Delivery.ID), - zap.String("event_id", deliveryEvent.Event.ID), + zap.String("attempt_id", attemptResult.ID), + zap.String("event_id", task.Event.ID), zap.String("tenant_id", destination.TenantID), zap.String("destination_id", destination.ID), zap.String("destination_type", destination.Type)) } -func (h *messageHandler) shouldScheduleRetry(deliveryEvent models.DeliveryEvent, err error) bool { - if deliveryEvent.Manual { +func (h *messageHandler) shouldScheduleRetry(task models.DeliveryTask, err error) bool { + if task.Manual { return false } - if !deliveryEvent.Event.EligibleForRetry { + if !task.Event.EligibleForRetry { return false } if _, ok := err.(*destregistry.ErrDestinationPublishAttempt); !ok { return false } // Attempt starts at 0 for initial attempt, so we can compare directly - return deliveryEvent.Attempt < h.retryMaxLimit + return task.Attempt < h.retryMaxLimit } func (h *messageHandler) shouldNackError(err error) bool { @@ -387,18 +366,18 @@ func (h *messageHandler) shouldNackError(err error) bool { } // Handle delivery errors - var delErr *DeliveryError - if errors.As(err, &delErr) { - return h.shouldNackDeliveryError(delErr.err) + var atmErr *AttemptError + if errors.As(err, &atmErr) { + return h.shouldNackDeliveryError(atmErr.err) } // Handle post-delivery errors var postErr *PostDeliveryError if errors.As(err, &postErr) { // Check if this wraps a delivery error - var delErr *DeliveryError - if errors.As(postErr.err, &delErr) { - return h.shouldNackDeliveryError(delErr.err) + var atmErr2 *AttemptError + if errors.As(postErr.err, &atmErr2) { + return h.shouldNackDeliveryError(atmErr2.err) } return true // Nack other post-delivery errors } @@ -415,74 +394,48 @@ func (h *messageHandler) shouldNackDeliveryError(err error) bool { return true // Nack other delivery errors } -func (h *messageHandler) scheduleRetry(ctx context.Context, deliveryEvent models.DeliveryEvent) error { - backoffDuration := h.retryBackoff.Duration(deliveryEvent.Attempt) +func (h *messageHandler) scheduleRetry(ctx context.Context, task models.DeliveryTask) error { + backoffDuration := h.retryBackoff.Duration(task.Attempt) - retryMessage := RetryMessageFromDeliveryEvent(deliveryEvent) - retryMessageStr, err := retryMessage.ToString() + retryTask := RetryTaskFromDeliveryTask(task) + retryTaskStr, err := retryTask.ToString() if err != nil { return err } - if err := h.retryScheduler.Schedule(ctx, retryMessageStr, backoffDuration, scheduler.WithTaskID(deliveryEvent.GetRetryID())); err != nil { + if err := h.retryScheduler.Schedule(ctx, retryTaskStr, backoffDuration, scheduler.WithTaskID(models.RetryID(task.Event.ID, task.DestinationID))); err != nil { h.logger.Ctx(ctx).Error("failed to schedule retry", zap.Error(err), - zap.String("delivery_event_id", deliveryEvent.ID), - zap.String("event_id", deliveryEvent.Event.ID), - zap.String("tenant_id", deliveryEvent.Event.TenantID), - zap.String("destination_id", deliveryEvent.DestinationID), - zap.Int("attempt", deliveryEvent.Attempt), + zap.String("event_id", task.Event.ID), + zap.String("tenant_id", task.Event.TenantID), + zap.String("destination_id", task.DestinationID), + zap.Int("attempt", task.Attempt), zap.Duration("backoff", backoffDuration)) return err } h.logger.Ctx(ctx).Audit("retry scheduled", - zap.String("delivery_event_id", deliveryEvent.ID), - zap.String("event_id", deliveryEvent.Event.ID), - zap.String("tenant_id", deliveryEvent.Event.TenantID), - zap.String("destination_id", deliveryEvent.DestinationID), - zap.Int("attempt", deliveryEvent.Attempt), + zap.String("event_id", task.Event.ID), + zap.String("tenant_id", task.Event.TenantID), + zap.String("destination_id", task.DestinationID), + zap.Int("attempt", task.Attempt), zap.Duration("backoff", backoffDuration)) return nil } -// ensureDeliveryEvent ensures that the delivery event struct has full data. -// In retry scenarios, the delivery event only has its ID and we'll need to query the full data. -func (h *messageHandler) ensureDeliveryEvent(ctx context.Context, deliveryEvent *models.DeliveryEvent) error { - // TODO: consider a more deliberate way to check for retry scenario? - if !deliveryEvent.Event.Time.IsZero() { - return nil - } - - event, err := h.logStore.RetrieveEvent(ctx, logstore.RetrieveEventRequest{ - TenantID: deliveryEvent.Event.TenantID, - EventID: deliveryEvent.Event.ID, - }) - if err != nil { - return err - } - if event == nil { - return errors.New("event not found") - } - deliveryEvent.Event = *event - - return nil -} - // ensurePublishableDestination ensures that the destination exists and is in a publishable state. // Returns an error if the destination is not found, deleted, disabled, or any other state that // would prevent publishing. -func (h *messageHandler) ensurePublishableDestination(ctx context.Context, deliveryEvent models.DeliveryEvent) (*models.Destination, error) { - destination, err := h.entityStore.RetrieveDestination(ctx, deliveryEvent.Event.TenantID, deliveryEvent.DestinationID) +func (h *messageHandler) ensurePublishableDestination(ctx context.Context, task models.DeliveryTask) (*models.Destination, error) { + destination, err := h.entityStore.RetrieveDestination(ctx, task.Event.TenantID, task.DestinationID) if err != nil { logger := h.logger.Ctx(ctx) fields := []zap.Field{ zap.Error(err), - zap.String("delivery_event_id", deliveryEvent.ID), - zap.String("event_id", deliveryEvent.Event.ID), - zap.String("tenant_id", deliveryEvent.Event.TenantID), - zap.String("destination_id", deliveryEvent.DestinationID), + zap.String("event_id", task.Event.ID), + zap.String("tenant_id", task.Event.TenantID), + zap.String("destination_id", task.DestinationID), } if errors.Is(err, models.ErrDestinationDeleted) { @@ -495,17 +448,15 @@ func (h *messageHandler) ensurePublishableDestination(ctx context.Context, deliv } if destination == nil { h.logger.Ctx(ctx).Info("destination not found", - zap.String("delivery_event_id", deliveryEvent.ID), - zap.String("event_id", deliveryEvent.Event.ID), - zap.String("tenant_id", deliveryEvent.Event.TenantID), - zap.String("destination_id", deliveryEvent.DestinationID)) + zap.String("event_id", task.Event.ID), + zap.String("tenant_id", task.Event.TenantID), + zap.String("destination_id", task.DestinationID)) return nil, models.ErrDestinationNotFound } if destination.DisabledAt != nil { h.logger.Ctx(ctx).Info("skipping disabled destination", - zap.String("delivery_event_id", deliveryEvent.ID), - zap.String("event_id", deliveryEvent.Event.ID), - zap.String("tenant_id", deliveryEvent.Event.TenantID), + zap.String("event_id", task.Event.ID), + zap.String("tenant_id", task.Event.TenantID), zap.String("destination_id", destination.ID), zap.String("destination_type", destination.Type), zap.Time("disabled_at", *destination.DisabledAt)) diff --git a/internal/deliverymq/messagehandler_test.go b/internal/deliverymq/messagehandler_test.go index 9f64c86e..0a3a5d75 100644 --- a/internal/deliverymq/messagehandler_test.go +++ b/internal/deliverymq/messagehandler_test.go @@ -40,8 +40,6 @@ func TestMessageHandler_DestinationGetterError(t *testing.T) { // Setup mocks destGetter := &mockDestinationGetter{err: errors.New("destination lookup failed")} - eventGetter := newMockEventGetter() - eventGetter.registerEvent(&event) retryScheduler := newMockRetryScheduler() alertMonitor := newMockAlertMonitor() @@ -50,7 +48,6 @@ func TestMessageHandler_DestinationGetterError(t *testing.T) { testutil.CreateTestLogger(t), newMockLogPublisher(nil), destGetter, - eventGetter, newMockPublisher(nil), // won't be called due to early error testutil.NewMockEventTracer(nil), retryScheduler, @@ -61,12 +58,11 @@ func TestMessageHandler_DestinationGetterError(t *testing.T) { ) // Create and handle message - deliveryEvent := models.DeliveryEvent{ - ID: idgen.DeliveryEvent(), + task := models.DeliveryTask{ Event: event, DestinationID: destination.ID, } - mockMsg, msg := newDeliveryMockMessage(deliveryEvent) + mockMsg, msg := newDeliveryMockMessage(task) // Handle message err := handler.Handle(context.Background(), msg) @@ -102,8 +98,6 @@ func TestMessageHandler_DestinationNotFound(t *testing.T) { // Setup mocks destGetter := &mockDestinationGetter{dest: nil, err: nil} // destination not found - eventGetter := newMockEventGetter() - eventGetter.registerEvent(&event) retryScheduler := newMockRetryScheduler() logPublisher := newMockLogPublisher(nil) alertMonitor := newMockAlertMonitor() @@ -113,7 +107,6 @@ func TestMessageHandler_DestinationNotFound(t *testing.T) { testutil.CreateTestLogger(t), logPublisher, destGetter, - eventGetter, newMockPublisher(nil), // won't be called testutil.NewMockEventTracer(nil), retryScheduler, @@ -124,12 +117,11 @@ func TestMessageHandler_DestinationNotFound(t *testing.T) { ) // Create and handle message - deliveryEvent := models.DeliveryEvent{ - ID: idgen.DeliveryEvent(), + task := models.DeliveryTask{ Event: event, DestinationID: destination.ID, } - mockMsg, msg := newDeliveryMockMessage(deliveryEvent) + mockMsg, msg := newDeliveryMockMessage(task) // Handle message err := handler.Handle(context.Background(), msg) @@ -139,7 +131,7 @@ func TestMessageHandler_DestinationNotFound(t *testing.T) { assert.True(t, mockMsg.nacked, "message should be nacked when destination not found") assert.False(t, mockMsg.acked, "message should not be acked when destination not found") assert.Empty(t, retryScheduler.schedules, "no retry should be scheduled") - assert.Empty(t, logPublisher.deliveries, "should not log delivery for pre-delivery error") + assert.Empty(t, logPublisher.entries, "should not log delivery for pre-delivery error") alertMonitor.AssertNotCalled(t, "HandleAttempt", mock.Anything, mock.Anything) } @@ -162,8 +154,6 @@ func TestMessageHandler_DestinationDeleted(t *testing.T) { // Setup mocks destGetter := &mockDestinationGetter{err: models.ErrDestinationDeleted} - eventGetter := newMockEventGetter() - eventGetter.registerEvent(&event) retryScheduler := newMockRetryScheduler() logPublisher := newMockLogPublisher(nil) alertMonitor := newMockAlertMonitor() @@ -173,7 +163,6 @@ func TestMessageHandler_DestinationDeleted(t *testing.T) { testutil.CreateTestLogger(t), logPublisher, destGetter, - eventGetter, newMockPublisher(nil), // won't be called testutil.NewMockEventTracer(nil), retryScheduler, @@ -184,12 +173,11 @@ func TestMessageHandler_DestinationDeleted(t *testing.T) { ) // Create and handle message - deliveryEvent := models.DeliveryEvent{ - ID: idgen.DeliveryEvent(), + task := models.DeliveryTask{ Event: event, DestinationID: destination.ID, } - mockMsg, msg := newDeliveryMockMessage(deliveryEvent) + mockMsg, msg := newDeliveryMockMessage(task) // Handle message err := handler.Handle(context.Background(), msg) @@ -199,7 +187,7 @@ func TestMessageHandler_DestinationDeleted(t *testing.T) { assert.False(t, mockMsg.nacked, "message should not be nacked when destination is deleted") assert.True(t, mockMsg.acked, "message should be acked when destination is deleted") assert.Empty(t, retryScheduler.schedules, "no retry should be scheduled") - assert.Empty(t, logPublisher.deliveries, "should not log delivery for pre-delivery error") + assert.Empty(t, logPublisher.entries, "should not log delivery for pre-delivery error") alertMonitor.AssertNotCalled(t, "HandleAttempt", mock.Anything, mock.Anything) } @@ -223,8 +211,6 @@ func TestMessageHandler_PublishError_EligibleForRetry(t *testing.T) { // Setup mocks destGetter := &mockDestinationGetter{dest: &destination} - eventGetter := newMockEventGetter() - eventGetter.registerEvent(&event) retryScheduler := newMockRetryScheduler() publishErr := &destregistry.ErrDestinationPublishAttempt{ Err: errors.New("webhook returned 429"), @@ -243,7 +229,6 @@ func TestMessageHandler_PublishError_EligibleForRetry(t *testing.T) { testutil.CreateTestLogger(t), logPublisher, destGetter, - eventGetter, publisher, testutil.NewMockEventTracer(nil), retryScheduler, @@ -254,12 +239,11 @@ func TestMessageHandler_PublishError_EligibleForRetry(t *testing.T) { ) // Create and handle message - deliveryEvent := models.DeliveryEvent{ - ID: idgen.DeliveryEvent(), + task := models.DeliveryTask{ Event: event, DestinationID: destination.ID, } - mockMsg, msg := newDeliveryMockMessage(deliveryEvent) + mockMsg, msg := newDeliveryMockMessage(task) // Handle message err := handler.Handle(context.Background(), msg) @@ -269,10 +253,10 @@ func TestMessageHandler_PublishError_EligibleForRetry(t *testing.T) { assert.False(t, mockMsg.nacked, "message should not be nacked when scheduling retry") assert.True(t, mockMsg.acked, "message should be acked when scheduling retry") assert.Len(t, retryScheduler.schedules, 1, "retry should be scheduled") - assert.Equal(t, deliveryEvent.GetRetryID(), retryScheduler.taskIDs[0], + assert.Equal(t, models.RetryID(task.Event.ID, task.DestinationID), retryScheduler.taskIDs[0], "should use GetRetryID for task ID") - require.Len(t, logPublisher.deliveries, 1, "should have one delivery") - assert.Equal(t, models.DeliveryStatusFailed, logPublisher.deliveries[0].Delivery.Status, "delivery status should be Failed") + require.Len(t, logPublisher.entries, 1, "should have one delivery") + assert.Equal(t, models.AttemptStatusFailed, logPublisher.entries[0].Attempt.Status, "delivery status should be Failed") assertAlertMonitor(t, alertMonitor, false, &destination, publishErr.Data) } @@ -296,8 +280,6 @@ func TestMessageHandler_PublishError_NotEligible(t *testing.T) { // Setup mocks destGetter := &mockDestinationGetter{dest: &destination} - eventGetter := newMockEventGetter() - eventGetter.registerEvent(&event) retryScheduler := newMockRetryScheduler() publishErr := &destregistry.ErrDestinationPublishAttempt{ Err: errors.New("webhook returned 400"), @@ -316,7 +298,6 @@ func TestMessageHandler_PublishError_NotEligible(t *testing.T) { testutil.CreateTestLogger(t), logPublisher, destGetter, - eventGetter, publisher, testutil.NewMockEventTracer(nil), retryScheduler, @@ -327,12 +308,11 @@ func TestMessageHandler_PublishError_NotEligible(t *testing.T) { ) // Create and handle message - deliveryEvent := models.DeliveryEvent{ - ID: idgen.DeliveryEvent(), + task := models.DeliveryTask{ Event: event, DestinationID: destination.ID, } - mockMsg, msg := newDeliveryMockMessage(deliveryEvent) + mockMsg, msg := newDeliveryMockMessage(task) // Handle message err := handler.Handle(context.Background(), msg) @@ -343,81 +323,15 @@ func TestMessageHandler_PublishError_NotEligible(t *testing.T) { assert.True(t, mockMsg.acked, "message should be acked for ineligible retry") assert.Empty(t, retryScheduler.schedules, "no retry should be scheduled") assert.Equal(t, 1, publisher.current, "should only attempt once") - require.Len(t, logPublisher.deliveries, 1, "should have one delivery") - assert.Equal(t, models.DeliveryStatusFailed, logPublisher.deliveries[0].Delivery.Status, "delivery status should be Failed") + require.Len(t, logPublisher.entries, 1, "should have one delivery") + assert.Equal(t, models.AttemptStatusFailed, logPublisher.entries[0].Attempt.Status, "delivery status should be Failed") assertAlertMonitor(t, alertMonitor, false, &destination, publishErr.Data) } -func TestMessageHandler_EventGetterError(t *testing.T) { - // Test scenario: - // - Event getter fails to retrieve event during retry - // - Should be treated as system error - // - Should nack for retry - - // Setup test data - tenant := models.Tenant{ID: idgen.String()} - destination := testutil.DestinationFactory.Any( - testutil.DestinationFactory.WithType("webhook"), - testutil.DestinationFactory.WithTenantID(tenant.ID), - ) - event := testutil.EventFactory.Any( - testutil.EventFactory.WithTenantID(tenant.ID), - testutil.EventFactory.WithDestinationID(destination.ID), - ) - - // Setup mocks - destGetter := &mockDestinationGetter{dest: &destination} - eventGetter := newMockEventGetter() - eventGetter.err = errors.New("failed to get event") - retryScheduler := newMockRetryScheduler() - publisher := newMockPublisher([]error{nil}) - logPublisher := newMockLogPublisher(nil) - alertMonitor := newMockAlertMonitor() - - // Setup message handler - handler := deliverymq.NewMessageHandler( - testutil.CreateTestLogger(t), - logPublisher, - destGetter, - eventGetter, - publisher, - testutil.NewMockEventTracer(nil), - retryScheduler, - &backoff.ConstantBackoff{Interval: 1 * time.Second}, - 10, - alertMonitor, - idempotence.New(testutil.CreateTestRedisClient(t), idempotence.WithSuccessfulTTL(24*time.Hour)), - ) - - // Create and handle message simulating a retry - deliveryEvent := models.DeliveryEvent{ - ID: idgen.DeliveryEvent(), - Attempt: 2, // Retry attempt - DestinationID: destination.ID, - Event: models.Event{ - ID: event.ID, - TenantID: event.TenantID, - // Minimal event data as it would be in a retry - }, - } - mockMsg, msg := newDeliveryMockMessage(deliveryEvent) - - // Handle message - err := handler.Handle(context.Background(), msg) - require.Error(t, err) - assert.Contains(t, err.Error(), "failed to get event") - - // Assert behavior - assert.True(t, mockMsg.nacked, "message should be nacked on event getter error") - assert.False(t, mockMsg.acked, "message should not be acked on event getter error") - assert.Empty(t, retryScheduler.schedules, "no retry should be scheduled for system error") - assert.Equal(t, 0, publisher.current, "publish should not be attempted") -} - func TestMessageHandler_RetryFlow(t *testing.T) { // Test scenario: // - Message is a retry attempt (Attempt > 1) - // - Event getter successfully retrieves full event data + // - DeliveryTask contains full event data (populated by retry scheduler from logstore) // - Message is processed normally // Setup test data @@ -433,8 +347,6 @@ func TestMessageHandler_RetryFlow(t *testing.T) { // Setup mocks destGetter := &mockDestinationGetter{dest: &destination} - eventGetter := newMockEventGetter() - eventGetter.registerEvent(&event) retryScheduler := newMockRetryScheduler() publisher := newMockPublisher([]error{nil}) // Successful publish logPublisher := newMockLogPublisher(nil) @@ -444,7 +356,6 @@ func TestMessageHandler_RetryFlow(t *testing.T) { testutil.CreateTestLogger(t), logPublisher, destGetter, - eventGetter, publisher, testutil.NewMockEventTracer(nil), retryScheduler, @@ -455,17 +366,13 @@ func TestMessageHandler_RetryFlow(t *testing.T) { ) // Create and handle message simulating a retry - deliveryEvent := models.DeliveryEvent{ - ID: idgen.DeliveryEvent(), + // Full event data is now populated by retry scheduler before publishing to deliverymq + task := models.DeliveryTask{ Attempt: 2, // Retry attempt DestinationID: destination.ID, - Event: models.Event{ - ID: event.ID, - TenantID: event.TenantID, - // Minimal event data as it would be in a retry - }, + Event: event, // Full event data (populated by retry scheduler) } - mockMsg, msg := newDeliveryMockMessage(deliveryEvent) + mockMsg, msg := newDeliveryMockMessage(task) // Handle message err := handler.Handle(context.Background(), msg) @@ -476,9 +383,8 @@ func TestMessageHandler_RetryFlow(t *testing.T) { assert.False(t, mockMsg.nacked, "message should not be nacked on successful retry") assert.Empty(t, retryScheduler.schedules, "no retry should be scheduled") assert.Equal(t, 1, publisher.current, "publish should succeed once") - assert.Equal(t, event.ID, eventGetter.lastRetrievedID, "event getter should be called with correct ID") - require.Len(t, logPublisher.deliveries, 1, "should have one delivery") - assert.Equal(t, models.DeliveryStatusSuccess, logPublisher.deliveries[0].Delivery.Status, "delivery status should be OK") + require.Len(t, logPublisher.entries, 1, "should have one delivery") + assert.Equal(t, models.AttemptStatusSuccess, logPublisher.entries[0].Attempt.Status, "delivery status should be OK") } func TestMessageHandler_Idempotency(t *testing.T) { @@ -500,8 +406,6 @@ func TestMessageHandler_Idempotency(t *testing.T) { // Setup mocks destGetter := &mockDestinationGetter{dest: &destination} - eventGetter := newMockEventGetter() - eventGetter.registerEvent(&event) retryScheduler := newMockRetryScheduler() publisher := newMockPublisher([]error{nil}) logPublisher := newMockLogPublisher(nil) @@ -512,7 +416,6 @@ func TestMessageHandler_Idempotency(t *testing.T) { testutil.CreateTestLogger(t), logPublisher, destGetter, - eventGetter, publisher, testutil.NewMockEventTracer(nil), retryScheduler, @@ -522,24 +425,22 @@ func TestMessageHandler_Idempotency(t *testing.T) { idempotence.New(redis, idempotence.WithSuccessfulTTL(24*time.Hour)), ) - // Create message with fixed ID for idempotency check - messageID := idgen.DeliveryEvent() - deliveryEvent := models.DeliveryEvent{ - ID: messageID, + // Create message for idempotency check + task := models.DeliveryTask{ Event: event, DestinationID: destination.ID, } // First attempt - mockMsg1, msg1 := newDeliveryMockMessage(deliveryEvent) + mockMsg1, msg1 := newDeliveryMockMessage(task) err := handler.Handle(context.Background(), msg1) require.NoError(t, err) assert.True(t, mockMsg1.acked, "first attempt should be acked") assert.False(t, mockMsg1.nacked, "first attempt should not be nacked") assert.Equal(t, 1, publisher.current, "first attempt should publish") - // Second attempt with same message ID - mockMsg2, msg2 := newDeliveryMockMessage(deliveryEvent) + // Second attempt with same task + mockMsg2, msg2 := newDeliveryMockMessage(task) err = handler.Handle(context.Background(), msg2) require.NoError(t, err) assert.True(t, mockMsg2.acked, "duplicate should be acked") @@ -549,7 +450,7 @@ func TestMessageHandler_Idempotency(t *testing.T) { func TestMessageHandler_IdempotencyWithSystemError(t *testing.T) { // Test scenario: - // - First attempt fails with system error (event getter error) + // - First attempt fails with system error (destination getter error) // - Second attempt with same message ID succeeds after error is cleared // - Should demonstrate that system errors don't affect idempotency @@ -564,11 +465,10 @@ func TestMessageHandler_IdempotencyWithSystemError(t *testing.T) { testutil.EventFactory.WithDestinationID(destination.ID), ) - // Setup mocks - destGetter := &mockDestinationGetter{dest: &destination} - eventGetter := newMockEventGetter() - eventGetter.registerEvent(&event) - eventGetter.err = errors.New("failed to get event") // Will fail first attempt + // Setup mocks - destGetter will fail first, then succeed + destGetter := newMockMultiDestinationGetter() + destGetter.registerDestination(&destination) + destGetter.err = errors.New("failed to get destination") // Will fail first attempt retryScheduler := newMockRetryScheduler() publisher := newMockPublisher([]error{nil}) logPublisher := newMockLogPublisher(nil) @@ -579,7 +479,6 @@ func TestMessageHandler_IdempotencyWithSystemError(t *testing.T) { testutil.CreateTestLogger(t), logPublisher, destGetter, - eventGetter, publisher, testutil.NewMockEventTracer(nil), retryScheduler, @@ -589,37 +488,32 @@ func TestMessageHandler_IdempotencyWithSystemError(t *testing.T) { idempotence.New(redis, idempotence.WithSuccessfulTTL(24*time.Hour)), ) - // Create retry message - deliveryEvent := models.DeliveryEvent{ - ID: idgen.DeliveryEvent(), + // Create retry message with full event data (populated by retry scheduler) + task := models.DeliveryTask{ Attempt: 2, DestinationID: destination.ID, - Event: models.Event{ - ID: event.ID, - TenantID: event.TenantID, - }, + Event: event, } // First attempt - should fail with system error - mockMsg1, msg1 := newDeliveryMockMessage(deliveryEvent) + mockMsg1, msg1 := newDeliveryMockMessage(task) err := handler.Handle(context.Background(), msg1) require.Error(t, err) - assert.Contains(t, err.Error(), "failed to get event") + assert.Contains(t, err.Error(), "failed to get destination") assert.True(t, mockMsg1.nacked, "first attempt should be nacked") assert.False(t, mockMsg1.acked, "first attempt should not be acked") assert.Equal(t, 0, publisher.current, "publish should not be attempted") // Clear the error for second attempt - eventGetter.clearError() + destGetter.err = nil - // Second attempt with same message ID - should succeed - mockMsg2, msg2 := newDeliveryMockMessage(deliveryEvent) + // Second attempt with same task - should succeed + mockMsg2, msg2 := newDeliveryMockMessage(task) err = handler.Handle(context.Background(), msg2) require.NoError(t, err) assert.True(t, mockMsg2.acked, "second attempt should be acked") assert.False(t, mockMsg2.nacked, "second attempt should not be nacked") assert.Equal(t, 1, publisher.current, "publish should succeed once") - assert.Equal(t, event.ID, eventGetter.lastRetrievedID, "event getter should be called with correct ID") } func TestMessageHandler_DestinationDisabled(t *testing.T) { @@ -643,8 +537,6 @@ func TestMessageHandler_DestinationDisabled(t *testing.T) { // Setup mocks destGetter := &mockDestinationGetter{dest: &destination} - eventGetter := newMockEventGetter() - eventGetter.registerEvent(&event) retryScheduler := newMockRetryScheduler() publisher := newMockPublisher([]error{nil}) // won't be called logPublisher := newMockLogPublisher(nil) @@ -655,7 +547,6 @@ func TestMessageHandler_DestinationDisabled(t *testing.T) { testutil.CreateTestLogger(t), logPublisher, destGetter, - eventGetter, publisher, testutil.NewMockEventTracer(nil), retryScheduler, @@ -666,12 +557,11 @@ func TestMessageHandler_DestinationDisabled(t *testing.T) { ) // Create and handle message - deliveryEvent := models.DeliveryEvent{ - ID: idgen.DeliveryEvent(), + task := models.DeliveryTask{ Event: event, DestinationID: destination.ID, } - mockMsg, msg := newDeliveryMockMessage(deliveryEvent) + mockMsg, msg := newDeliveryMockMessage(task) // Handle message err := handler.Handle(context.Background(), msg) @@ -683,7 +573,7 @@ func TestMessageHandler_DestinationDisabled(t *testing.T) { assert.Equal(t, 0, publisher.current, "should not attempt to publish to disabled destination") assert.Empty(t, retryScheduler.schedules, "should not schedule retry") assert.Empty(t, retryScheduler.canceled, "should not attempt to cancel retries") - assert.Empty(t, logPublisher.deliveries, "should not log delivery for pre-delivery error") + assert.Empty(t, logPublisher.entries, "should not log delivery for pre-delivery error") alertMonitor.AssertNotCalled(t, "HandleAttempt", mock.Anything, mock.Anything) } @@ -706,8 +596,6 @@ func TestMessageHandler_LogPublisherError(t *testing.T) { // Setup mocks destGetter := &mockDestinationGetter{dest: &destination} - eventGetter := newMockEventGetter() - eventGetter.registerEvent(&event) retryScheduler := newMockRetryScheduler() publisher := newMockPublisher([]error{nil}) // publish succeeds logPublisher := newMockLogPublisher(errors.New("log publish failed")) @@ -717,7 +605,6 @@ func TestMessageHandler_LogPublisherError(t *testing.T) { testutil.CreateTestLogger(t), logPublisher, destGetter, - eventGetter, publisher, testutil.NewMockEventTracer(nil), retryScheduler, @@ -728,12 +615,11 @@ func TestMessageHandler_LogPublisherError(t *testing.T) { ) // Create and handle message - deliveryEvent := models.DeliveryEvent{ - ID: idgen.DeliveryEvent(), + task := models.DeliveryTask{ Event: event, DestinationID: destination.ID, } - mockMsg, msg := newDeliveryMockMessage(deliveryEvent) + mockMsg, msg := newDeliveryMockMessage(task) // Handle message err := handler.Handle(context.Background(), msg) @@ -767,8 +653,6 @@ func TestMessageHandler_PublishAndLogError(t *testing.T) { // Setup mocks destGetter := &mockDestinationGetter{dest: &destination} - eventGetter := newMockEventGetter() - eventGetter.registerEvent(&event) retryScheduler := newMockRetryScheduler() publisher := newMockPublisher([]error{errors.New("publish failed")}) logPublisher := newMockLogPublisher(errors.New("log publish failed")) @@ -778,7 +662,6 @@ func TestMessageHandler_PublishAndLogError(t *testing.T) { testutil.CreateTestLogger(t), logPublisher, destGetter, - eventGetter, publisher, testutil.NewMockEventTracer(nil), retryScheduler, @@ -789,12 +672,11 @@ func TestMessageHandler_PublishAndLogError(t *testing.T) { ) // Create and handle message - deliveryEvent := models.DeliveryEvent{ - ID: idgen.DeliveryEvent(), + task := models.DeliveryTask{ Event: event, DestinationID: destination.ID, } - mockMsg, msg := newDeliveryMockMessage(deliveryEvent) + mockMsg, msg := newDeliveryMockMessage(task) // Handle message err := handler.Handle(context.Background(), msg) @@ -828,8 +710,6 @@ func TestManualDelivery_Success(t *testing.T) { // Setup mocks destGetter := &mockDestinationGetter{dest: &destination} - eventGetter := newMockEventGetter() - eventGetter.registerEvent(&event) retryScheduler := newMockRetryScheduler() publishErr := &destregistry.ErrDestinationPublishAttempt{ Err: errors.New("webhook returned 500"), @@ -845,7 +725,6 @@ func TestManualDelivery_Success(t *testing.T) { testutil.CreateTestLogger(t), logPublisher, destGetter, - eventGetter, publisher, testutil.NewMockEventTracer(nil), retryScheduler, @@ -856,26 +735,24 @@ func TestManualDelivery_Success(t *testing.T) { ) // Step 1: Automatic delivery fails and schedules retry - autoDeliveryEvent := models.DeliveryEvent{ - ID: idgen.DeliveryEvent(), + autoTask := models.DeliveryTask{ Event: event, DestinationID: destination.ID, Manual: false, } - _, autoMsg := newDeliveryMockMessage(autoDeliveryEvent) + _, autoMsg := newDeliveryMockMessage(autoTask) _ = handler.Handle(context.Background(), autoMsg) require.Len(t, retryScheduler.taskIDs, 1, "should schedule one retry") scheduledRetryID := retryScheduler.taskIDs[0] // Step 2: Manual retry succeeds and cancels pending retry - manualDeliveryEvent := models.DeliveryEvent{ - ID: idgen.DeliveryEvent(), // New delivery event ID - Event: event, // Same event - DestinationID: destination.ID, // Same destination + manualTask := models.DeliveryTask{ + Event: event, // Same event + DestinationID: destination.ID, // Same destination Manual: true, } - mockMsg, manualMsg := newDeliveryMockMessage(manualDeliveryEvent) + mockMsg, manualMsg := newDeliveryMockMessage(manualTask) err := handler.Handle(context.Background(), manualMsg) require.NoError(t, err) @@ -909,8 +786,6 @@ func TestManualDelivery_PublishError(t *testing.T) { // Setup mocks destGetter := &mockDestinationGetter{dest: &destination} - eventGetter := newMockEventGetter() - eventGetter.registerEvent(&event) retryScheduler := newMockRetryScheduler() publishErr := &destregistry.ErrDestinationPublishAttempt{ Err: errors.New("webhook returned 429"), @@ -929,7 +804,6 @@ func TestManualDelivery_PublishError(t *testing.T) { testutil.CreateTestLogger(t), logPublisher, destGetter, - eventGetter, publisher, testutil.NewMockEventTracer(nil), retryScheduler, @@ -940,13 +814,12 @@ func TestManualDelivery_PublishError(t *testing.T) { ) // Create and handle message - deliveryEvent := models.DeliveryEvent{ - ID: idgen.DeliveryEvent(), + task := models.DeliveryTask{ Event: event, DestinationID: destination.ID, Manual: true, // Manual delivery } - mockMsg, msg := newDeliveryMockMessage(deliveryEvent) + mockMsg, msg := newDeliveryMockMessage(task) // Handle message err := handler.Handle(context.Background(), msg) @@ -957,8 +830,8 @@ func TestManualDelivery_PublishError(t *testing.T) { assert.False(t, mockMsg.nacked, "message should not be nacked") assert.Equal(t, 1, publisher.current, "should attempt publish once") assert.Empty(t, retryScheduler.schedules, "should not schedule retry for manual delivery") - require.Len(t, logPublisher.deliveries, 1, "should have one delivery") - assert.Equal(t, models.DeliveryStatusFailed, logPublisher.deliveries[0].Delivery.Status, "delivery status should be Failed") + require.Len(t, logPublisher.entries, 1, "should have one delivery") + assert.Equal(t, models.AttemptStatusFailed, logPublisher.entries[0].Attempt.Status, "delivery status should be Failed") assertAlertMonitor(t, alertMonitor, false, &destination, publishErr.Data) } @@ -981,8 +854,6 @@ func TestManualDelivery_CancelError(t *testing.T) { // Setup mocks destGetter := &mockDestinationGetter{dest: &destination} - eventGetter := newMockEventGetter() - eventGetter.registerEvent(&event) retryScheduler := newMockRetryScheduler() retryScheduler.cancelResp = []error{errors.New("failed to cancel retry")} publisher := newMockPublisher([]error{nil}) // successful publish @@ -994,7 +865,6 @@ func TestManualDelivery_CancelError(t *testing.T) { testutil.CreateTestLogger(t), logPublisher, destGetter, - eventGetter, publisher, testutil.NewMockEventTracer(nil), retryScheduler, @@ -1005,13 +875,12 @@ func TestManualDelivery_CancelError(t *testing.T) { ) // Create and handle message - deliveryEvent := models.DeliveryEvent{ - ID: idgen.DeliveryEvent(), + task := models.DeliveryTask{ Event: event, DestinationID: destination.ID, Manual: true, // Manual delivery } - mockMsg, msg := newDeliveryMockMessage(deliveryEvent) + mockMsg, msg := newDeliveryMockMessage(task) // Handle message err := handler.Handle(context.Background(), msg) @@ -1023,9 +892,9 @@ func TestManualDelivery_CancelError(t *testing.T) { assert.False(t, mockMsg.acked, "message should not be acked on retry cancel error") assert.Equal(t, 1, publisher.current, "should publish once") assert.Len(t, retryScheduler.canceled, 1, "should attempt to cancel retry") - assert.Equal(t, deliveryEvent.GetRetryID(), retryScheduler.canceled[0], "should cancel with correct retry ID") - require.Len(t, logPublisher.deliveries, 1, "should have one delivery") - assert.Equal(t, models.DeliveryStatusSuccess, logPublisher.deliveries[0].Delivery.Status, "delivery status should be OK despite cancel error") + assert.Equal(t, models.RetryID(task.Event.ID, task.DestinationID), retryScheduler.canceled[0], "should cancel with correct retry ID") + require.Len(t, logPublisher.entries, 1, "should have one delivery") + assert.Equal(t, models.AttemptStatusSuccess, logPublisher.entries[0].Attempt.Status, "delivery status should be OK despite cancel error") assertAlertMonitor(t, alertMonitor, true, &destination, nil) } @@ -1049,8 +918,6 @@ func TestManualDelivery_DestinationDisabled(t *testing.T) { // Setup mocks destGetter := &mockDestinationGetter{dest: &destination} - eventGetter := newMockEventGetter() - eventGetter.registerEvent(&event) retryScheduler := newMockRetryScheduler() publisher := newMockPublisher([]error{nil}) // won't be called logPublisher := newMockLogPublisher(nil) @@ -1061,7 +928,6 @@ func TestManualDelivery_DestinationDisabled(t *testing.T) { testutil.CreateTestLogger(t), logPublisher, destGetter, - eventGetter, publisher, testutil.NewMockEventTracer(nil), retryScheduler, @@ -1072,13 +938,12 @@ func TestManualDelivery_DestinationDisabled(t *testing.T) { ) // Create and handle message - deliveryEvent := models.DeliveryEvent{ - ID: idgen.DeliveryEvent(), + task := models.DeliveryTask{ Event: event, DestinationID: destination.ID, Manual: true, // Manual delivery } - mockMsg, msg := newDeliveryMockMessage(deliveryEvent) + mockMsg, msg := newDeliveryMockMessage(task) // Handle message err := handler.Handle(context.Background(), msg) @@ -1090,7 +955,7 @@ func TestManualDelivery_DestinationDisabled(t *testing.T) { assert.Equal(t, 0, publisher.current, "should not attempt to publish to disabled destination") assert.Empty(t, retryScheduler.schedules, "should not schedule retry") assert.Empty(t, retryScheduler.canceled, "should not attempt to cancel retries") - assert.Empty(t, logPublisher.deliveries, "should not log delivery for pre-delivery error") + assert.Empty(t, logPublisher.entries, "should not log delivery for pre-delivery error") alertMonitor.AssertNotCalled(t, "HandleAttempt", mock.Anything, mock.Anything) } @@ -1112,8 +977,6 @@ func TestMessageHandler_PublishSuccess(t *testing.T) { // Setup mocks destGetter := &mockDestinationGetter{dest: &destination} - eventGetter := newMockEventGetter() - eventGetter.registerEvent(&event) retryScheduler := newMockRetryScheduler() publisher := newMockPublisher([]error{nil}) // Successful publish logPublisher := newMockLogPublisher(nil) @@ -1124,7 +987,7 @@ func TestMessageHandler_PublishSuccess(t *testing.T) { alertMonitor.On("HandleAttempt", mock.Anything, mock.MatchedBy(func(attempt alert.DeliveryAttempt) bool { return attempt.Success && // Should be a successful attempt attempt.Destination.ID == destination.ID && // Should have correct destination - attempt.DeliveryEvent != nil && // Should have delivery event + attempt.DeliveryTask != nil && // Should have delivery task attempt.DeliveryResponse == nil // No error data for success })).Return(nil) @@ -1133,7 +996,6 @@ func TestMessageHandler_PublishSuccess(t *testing.T) { testutil.CreateTestLogger(t), logPublisher, destGetter, - eventGetter, publisher, testutil.NewMockEventTracer(nil), retryScheduler, @@ -1144,12 +1006,11 @@ func TestMessageHandler_PublishSuccess(t *testing.T) { ) // Create and handle message - deliveryEvent := models.DeliveryEvent{ - ID: idgen.DeliveryEvent(), + task := models.DeliveryTask{ Event: event, DestinationID: destination.ID, } - mockMsg, msg := newDeliveryMockMessage(deliveryEvent) + mockMsg, msg := newDeliveryMockMessage(task) // Handle message err := handler.Handle(context.Background(), msg) @@ -1180,8 +1041,6 @@ func TestMessageHandler_AlertMonitorError(t *testing.T) { // Setup mocks destGetter := &mockDestinationGetter{dest: &destination} - eventGetter := newMockEventGetter() - eventGetter.registerEvent(&event) retryScheduler := newMockRetryScheduler() publisher := newMockPublisher([]error{nil}) // Successful publish logPublisher := newMockLogPublisher(nil) @@ -1193,7 +1052,6 @@ func TestMessageHandler_AlertMonitorError(t *testing.T) { testutil.CreateTestLogger(t), logPublisher, destGetter, - eventGetter, publisher, testutil.NewMockEventTracer(nil), retryScheduler, @@ -1204,12 +1062,11 @@ func TestMessageHandler_AlertMonitorError(t *testing.T) { ) // Create and handle message - deliveryEvent := models.DeliveryEvent{ - ID: idgen.DeliveryEvent(), + task := models.DeliveryTask{ Event: event, DestinationID: destination.ID, } - mockMsg, msg := newDeliveryMockMessage(deliveryEvent) + mockMsg, msg := newDeliveryMockMessage(task) // Handle message err := handler.Handle(context.Background(), msg) @@ -1219,8 +1076,8 @@ func TestMessageHandler_AlertMonitorError(t *testing.T) { assert.True(t, mockMsg.acked, "message should be acked despite alert monitor error") assert.False(t, mockMsg.nacked, "message should not be nacked despite alert monitor error") assert.Equal(t, 1, publisher.current, "should publish once") - require.Len(t, logPublisher.deliveries, 1, "should have one delivery") - assert.Equal(t, models.DeliveryStatusSuccess, logPublisher.deliveries[0].Delivery.Status, "delivery status should be OK") + require.Len(t, logPublisher.entries, 1, "should have one delivery") + assert.Equal(t, models.AttemptStatusSuccess, logPublisher.entries[0].Attempt.Status, "delivery status should be OK") // Verify alert monitor was called but error was ignored // Wait for the HandleAttempt call to be made @@ -1251,7 +1108,7 @@ func assertAlertMonitor(t *testing.T, m *mockAlertMonitor, success bool, destina assert.Equal(t, success, attempt.Success, "alert attempt success should match") assert.Equal(t, destination.ID, attempt.Destination.ID, "alert attempt destination should match") - assert.NotNil(t, attempt.DeliveryEvent, "alert attempt should have delivery event") + assert.NotNil(t, attempt.DeliveryTask, "alert attempt should have delivery task") if expectedData != nil { assert.Equal(t, expectedData, attempt.DeliveryResponse, "alert attempt data should match") @@ -1289,8 +1146,6 @@ func TestMessageHandler_RetryID_MultipleDestinations(t *testing.T) { destGetter := newMockMultiDestinationGetter() destGetter.registerDestination(&destination1) destGetter.registerDestination(&destination2) - eventGetter := newMockEventGetter() - eventGetter.registerEvent(&event) retryScheduler := newMockRetryScheduler() publishErr := &destregistry.ErrDestinationPublishAttempt{ Err: errors.New("webhook returned 500"), @@ -1306,7 +1161,6 @@ func TestMessageHandler_RetryID_MultipleDestinations(t *testing.T) { testutil.CreateTestLogger(t), logPublisher, destGetter, - eventGetter, publisher, testutil.NewMockEventTracer(nil), retryScheduler, @@ -1316,21 +1170,19 @@ func TestMessageHandler_RetryID_MultipleDestinations(t *testing.T) { idempotence.New(testutil.CreateTestRedisClient(t), idempotence.WithSuccessfulTTL(24*time.Hour)), ) - // Create delivery events for SAME event to DIFFERENT destinations - deliveryEvent1 := models.DeliveryEvent{ - ID: idgen.DeliveryEvent(), + // Create delivery tasks for SAME event to DIFFERENT destinations + task1 := models.DeliveryTask{ Event: event, DestinationID: destination1.ID, } - deliveryEvent2 := models.DeliveryEvent{ - ID: idgen.DeliveryEvent(), + task2 := models.DeliveryTask{ Event: event, DestinationID: destination2.ID, } // Handle both messages - _, msg1 := newDeliveryMockMessage(deliveryEvent1) - _, msg2 := newDeliveryMockMessage(deliveryEvent2) + _, msg1 := newDeliveryMockMessage(task1) + _, msg2 := newDeliveryMockMessage(task2) _ = handler.Handle(context.Background(), msg1) _ = handler.Handle(context.Background(), msg2) diff --git a/internal/deliverymq/mock_test.go b/internal/deliverymq/mock_test.go index 871800fe..d90f6ea8 100644 --- a/internal/deliverymq/mock_test.go +++ b/internal/deliverymq/mock_test.go @@ -3,7 +3,6 @@ package deliverymq_test import ( "context" "encoding/json" - "errors" "sync" "time" @@ -16,11 +15,6 @@ import ( "github.com/stretchr/testify/mock" ) -// scheduleOptions mirrors the private type in scheduler package -type scheduleOptions struct { - id string -} - type mockPublisher struct { responses []error current int @@ -31,47 +25,44 @@ func newMockPublisher(responses []error) *mockPublisher { return &mockPublisher{responses: responses} } -func (m *mockPublisher) PublishEvent(ctx context.Context, destination *models.Destination, event *models.Event) (*models.Delivery, error) { +func (m *mockPublisher) PublishEvent(ctx context.Context, destination *models.Destination, event *models.Event) (*models.Attempt, error) { m.mu.Lock() defer m.mu.Unlock() if m.current >= len(m.responses) { m.current++ - return &models.Delivery{ - ID: idgen.Delivery(), - DeliveryEventID: idgen.DeliveryEvent(), - EventID: event.ID, - DestinationID: destination.ID, - Status: models.DeliveryStatusSuccess, - Code: "OK", - ResponseData: map[string]interface{}{}, - Time: time.Now(), + return &models.Attempt{ + ID: idgen.Attempt(), + EventID: event.ID, + DestinationID: destination.ID, + Status: models.AttemptStatusSuccess, + Code: "OK", + ResponseData: map[string]interface{}{}, + Time: time.Now(), }, nil } resp := m.responses[m.current] m.current++ if resp == nil { - return &models.Delivery{ - ID: idgen.Delivery(), - DeliveryEventID: idgen.DeliveryEvent(), - EventID: event.ID, - DestinationID: destination.ID, - Status: models.DeliveryStatusSuccess, - Code: "OK", - ResponseData: map[string]interface{}{}, - Time: time.Now(), + return &models.Attempt{ + ID: idgen.Attempt(), + EventID: event.ID, + DestinationID: destination.ID, + Status: models.AttemptStatusSuccess, + Code: "OK", + ResponseData: map[string]interface{}{}, + Time: time.Now(), }, nil } - return &models.Delivery{ - ID: idgen.Delivery(), - DeliveryEventID: idgen.DeliveryEvent(), - EventID: event.ID, - DestinationID: destination.ID, - Status: models.DeliveryStatusFailed, - Code: "ERR", - ResponseData: map[string]interface{}{}, - Time: time.Now(), + return &models.Attempt{ + ID: idgen.Attempt(), + EventID: event.ID, + DestinationID: destination.ID, + Status: models.AttemptStatusFailed, + Code: "ERR", + ResponseData: map[string]interface{}{}, + Time: time.Now(), }, resp } @@ -140,27 +131,52 @@ func (m *mockEventGetter) RetrieveEvent(ctx context.Context, req logstore.Retrie return nil, m.err } m.lastRetrievedID = req.EventID - event, ok := m.events[req.EventID] - if !ok { - return nil, errors.New("event not found") + // Match actual logstore behavior: return (nil, nil) when event not found + return m.events[req.EventID], nil +} + +// mockDelayedEventGetter simulates the race condition where event is not yet +// persisted to logstore when retry scheduler first queries it. +// Returns (nil, nil) for the first N calls, then returns the event. +type mockDelayedEventGetter struct { + event *models.Event + callCount int + returnAfterCall int // Return event after this many calls + mu sync.Mutex +} + +func newMockDelayedEventGetter(event *models.Event, returnAfterCall int) *mockDelayedEventGetter { + return &mockDelayedEventGetter{ + event: event, + returnAfterCall: returnAfterCall, + } +} + +func (m *mockDelayedEventGetter) RetrieveEvent(ctx context.Context, req logstore.RetrieveEventRequest) (*models.Event, error) { + m.mu.Lock() + defer m.mu.Unlock() + m.callCount++ + if m.callCount <= m.returnAfterCall { + // Simulate event not yet persisted + return nil, nil } - return event, nil + return m.event, nil } type mockLogPublisher struct { - err error - deliveries []models.DeliveryEvent + err error + entries []models.LogEntry } func newMockLogPublisher(err error) *mockLogPublisher { return &mockLogPublisher{ - err: err, - deliveries: make([]models.DeliveryEvent, 0), + err: err, + entries: make([]models.LogEntry, 0), } } -func (m *mockLogPublisher) Publish(ctx context.Context, deliveryEvent models.DeliveryEvent) error { - m.deliveries = append(m.deliveries, deliveryEvent) +func (m *mockLogPublisher) Publish(ctx context.Context, entry models.LogEntry) error { + m.entries = append(m.entries, entry) return m.err } @@ -220,9 +236,9 @@ type mockMessage struct { nacked bool } -func newDeliveryMockMessage(deliveryEvent models.DeliveryEvent) (*mockMessage, *mqs.Message) { - mock := &mockMessage{id: deliveryEvent.ID} - body, err := json.Marshal(deliveryEvent) +func newDeliveryMockMessage(task models.DeliveryTask) (*mockMessage, *mqs.Message) { + mock := &mockMessage{id: task.IdempotencyKey()} + body, err := json.Marshal(task) if err != nil { panic(err) } @@ -232,14 +248,6 @@ func newDeliveryMockMessage(deliveryEvent models.DeliveryEvent) (*mockMessage, * } } -func newMockMessage(id string) *mqs.Message { - mock := &mockMessage{id: id} - return &mqs.Message{ - QueueMessage: mock, - Body: nil, - } -} - func (m *mockMessage) ID() string { return m.id } diff --git a/internal/deliverymq/retry.go b/internal/deliverymq/retry.go index e4a5b086..5dd78f71 100644 --- a/internal/deliverymq/retry.go +++ b/internal/deliverymq/retry.go @@ -7,21 +7,48 @@ import ( "time" "github.com/hookdeck/outpost/internal/logging" + "github.com/hookdeck/outpost/internal/logstore" "github.com/hookdeck/outpost/internal/models" "github.com/hookdeck/outpost/internal/redis" "github.com/hookdeck/outpost/internal/rsmq" "github.com/hookdeck/outpost/internal/scheduler" + "go.uber.org/zap" ) -func NewRetryScheduler(deliverymq *DeliveryMQ, redisConfig *redis.RedisConfig, deploymentID string, pollBackoff time.Duration, logger *logging.Logger) (scheduler.Scheduler, error) { - // Create Redis client for RSMQ +// RetryEventGetter is the interface for fetching events from logstore. +// This is defined separately from EventGetter in messagehandler.go to avoid circular dependencies. +type RetryEventGetter interface { + RetrieveEvent(ctx context.Context, request logstore.RetrieveEventRequest) (*models.Event, error) +} + +// RetrySchedulerOption is a functional option for configuring the retry scheduler. +type RetrySchedulerOption func(*retrySchedulerConfig) + +type retrySchedulerConfig struct { + visibilityTimeout uint +} + +// WithRetryVisibilityTimeout sets the visibility timeout for the retry scheduler queue. +// This controls how long a message is hidden after being received before it becomes +// visible again (for retry if the executor returned an error). +func WithRetryVisibilityTimeout(vt uint) RetrySchedulerOption { + return func(c *retrySchedulerConfig) { + c.visibilityTimeout = vt + } +} + +func NewRetryScheduler(deliverymq *DeliveryMQ, redisConfig *redis.RedisConfig, deploymentID string, pollBackoff time.Duration, logger *logging.Logger, eventGetter RetryEventGetter, opts ...RetrySchedulerOption) (scheduler.Scheduler, error) { + cfg := &retrySchedulerConfig{} + for _, opt := range opts { + opt(cfg) + } + ctx := context.Background() redisClient, err := redis.New(ctx, redisConfig) if err != nil { return nil, fmt.Errorf("failed to create Redis client for retry scheduler: %w", err) } - // Create RSMQ adapter adapter := rsmq.NewRedisAdapter(redisClient) // Construct RSMQ namespace with deployment prefix if provided @@ -32,7 +59,6 @@ func NewRetryScheduler(deliverymq *DeliveryMQ, redisConfig *redis.RedisConfig, d namespace = fmt.Sprintf("%s:rsmq", deploymentID) } - // Create RSMQ client with deployment-aware namespace var rsmqClient *rsmq.RedisSMQ if logger != nil { rsmqClient = rsmq.NewRedisSMQ(adapter, namespace, logger) @@ -40,32 +66,69 @@ func NewRetryScheduler(deliverymq *DeliveryMQ, redisConfig *redis.RedisConfig, d rsmqClient = rsmq.NewRedisSMQ(adapter, namespace) } - // Define execution function exec := func(ctx context.Context, msg string) error { - retryMessage := RetryMessage{} - if err := retryMessage.FromString(msg); err != nil { + retryTask := RetryTask{} + if err := retryTask.FromString(msg); err != nil { + return err + } + + // Fetch full event data from logstore + event, err := eventGetter.RetrieveEvent(ctx, logstore.RetrieveEventRequest{ + TenantID: retryTask.TenantID, + EventID: retryTask.EventID, + }) + if err != nil { + // Returning an error leaves the message in the RSMQ queue. After the + // visibility timeout expires, the message becomes visible again and will + // be reprocessed. This handles both transient DB errors and the race + // condition where logmq hasn't flushed the event yet. + if logger != nil { + logger.Ctx(ctx).Error("failed to fetch event for retry", + zap.Error(err), + zap.String("event_id", retryTask.EventID), + zap.String("tenant_id", retryTask.TenantID), + zap.String("destination_id", retryTask.DestinationID)) + } return err } - deliveryEvent := retryMessage.ToDeliveryEvent() - if err := deliverymq.Publish(ctx, deliveryEvent); err != nil { + if event == nil { + // Event not found - may be race condition with logmq batching delay. + // Return error so scheduler retries later. + if logger != nil { + logger.Ctx(ctx).Warn("event not found in logstore, will retry", + zap.String("event_id", retryTask.EventID), + zap.String("tenant_id", retryTask.TenantID), + zap.String("destination_id", retryTask.DestinationID)) + } + return fmt.Errorf("event not found in logstore") + } + + deliveryTask := retryTask.ToDeliveryTask(*event) + if err := deliverymq.Publish(ctx, deliveryTask); err != nil { return err } return nil } + if cfg.visibilityTimeout > 0 { + return scheduler.New("deliverymq-retry", rsmqClient, exec, + scheduler.WithPollBackoff(pollBackoff), + scheduler.WithVisibilityTimeout(cfg.visibilityTimeout)), nil + } return scheduler.New("deliverymq-retry", rsmqClient, exec, scheduler.WithPollBackoff(pollBackoff)), nil } -type RetryMessage struct { - DeliveryEventID string - EventID string - TenantID string - DestinationID string - Attempt int - Telemetry *models.DeliveryEventTelemetry +// RetryTask contains the minimal info needed to retry a delivery. +// The full Event data will be fetched from logstore when the retry executes. +type RetryTask struct { + EventID string + TenantID string + DestinationID string + Attempt int + Telemetry *models.DeliveryTelemetry } -func (m *RetryMessage) ToString() (string, error) { +func (m *RetryTask) ToString() (string, error) { json, err := json.Marshal(m) if err != nil { return "", err @@ -73,27 +136,25 @@ func (m *RetryMessage) ToString() (string, error) { return string(json), nil } -func (m *RetryMessage) FromString(str string) error { +func (m *RetryTask) FromString(str string) error { return json.Unmarshal([]byte(str), &m) } -func (m *RetryMessage) ToDeliveryEvent() models.DeliveryEvent { - return models.DeliveryEvent{ - ID: m.DeliveryEventID, +func (m *RetryTask) ToDeliveryTask(event models.Event) models.DeliveryTask { + return models.DeliveryTask{ Attempt: m.Attempt, DestinationID: m.DestinationID, - Event: models.Event{ID: m.EventID, TenantID: m.TenantID}, + Event: event, Telemetry: m.Telemetry, } } -func RetryMessageFromDeliveryEvent(deliveryEvent models.DeliveryEvent) RetryMessage { - return RetryMessage{ - DeliveryEventID: deliveryEvent.ID, - EventID: deliveryEvent.Event.ID, - TenantID: deliveryEvent.Event.TenantID, - DestinationID: deliveryEvent.DestinationID, - Attempt: deliveryEvent.Attempt + 1, - Telemetry: deliveryEvent.Telemetry, +func RetryTaskFromDeliveryTask(task models.DeliveryTask) RetryTask { + return RetryTask{ + EventID: task.Event.ID, + TenantID: task.Event.TenantID, + DestinationID: task.DestinationID, + Attempt: task.Attempt + 1, + Telemetry: task.Telemetry, } } diff --git a/internal/deliverymq/retry_test.go b/internal/deliverymq/retry_test.go index 10a1b012..86257a96 100644 --- a/internal/deliverymq/retry_test.go +++ b/internal/deliverymq/retry_test.go @@ -25,7 +25,7 @@ type RetryDeliveryMQSuite struct { retryBackoff backoff.Backoff schedulerPollBackoff time.Duration publisher deliverymq.Publisher - eventGetter deliverymq.EventGetter + eventGetter deliverymq.RetryEventGetter logPublisher deliverymq.LogPublisher destGetter deliverymq.DestinationGetter alertMonitor deliverymq.AlertMonitor @@ -53,7 +53,7 @@ func (s *RetryDeliveryMQSuite) SetupTest(t *testing.T) { if pollBackoff == 0 { pollBackoff = 100 * time.Millisecond } - retryScheduler, err := deliverymq.NewRetryScheduler(s.deliveryMQ, testutil.CreateTestRedisConfig(t), "", pollBackoff, testutil.CreateTestLogger(t)) + retryScheduler, err := deliverymq.NewRetryScheduler(s.deliveryMQ, testutil.CreateTestRedisConfig(t), "", pollBackoff, testutil.CreateTestLogger(t), s.eventGetter) require.NoError(t, err) require.NoError(t, retryScheduler.Init(s.ctx)) go retryScheduler.Monitor(s.ctx) @@ -68,7 +68,6 @@ func (s *RetryDeliveryMQSuite) SetupTest(t *testing.T) { testutil.CreateTestLogger(t), s.logPublisher, s.destGetter, - s.eventGetter, s.publisher, testutil.NewMockEventTracer(nil), retryScheduler, @@ -150,12 +149,11 @@ func TestDeliveryMQRetry_EligibleForRetryFalse(t *testing.T) { suite.SetupTest(t) defer suite.TeardownTest(t) - deliveryEvent := models.DeliveryEvent{ - ID: idgen.DeliveryEvent(), + task := models.DeliveryTask{ Event: event, DestinationID: destination.ID, } - require.NoError(t, suite.deliveryMQ.Publish(ctx, deliveryEvent)) + require.NoError(t, suite.deliveryMQ.Publish(ctx, task)) <-ctx.Done() assert.Equal(t, 1, publisher.Current(), "should only attempt once when retry is not eligible") @@ -213,12 +211,11 @@ func TestDeliveryMQRetry_EligibleForRetryTrue(t *testing.T) { suite.SetupTest(t) defer suite.TeardownTest(t) - deliveryEvent := models.DeliveryEvent{ - ID: idgen.DeliveryEvent(), + task := models.DeliveryTask{ Event: event, DestinationID: destination.ID, } - require.NoError(t, suite.deliveryMQ.Publish(ctx, deliveryEvent)) + require.NoError(t, suite.deliveryMQ.Publish(ctx, task)) // Wait for all attempts to complete // Note: 50ms backoff + 10ms poll interval = fast, deterministic retries @@ -271,12 +268,11 @@ func TestDeliveryMQRetry_SystemError(t *testing.T) { suite.SetupTest(t) defer suite.TeardownTest(t) - deliveryEvent := models.DeliveryEvent{ - ID: idgen.DeliveryEvent(), + task := models.DeliveryTask{ Event: event, DestinationID: destination.ID, } - require.NoError(t, suite.deliveryMQ.Publish(ctx, deliveryEvent)) + require.NoError(t, suite.deliveryMQ.Publish(ctx, task)) <-ctx.Done() assert.Greater(t, destGetter.current, 1, "handler should execute multiple times on system error") @@ -341,12 +337,11 @@ func TestDeliveryMQRetry_RetryMaxCount(t *testing.T) { suite.SetupTest(t) defer suite.TeardownTest(t) - deliveryEvent := models.DeliveryEvent{ - ID: idgen.DeliveryEvent(), + task := models.DeliveryTask{ Event: event, DestinationID: destination.ID, } - require.NoError(t, suite.deliveryMQ.Publish(ctx, deliveryEvent)) + require.NoError(t, suite.deliveryMQ.Publish(ctx, task)) // Poll until we get 3 attempts or timeout // With 50ms backoff + 10ms poll: initial + 60ms + retry + 60ms + retry = ~150ms minimum @@ -356,3 +351,334 @@ func TestDeliveryMQRetry_RetryMaxCount(t *testing.T) { assert.Equal(t, 3, publisher.Current(), "should stop after max retries (1 initial + 2 retries = 3 total attempts)") } + +func TestRetryScheduler_EventNotFound(t *testing.T) { + // Test scenario: + // - Initial delivery fails and schedules a retry + // - Before retry executes, the event is deleted from logstore + // - Retry scheduler should skip publishing (not error) when event returns (nil, nil) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + // Setup test data + tenant := models.Tenant{ID: idgen.String()} + destination := testutil.DestinationFactory.Any( + testutil.DestinationFactory.WithType("webhook"), + testutil.DestinationFactory.WithTenantID(tenant.ID), + ) + event := testutil.EventFactory.Any( + testutil.EventFactory.WithTenantID(tenant.ID), + testutil.EventFactory.WithDestinationID(destination.ID), + testutil.EventFactory.WithEligibleForRetry(true), + ) + + // Setup mocks - publisher fails on first attempt + publisher := newMockPublisher([]error{ + &destregistry.ErrDestinationPublishAttempt{ + Err: errors.New("webhook returned 503"), + Provider: "webhook", + }, + }) + + // Event getter does NOT have the event registered + // This simulates event being deleted from logstore before retry + eventGetter := newMockEventGetter() + // Intentionally NOT calling: eventGetter.registerEvent(&event) + + suite := &RetryDeliveryMQSuite{ + ctx: ctx, + mqConfig: &mqs.QueueConfig{InMemory: &mqs.InMemoryConfig{Name: testutil.RandomString(5)}}, + publisher: publisher, + eventGetter: eventGetter, + logPublisher: newMockLogPublisher(nil), + destGetter: &mockDestinationGetter{dest: &destination}, + alertMonitor: newMockAlertMonitor(), + retryMaxCount: 10, + retryBackoff: &backoff.ConstantBackoff{Interval: 50 * time.Millisecond}, + schedulerPollBackoff: 10 * time.Millisecond, + } + suite.SetupTest(t) + defer suite.TeardownTest(t) + + // Publish task with full event data (simulates initial delivery) + task := models.DeliveryTask{ + Event: event, + DestinationID: destination.ID, + } + require.NoError(t, suite.deliveryMQ.Publish(ctx, task)) + + // Wait for initial delivery attempt and retry scheduling + require.Eventually(t, func() bool { + return publisher.Current() >= 1 + }, 2*time.Second, 10*time.Millisecond, "should complete initial delivery attempt") + + // Wait enough time for retry to be processed (if it were to happen) + // 50ms backoff + 10ms poll = 60ms minimum for retry + time.Sleep(200 * time.Millisecond) + + // Should only have 1 attempt - the retry was skipped because event not found + assert.Equal(t, 1, publisher.Current(), "should skip retry when event not found in logstore (returns nil, nil)") +} + +func TestRetryScheduler_EventFetchError(t *testing.T) { + // Test scenario: + // - Initial delivery fails and schedules a retry + // - When retry scheduler tries to fetch event, it gets a transient error + // - Retry scheduler should return error (which means message is not deleted) + // - The message stays in queue for retry after visibility timeout + // - Delivery should NOT proceed when event fetch fails + + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + + // Setup test data + tenant := models.Tenant{ID: idgen.String()} + destination := testutil.DestinationFactory.Any( + testutil.DestinationFactory.WithType("webhook"), + testutil.DestinationFactory.WithTenantID(tenant.ID), + ) + event := testutil.EventFactory.Any( + testutil.EventFactory.WithTenantID(tenant.ID), + testutil.EventFactory.WithDestinationID(destination.ID), + testutil.EventFactory.WithEligibleForRetry(true), + ) + + // Setup mocks - publisher fails on first attempt + publisher := newMockPublisher([]error{ + &destregistry.ErrDestinationPublishAttempt{ + Err: errors.New("webhook returned 503"), + Provider: "webhook", + }, + nil, // Second attempt would succeed if it were reached + }) + + // Event getter returns error (simulating transient DB error) + eventGetter := newMockEventGetter() + eventGetter.registerEvent(&event) + eventGetter.err = errors.New("database connection error") + + suite := &RetryDeliveryMQSuite{ + ctx: ctx, + mqConfig: &mqs.QueueConfig{InMemory: &mqs.InMemoryConfig{Name: testutil.RandomString(5)}}, + publisher: publisher, + eventGetter: eventGetter, + logPublisher: newMockLogPublisher(nil), + destGetter: &mockDestinationGetter{dest: &destination}, + alertMonitor: newMockAlertMonitor(), + retryMaxCount: 10, + retryBackoff: &backoff.ConstantBackoff{Interval: 50 * time.Millisecond}, + schedulerPollBackoff: 10 * time.Millisecond, + } + suite.SetupTest(t) + defer suite.TeardownTest(t) + + // Publish task with full event data (simulates initial delivery) + task := models.DeliveryTask{ + Event: event, + DestinationID: destination.ID, + } + require.NoError(t, suite.deliveryMQ.Publish(ctx, task)) + + // Wait for initial delivery attempt + require.Eventually(t, func() bool { + return publisher.Current() >= 1 + }, 2*time.Second, 10*time.Millisecond, "should complete initial delivery attempt") + + // Wait enough time for retry to be attempted (but it should fail with event fetch error) + // 50ms backoff + 10ms poll = 60ms minimum for retry attempt + time.Sleep(200 * time.Millisecond) + + // Delivery should still be at 1 because event fetch error prevented retry delivery + // Note: The retry message is NOT deleted, it will be retried after visibility timeout (30s) + assert.Equal(t, 1, publisher.Current(), "retry delivery should not proceed when event fetch fails") +} + +func TestRetryScheduler_EventFetchSuccess(t *testing.T) { + // Test scenario: + // - Initial delivery fails and schedules a retry + // - Retry scheduler successfully fetches event from logstore + // - DeliveryTask published to deliverymq should have full event data (non-zero Time) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + // Setup test data + tenant := models.Tenant{ID: idgen.String()} + destination := testutil.DestinationFactory.Any( + testutil.DestinationFactory.WithType("webhook"), + testutil.DestinationFactory.WithTenantID(tenant.ID), + ) + event := testutil.EventFactory.Any( + testutil.EventFactory.WithTenantID(tenant.ID), + testutil.EventFactory.WithDestinationID(destination.ID), + testutil.EventFactory.WithEligibleForRetry(true), + ) + + // Setup mocks - publisher fails on first attempt, succeeds on second + publisher := newMockPublisher([]error{ + &destregistry.ErrDestinationPublishAttempt{ + Err: errors.New("webhook returned 503"), + Provider: "webhook", + }, + nil, // Second attempt succeeds + }) + + // Event getter has the event registered + eventGetter := newMockEventGetter() + eventGetter.registerEvent(&event) + + logPublisher := newMockLogPublisher(nil) + + suite := &RetryDeliveryMQSuite{ + ctx: ctx, + mqConfig: &mqs.QueueConfig{InMemory: &mqs.InMemoryConfig{Name: testutil.RandomString(5)}}, + publisher: publisher, + eventGetter: eventGetter, + logPublisher: logPublisher, + destGetter: &mockDestinationGetter{dest: &destination}, + alertMonitor: newMockAlertMonitor(), + retryMaxCount: 10, + retryBackoff: &backoff.ConstantBackoff{Interval: 50 * time.Millisecond}, + schedulerPollBackoff: 10 * time.Millisecond, + } + suite.SetupTest(t) + defer suite.TeardownTest(t) + + // Publish task with full event data (simulates initial delivery) + task := models.DeliveryTask{ + Event: event, + DestinationID: destination.ID, + } + require.NoError(t, suite.deliveryMQ.Publish(ctx, task)) + + // Wait for both delivery attempts to complete + require.Eventually(t, func() bool { + return publisher.Current() >= 2 + }, 3*time.Second, 10*time.Millisecond, "should complete 2 delivery attempts") + + assert.Equal(t, 2, publisher.Current(), "should complete 2 delivery attempts (initial failure + successful retry)") + + // Verify that the retry delivery had full event data by checking log entries + require.Len(t, logPublisher.entries, 2, "should have 2 delivery log entries") + + // Both log entries should have non-zero event Time (full event data) + assert.False(t, logPublisher.entries[0].Event.Time.IsZero(), "first delivery should have full event data") + assert.False(t, logPublisher.entries[1].Event.Time.IsZero(), "retry delivery should have full event data (fetched from logstore)") +} + +// TestRetryScheduler_RaceCondition_EventNotYetPersisted verifies that retries are not +// lost when the retry scheduler queries logstore before the event has been persisted. +// +// Scenario: +// 1. Initial delivery fails, retry is scheduled +// 2. Retry scheduler runs and queries logstore for event data +// 3. Event is not yet persisted (logmq batching delay) +// 4. Retry should remain in queue and be reprocessed later +// 5. Once event is available, retry succeeds +func TestRetryScheduler_RaceCondition_EventNotYetPersisted(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + // Setup test data + tenant := models.Tenant{ID: idgen.String()} + destination := testutil.DestinationFactory.Any( + testutil.DestinationFactory.WithType("webhook"), + testutil.DestinationFactory.WithTenantID(tenant.ID), + ) + event := testutil.EventFactory.Any( + testutil.EventFactory.WithTenantID(tenant.ID), + testutil.EventFactory.WithDestinationID(destination.ID), + testutil.EventFactory.WithEligibleForRetry(true), + ) + + // Publisher: fails first attempt, succeeds after + publisher := newMockPublisher([]error{ + &destregistry.ErrDestinationPublishAttempt{ + Err: errors.New("webhook returned 503"), + Provider: "webhook", + }, + }) + logPublisher := newMockLogPublisher(nil) + destGetter := &mockDestinationGetter{dest: &destination} + alertMonitor := newMockAlertMonitor() + + // Event getter returns (nil, nil) on first call, then returns event + // This simulates: logmq hasn't persisted the event yet when retry first runs + eventGetter := newMockDelayedEventGetter(&event, 1) // Return nil for first call + + // Setup deliveryMQ + mqConfig := &mqs.QueueConfig{InMemory: &mqs.InMemoryConfig{Name: testutil.RandomString(5)}} + deliveryMQ := deliverymq.New(deliverymq.WithQueue(mqConfig)) + cleanup, err := deliveryMQ.Init(ctx) + require.NoError(t, err) + defer cleanup() + + // Setup retry scheduler with short visibility timeout for faster test + // When event is not found, the message will be retried after 1 second + retryScheduler, err := deliverymq.NewRetryScheduler( + deliveryMQ, + testutil.CreateTestRedisConfig(t), + "", + 10*time.Millisecond, // Fast polling + testutil.CreateTestLogger(t), + eventGetter, + deliverymq.WithRetryVisibilityTimeout(1), // 1 second visibility timeout + ) + require.NoError(t, err) + require.NoError(t, retryScheduler.Init(ctx)) + go retryScheduler.Monitor(ctx) + defer retryScheduler.Shutdown() + + // Setup message handler with short retry backoff + handler := deliverymq.NewMessageHandler( + testutil.CreateTestLogger(t), + logPublisher, + destGetter, + publisher, + testutil.NewMockEventTracer(nil), + retryScheduler, + &backoff.ConstantBackoff{Interval: 50 * time.Millisecond}, // Short backoff + 10, + alertMonitor, + idempotence.New(testutil.CreateTestRedisClient(t), idempotence.WithSuccessfulTTL(24*time.Hour)), + ) + + // Setup message consumer + mq := mqs.NewQueue(mqConfig) + subscription, err := mq.Subscribe(ctx) + require.NoError(t, err) + defer subscription.Shutdown(ctx) + + go func() { + for { + msg, err := subscription.Receive(ctx) + if err != nil { + return + } + handler.Handle(ctx, msg) + } + }() + + // Publish task with full event data (simulates initial delivery) + task := models.DeliveryTask{ + Event: event, + DestinationID: destination.ID, + } + require.NoError(t, deliveryMQ.Publish(ctx, task)) + + // Wait for initial delivery to fail and retry to be scheduled + require.Eventually(t, func() bool { + return publisher.Current() >= 1 + }, 2*time.Second, 10*time.Millisecond, "initial delivery should complete") + + // Wait for retry to be processed: + // - First retry attempt: event not found, message returns to queue + // - After 1s visibility timeout: message becomes visible again + // - Second retry attempt: event now available, delivery succeeds + time.Sleep(2 * time.Second) + + // Should have 2 publish attempts: initial failure + successful retry + assert.Equal(t, 2, publisher.Current(), + "expected 2 delivery attempts (initial + retry after event becomes available)") +} diff --git a/internal/deliverymq/tracer_test.go b/internal/deliverymq/tracer_test.go index 307ef4c3..788d473f 100644 --- a/internal/deliverymq/tracer_test.go +++ b/internal/deliverymq/tracer_test.go @@ -8,7 +8,7 @@ and metrics functionality. These are recommendations and can be adapted based on Potential Test Scenarios: 1. Successful Delivery - - Verify eventTracer.Deliver() is called with correct DeliveryEvent + - Verify eventTracer.Deliver() is called with correct DeliveryTask - Verify span is created and ended properly - Verify metrics are recorded: * Delivery latency diff --git a/internal/destinationmockserver/model.go b/internal/destinationmockserver/model.go index d4281fd9..dc638874 100644 --- a/internal/destinationmockserver/model.go +++ b/internal/destinationmockserver/model.go @@ -195,7 +195,6 @@ func verifySignature(secret string, payload []byte, signature string, algorithm return false } - // Create a new signature manager with the secret secrets := []destwebhook.WebhookSecret{ { Key: secret, @@ -209,7 +208,6 @@ func verifySignature(secret string, payload []byte, signature string, algorithm destwebhook.WithAlgorithm(destwebhook.GetAlgorithm(algorithm)), ) - // Try each signature for _, sig := range signatures { if sm.VerifySignature(sig, secret, destwebhook.SignaturePayload{ Body: string(payload), diff --git a/internal/destinationmockserver/server.go b/internal/destinationmockserver/server.go index 3b2b6664..9821a83a 100644 --- a/internal/destinationmockserver/server.go +++ b/internal/destinationmockserver/server.go @@ -23,7 +23,6 @@ type DestinationMockServer struct { func (s *DestinationMockServer) Run(ctx context.Context) error { go func() { - // service connections if err := s.server.ListenAndServe(); err != nil && err != http.ErrServerClosed { s.logger.Fatal("listen: %s\n", zap.Error(err)) } diff --git a/internal/destregistry/registry.go b/internal/destregistry/registry.go index 8b670da0..b11763f4 100644 --- a/internal/destregistry/registry.go +++ b/internal/destregistry/registry.go @@ -25,7 +25,7 @@ type PreprocessDestinationOpts struct { type Registry interface { // Operations ValidateDestination(ctx context.Context, destination *models.Destination) error - PublishEvent(ctx context.Context, destination *models.Destination, event *models.Event) (*models.Delivery, error) + PublishEvent(ctx context.Context, destination *models.Destination, event *models.Event) (*models.Attempt, error) DisplayDestination(destination *models.Destination) (*DestinationDisplay, error) PreprocessDestination(newDestination *models.Destination, originalDestination *models.Destination, opts *PreprocessDestinationOpts) error @@ -135,14 +135,14 @@ func (r *registry) ValidateDestination(ctx context.Context, destination *models. return nil } -func (r *registry) PublishEvent(ctx context.Context, destination *models.Destination, event *models.Event) (*models.Delivery, error) { +func (r *registry) PublishEvent(ctx context.Context, destination *models.Destination, event *models.Event) (*models.Attempt, error) { publisher, err := r.ResolvePublisher(ctx, destination) if err != nil { return nil, err } - delivery := &models.Delivery{ - ID: idgen.Delivery(), + attempt := &models.Attempt{ + ID: idgen.Attempt(), DestinationID: destination.ID, EventID: event.ID, } @@ -153,7 +153,7 @@ func (r *registry) PublishEvent(ctx context.Context, destination *models.Destina deliveryData, err := publisher.Publish(timeoutCtx, event) if err != nil { - // Context canceled = system shutdown, return nil delivery to trigger nack → requeue. + // Context canceled = system shutdown, return nil attempt to trigger nack → requeue. // This is handled centrally so individual publishers don't need to check for it. // See: https://github.com/hookdeck/outpost/issues/571 if errors.Is(err, context.Canceled) { @@ -161,18 +161,18 @@ func (r *registry) PublishEvent(ctx context.Context, destination *models.Destina } if deliveryData != nil { - delivery.Time = time.Now() - delivery.Status = deliveryData.Status - delivery.Code = deliveryData.Code - delivery.ResponseData = deliveryData.Response + attempt.Time = time.Now() + attempt.Status = deliveryData.Status + attempt.Code = deliveryData.Code + attempt.ResponseData = deliveryData.Response } else { - delivery = nil + attempt = nil } var publishErr *ErrDestinationPublishAttempt if errors.As(err, &publishErr) { // Check if the wrapped error is a timeout if errors.Is(publishErr.Err, context.DeadlineExceeded) { - return delivery, &ErrDestinationPublishAttempt{ + return attempt, &ErrDestinationPublishAttempt{ Err: publishErr.Err, Provider: destination.Type, Data: map[string]interface{}{ @@ -181,11 +181,11 @@ func (r *registry) PublishEvent(ctx context.Context, destination *models.Destina }, } } - return delivery, publishErr + return attempt, publishErr } if errors.Is(err, context.DeadlineExceeded) { - return delivery, &ErrDestinationPublishAttempt{ + return attempt, &ErrDestinationPublishAttempt{ Err: err, Provider: destination.Type, Data: map[string]interface{}{ @@ -195,7 +195,7 @@ func (r *registry) PublishEvent(ctx context.Context, destination *models.Destina } } - return delivery, &ErrDestinationPublishAttempt{ + return attempt, &ErrDestinationPublishAttempt{ Err: err, Provider: destination.Type, Data: map[string]interface{}{ @@ -217,12 +217,12 @@ func (r *registry) PublishEvent(ctx context.Context, destination *models.Destina } } - delivery.Time = time.Now() - delivery.Status = deliveryData.Status - delivery.Code = deliveryData.Code - delivery.ResponseData = deliveryData.Response + attempt.Time = time.Now() + attempt.Status = deliveryData.Status + attempt.Code = deliveryData.Code + attempt.ResponseData = deliveryData.Response - return delivery, nil + return attempt, nil } func (r *registry) RegisterProvider(destinationType string, provider Provider) error { diff --git a/internal/emetrics/emetrics.go b/internal/emetrics/emetrics.go index c7b2dd93..5b2f3223 100644 --- a/internal/emetrics/emetrics.go +++ b/internal/emetrics/emetrics.go @@ -12,7 +12,7 @@ import ( type OutpostMetrics interface { DeliveryLatency(ctx context.Context, latency time.Duration, opts DeliveryLatencyOpts) - EventDelivered(ctx context.Context, deliveryEvent *models.DeliveryEvent, ok bool, destinationType string) + EventDelivered(ctx context.Context, ok bool, destinationType string) EventPublished(ctx context.Context, event *models.Event) EventEligbible(ctx context.Context, event *models.Event) APIResponseLatency(ctx context.Context, latency time.Duration, opts APIResponseLatencyOpts) @@ -99,12 +99,12 @@ func (e *emetricsImpl) DeliveryLatency(ctx context.Context, latency time.Duratio e.deliveryLatency.Record(ctx, latency.Milliseconds(), metric.WithAttributes(attribute.String("type", opts.Type))) } -func (e *emetricsImpl) EventDelivered(ctx context.Context, deliveryEvent *models.DeliveryEvent, ok bool, destinationType string) { +func (e *emetricsImpl) EventDelivered(ctx context.Context, ok bool, destinationType string) { var status string if ok { - status = models.DeliveryStatusSuccess + status = models.AttemptStatusSuccess } else { - status = models.DeliveryStatusFailed + status = models.AttemptStatusFailed } e.eventDeliveredCounter.Add(ctx, 1, metric.WithAttributes( attribute.String("type", destinationType), diff --git a/internal/eventtracer/eventtracer.go b/internal/eventtracer/eventtracer.go index 52a1d820..de669245 100644 --- a/internal/eventtracer/eventtracer.go +++ b/internal/eventtracer/eventtracer.go @@ -12,8 +12,8 @@ import ( type EventTracer interface { Receive(context.Context, *models.Event) (context.Context, trace.Span) - StartDelivery(context.Context, *models.DeliveryEvent) (context.Context, trace.Span) - Deliver(context.Context, *models.DeliveryEvent, *models.Destination) (context.Context, trace.Span) + StartDelivery(context.Context, *models.DeliveryTask) (context.Context, trace.Span) + Deliver(context.Context, *models.DeliveryTask, *models.Destination) (context.Context, trace.Span) } type eventTracerImpl struct { @@ -47,10 +47,10 @@ func (t *eventTracerImpl) Receive(ctx context.Context, event *models.Event) (con return ctx, span } -func (t *eventTracerImpl) StartDelivery(_ context.Context, deliveryEvent *models.DeliveryEvent) (context.Context, trace.Span) { - ctx, span := t.tracer.Start(t.getRemoteEventSpanContext(&deliveryEvent.Event), "EventTracer.StartDelivery") +func (t *eventTracerImpl) StartDelivery(_ context.Context, task *models.DeliveryTask) (context.Context, trace.Span) { + ctx, span := t.tracer.Start(t.getRemoteEventSpanContext(&task.Event), "EventTracer.StartDelivery") - deliveryEvent.Telemetry = &models.DeliveryEventTelemetry{ + task.Telemetry = &models.DeliveryTelemetry{ TraceID: span.SpanContext().TraceID().String(), SpanID: span.SpanContext().SpanID().String(), } @@ -60,10 +60,10 @@ func (t *eventTracerImpl) StartDelivery(_ context.Context, deliveryEvent *models type DeliverSpan struct { trace.Span - emeter emetrics.OutpostMetrics - deliveryEvent *models.DeliveryEvent - destination *models.Destination - err error + emeter emetrics.OutpostMetrics + task *models.DeliveryTask + destination *models.Destination + err error } func (d *DeliverSpan) RecordError(err error, options ...trace.EventOption) { @@ -72,17 +72,12 @@ func (d *DeliverSpan) RecordError(err error, options ...trace.EventOption) { } func (d *DeliverSpan) End(options ...trace.SpanEndOption) { - if d.deliveryEvent.Event.Telemetry == nil { - d.Span.End(options...) - return - } - if d.deliveryEvent.Delivery == nil { + if d.task.Event.Telemetry == nil { d.Span.End(options...) return } - ok := d.deliveryEvent.Delivery.Status == models.DeliveryStatusSuccess - startTime, err := time.Parse(time.RFC3339Nano, d.deliveryEvent.Event.Telemetry.ReceivedTime) + startTime, err := time.Parse(time.RFC3339Nano, d.task.Event.Telemetry.ReceivedTime) if err != nil { // TODO: handle error? d.Span.End(options...) @@ -92,14 +87,13 @@ func (d *DeliverSpan) End(options ...trace.SpanEndOption) { d.emeter.DeliveryLatency(context.Background(), time.Since(startTime), emetrics.DeliveryLatencyOpts{Type: d.destination.Type}) - d.emeter.EventDelivered(context.Background(), d.deliveryEvent, ok, d.destination.Type) d.Span.End(options...) } -func (t *eventTracerImpl) Deliver(_ context.Context, deliveryEvent *models.DeliveryEvent, destination *models.Destination) (context.Context, trace.Span) { - ctx, span := t.tracer.Start(t.getRemoteDeliveryEventSpanContext(deliveryEvent), "EventTracer.Deliver") - deliverySpan := &DeliverSpan{Span: span, emeter: t.emeter, deliveryEvent: deliveryEvent, destination: destination} +func (t *eventTracerImpl) Deliver(_ context.Context, task *models.DeliveryTask, destination *models.Destination) (context.Context, trace.Span) { + ctx, span := t.tracer.Start(t.getRemoteDeliveryTaskSpanContext(task), "EventTracer.Deliver") + deliverySpan := &DeliverSpan{Span: span, emeter: t.emeter, task: task, destination: destination} return ctx, deliverySpan } @@ -128,17 +122,17 @@ func (t *eventTracerImpl) getRemoteEventSpanContext(event *models.Event) context return trace.ContextWithRemoteSpanContext(context.Background(), remoteCtx) } -func (t *eventTracerImpl) getRemoteDeliveryEventSpanContext(deliveryEvent *models.DeliveryEvent) context.Context { - if deliveryEvent.Telemetry == nil { +func (t *eventTracerImpl) getRemoteDeliveryTaskSpanContext(task *models.DeliveryTask) context.Context { + if task.Telemetry == nil { return context.Background() } - traceID, err := trace.TraceIDFromHex(deliveryEvent.Telemetry.TraceID) + traceID, err := trace.TraceIDFromHex(task.Telemetry.TraceID) if err != nil { // TODO: handle error return context.Background() } - spanID, err := trace.SpanIDFromHex(deliveryEvent.Telemetry.SpanID) + spanID, err := trace.SpanIDFromHex(task.Telemetry.SpanID) if err != nil { // TODO: handle error return context.Background() diff --git a/internal/eventtracer/noop.go b/internal/eventtracer/noop.go index 67eeb691..ef051813 100644 --- a/internal/eventtracer/noop.go +++ b/internal/eventtracer/noop.go @@ -24,12 +24,12 @@ func (t *noopEventTracer) Receive(ctx context.Context, _ *models.Event) (context return ctx, span } -func (t *noopEventTracer) StartDelivery(ctx context.Context, deliveryEvent *models.DeliveryEvent) (context.Context, trace.Span) { +func (t *noopEventTracer) StartDelivery(ctx context.Context, task *models.DeliveryTask) (context.Context, trace.Span) { _, span := t.tracer.Start(ctx, "EventTracer.StartDelivery") return ctx, span } -func (t *noopEventTracer) Deliver(ctx context.Context, deliveryEvent *models.DeliveryEvent, destination *models.Destination) (context.Context, trace.Span) { +func (t *noopEventTracer) Deliver(ctx context.Context, task *models.DeliveryTask, destination *models.Destination) (context.Context, trace.Span) { _, span := t.tracer.Start(ctx, "EventTracer.Deliver") return ctx, span } diff --git a/internal/idempotence/idempotence_test.go b/internal/idempotence/idempotence_test.go index 9bc81c05..269bcb36 100644 --- a/internal/idempotence/idempotence_test.go +++ b/internal/idempotence/idempotence_test.go @@ -450,17 +450,6 @@ func TestIntegrationIdempotence_WithConcurrentHandlerAndSuccess(t *testing.T) { go consumerFn("1") go consumerFn("2") - errs := []error{} - go func() { - for { - select { - case err := <-errchan: - errs = append(errs, err) - case <-ctx.Done(): - return - } - } - }() id := idgen.String() err = mq.Publish(ctx, &MockMsg{ID: id}) @@ -468,7 +457,16 @@ func TestIntegrationIdempotence_WithConcurrentHandlerAndSuccess(t *testing.T) { err = mq.Publish(ctx, &MockMsg{ID: id}) require.Nil(t, err) - <-ctx.Done() + // Collect exactly 2 errors (one per published message) + errs := make([]error, 0, 2) + for i := 0; i < 2; i++ { + select { + case err := <-errchan: + errs = append(errs, err) + case <-ctx.Done(): + require.Fail(t, "timeout waiting for consumer results") + } + } assert.Len(t, execTimestamps, 1) require.Len(t, errs, 2, "should have 2 errors") diff --git a/internal/idgen/idgen.go b/internal/idgen/idgen.go index 8159610c..0d7e0ab8 100644 --- a/internal/idgen/idgen.go +++ b/internal/idgen/idgen.go @@ -14,11 +14,10 @@ var ( func init() { // Initialize with default UUID v4 generator globalGenerator = &IDGenerator{ - generator: &uuidv4Generator{}, - eventPrefix: "", - destinationPrefix: "", - deliveryPrefix: "", - deliveryEventPrefix: "", + generator: &uuidv4Generator{}, + eventPrefix: "", + destinationPrefix: "", + attemptPrefix: "", } } @@ -27,11 +26,10 @@ type idGenerator interface { } type IDGenerator struct { - generator idGenerator - eventPrefix string - destinationPrefix string - deliveryPrefix string - deliveryEventPrefix string + generator idGenerator + eventPrefix string + destinationPrefix string + attemptPrefix string } func (g *IDGenerator) Event() string { @@ -42,12 +40,8 @@ func (g *IDGenerator) Destination() string { return g.generate(g.destinationPrefix) } -func (g *IDGenerator) Delivery() string { - return g.generate(g.deliveryPrefix) -} - -func (g *IDGenerator) DeliveryEvent() string { - return g.generate(g.deliveryEventPrefix) +func (g *IDGenerator) Attempt() string { + return g.generate(g.attemptPrefix) } func (g *IDGenerator) Installation() string { @@ -68,7 +62,6 @@ func newIDGenerator(idType string) (idGenerator, error) { idType = "uuidv4" } - // Select the appropriate generator implementation switch idType { case "uuidv4": return &uuidv4Generator{}, nil @@ -113,11 +106,10 @@ func (g *nanoidGenerator) generate() string { } type IDGenConfig struct { - Type string - EventPrefix string - DestinationPrefix string - DeliveryPrefix string - DeliveryEventPrefix string + Type string + EventPrefix string + DestinationPrefix string + AttemptPrefix string } func Configure(cfg IDGenConfig) error { @@ -127,11 +119,10 @@ func Configure(cfg IDGenConfig) error { } globalGenerator = &IDGenerator{ - generator: gen, - eventPrefix: cfg.EventPrefix, - destinationPrefix: cfg.DestinationPrefix, - deliveryPrefix: cfg.DeliveryPrefix, - deliveryEventPrefix: cfg.DeliveryEventPrefix, + generator: gen, + eventPrefix: cfg.EventPrefix, + destinationPrefix: cfg.DestinationPrefix, + attemptPrefix: cfg.AttemptPrefix, } return nil @@ -145,12 +136,8 @@ func Destination() string { return globalGenerator.Destination() } -func Delivery() string { - return globalGenerator.Delivery() -} - -func DeliveryEvent() string { - return globalGenerator.DeliveryEvent() +func Attempt() string { + return globalGenerator.Attempt() } func Installation() string { diff --git a/internal/logmq/batchprocessor.go b/internal/logmq/batchprocessor.go new file mode 100644 index 00000000..cfefecaf --- /dev/null +++ b/internal/logmq/batchprocessor.go @@ -0,0 +1,125 @@ +package logmq + +import ( + "context" + "errors" + "time" + + "github.com/hookdeck/outpost/internal/logging" + "github.com/hookdeck/outpost/internal/models" + "github.com/hookdeck/outpost/internal/mqs" + "github.com/mikestefanello/batcher" + "go.uber.org/zap" +) + +// ErrInvalidLogEntry is returned when a LogEntry is missing required fields. +var ErrInvalidLogEntry = errors.New("invalid log entry: both event and attempt are required") + +// LogStore defines the interface for persisting log entries. +// This is a consumer-defined interface containing only what logmq needs. +type LogStore interface { + InsertMany(ctx context.Context, entries []*models.LogEntry) error +} + +// BatchProcessorConfig configures the batch processor. +type BatchProcessorConfig struct { + ItemCountThreshold int + DelayThreshold time.Duration +} + +// BatchProcessor batches log entries and writes them to the log store. +type BatchProcessor struct { + ctx context.Context + logger *logging.Logger + logStore LogStore + batcher *batcher.Batcher[*mqs.Message] +} + +// NewBatchProcessor creates a new batch processor for log entries. +func NewBatchProcessor(ctx context.Context, logger *logging.Logger, logStore LogStore, cfg BatchProcessorConfig) (*BatchProcessor, error) { + bp := &BatchProcessor{ + ctx: ctx, + logger: logger, + logStore: logStore, + } + + b, err := batcher.NewBatcher(batcher.Config[*mqs.Message]{ + GroupCountThreshold: 2, + ItemCountThreshold: cfg.ItemCountThreshold, + DelayThreshold: cfg.DelayThreshold, + NumGoroutines: 1, + Processor: bp.processBatch, + }) + if err != nil { + return nil, err + } + + bp.batcher = b + return bp, nil +} + +// Add adds a message to the batch. +func (bp *BatchProcessor) Add(ctx context.Context, msg *mqs.Message) error { + bp.batcher.Add("", msg) + return nil +} + +// Shutdown gracefully shuts down the batch processor. +func (bp *BatchProcessor) Shutdown() { + bp.batcher.Shutdown() +} + +// processBatch processes a batch of messages. +func (bp *BatchProcessor) processBatch(_ string, msgs []*mqs.Message) { + logger := bp.logger.Ctx(bp.ctx) + logger.Info("processing batch", zap.Int("message_count", len(msgs))) + + entries := make([]*models.LogEntry, 0, len(msgs)) + validMsgs := make([]*mqs.Message, 0, len(msgs)) + + for _, msg := range msgs { + entry := &models.LogEntry{} + if err := entry.FromMessage(msg); err != nil { + logger.Error("failed to parse log entry", + zap.Error(err), + zap.String("message_id", msg.LoggableID)) + msg.Nack() + continue + } + + // Validate that both Event and Attempt are present. + // The logstore requires both for data consistency. + if entry.Event == nil || entry.Attempt == nil { + logger.Error("invalid log entry: both event and attempt are required", + zap.Bool("has_event", entry.Event != nil), + zap.Bool("has_attempt", entry.Attempt != nil), + zap.String("message_id", msg.LoggableID)) + msg.Nack() + continue + } + + entries = append(entries, entry) + validMsgs = append(validMsgs, msg) + } + + // Nothing valid to insert + if len(entries) == 0 { + return + } + + if err := bp.logStore.InsertMany(bp.ctx, entries); err != nil { + logger.Error("failed to insert log entries", + zap.Error(err), + zap.Int("entry_count", len(entries))) + for _, msg := range validMsgs { + msg.Nack() + } + return + } + + logger.Info("batch processed successfully", zap.Int("count", len(validMsgs))) + + for _, msg := range validMsgs { + msg.Ack() + } +} diff --git a/internal/logmq/batchprocessor_test.go b/internal/logmq/batchprocessor_test.go new file mode 100644 index 00000000..74e8473d --- /dev/null +++ b/internal/logmq/batchprocessor_test.go @@ -0,0 +1,254 @@ +package logmq_test + +import ( + "context" + "encoding/json" + "sync" + "testing" + "time" + + "github.com/hookdeck/outpost/internal/logmq" + "github.com/hookdeck/outpost/internal/models" + "github.com/hookdeck/outpost/internal/mqs" + "github.com/hookdeck/outpost/internal/util/testutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type mockLogStore struct { + mu sync.Mutex + entries []*models.LogEntry + err error +} + +func (m *mockLogStore) InsertMany(ctx context.Context, entries []*models.LogEntry) error { + m.mu.Lock() + defer m.mu.Unlock() + if m.err != nil { + return m.err + } + m.entries = append(m.entries, entries...) + return nil +} + +func (m *mockLogStore) getInserted() (events []*models.Event, attempts []*models.Attempt) { + m.mu.Lock() + defer m.mu.Unlock() + for _, entry := range m.entries { + events = append(events, entry.Event) + attempts = append(attempts, entry.Attempt) + } + return events, attempts +} + +// mockQueueMessage implements mqs.QueueMessage for testing. +type mockQueueMessage struct { + acked bool + nacked bool +} + +func (m *mockQueueMessage) Ack() { m.acked = true } +func (m *mockQueueMessage) Nack() { m.nacked = true } + +func newMockMessage(entry models.LogEntry) (*mockQueueMessage, *mqs.Message) { + body, _ := json.Marshal(entry) + mock := &mockQueueMessage{} + msg := &mqs.Message{ + QueueMessage: mock, + Body: body, + LoggableID: "test-msg", + } + return mock, msg +} + +func newMockMessageFromBytes(body []byte) (*mockQueueMessage, *mqs.Message) { + mock := &mockQueueMessage{} + msg := &mqs.Message{ + QueueMessage: mock, + Body: body, + LoggableID: "test-msg", + } + return mock, msg +} + +func TestBatchProcessor_ValidEntry(t *testing.T) { + ctx := context.Background() + logger := testutil.CreateTestLogger(t) + logStore := &mockLogStore{} + + bp, err := logmq.NewBatchProcessor(ctx, logger, logStore, logmq.BatchProcessorConfig{ + ItemCountThreshold: 1, + DelayThreshold: 1 * time.Second, + }) + require.NoError(t, err) + defer bp.Shutdown() + + event := testutil.EventFactory.Any() + attempt := testutil.AttemptFactory.Any() + entry := models.LogEntry{ + Event: &event, + Attempt: &attempt, + } + + mock, msg := newMockMessage(entry) + err = bp.Add(ctx, msg) + require.NoError(t, err) + + // Wait for batch to process + time.Sleep(200 * time.Millisecond) + + assert.True(t, mock.acked, "valid message should be acked") + assert.False(t, mock.nacked, "valid message should not be nacked") + + events, attempts := logStore.getInserted() + assert.Len(t, events, 1) + assert.Len(t, attempts, 1) +} + +func TestBatchProcessor_InvalidEntry_MissingEvent(t *testing.T) { + ctx := context.Background() + logger := testutil.CreateTestLogger(t) + logStore := &mockLogStore{} + + bp, err := logmq.NewBatchProcessor(ctx, logger, logStore, logmq.BatchProcessorConfig{ + ItemCountThreshold: 1, + DelayThreshold: 1 * time.Second, + }) + require.NoError(t, err) + defer bp.Shutdown() + + attempt := testutil.AttemptFactory.Any() + entry := models.LogEntry{ + Event: nil, // Missing event + Attempt: &attempt, + } + + mock, msg := newMockMessage(entry) + err = bp.Add(ctx, msg) + require.NoError(t, err) + + // Wait for batch to process + time.Sleep(200 * time.Millisecond) + + assert.False(t, mock.acked, "invalid message should not be acked") + assert.True(t, mock.nacked, "invalid message should be nacked") + + events, attempts := logStore.getInserted() + assert.Empty(t, events, "no events should be inserted for invalid entry") + assert.Empty(t, attempts, "no attempts should be inserted for invalid entry") +} + +func TestBatchProcessor_InvalidEntry_MissingAttempt(t *testing.T) { + ctx := context.Background() + logger := testutil.CreateTestLogger(t) + logStore := &mockLogStore{} + + bp, err := logmq.NewBatchProcessor(ctx, logger, logStore, logmq.BatchProcessorConfig{ + ItemCountThreshold: 1, + DelayThreshold: 1 * time.Second, + }) + require.NoError(t, err) + defer bp.Shutdown() + + event := testutil.EventFactory.Any() + entry := models.LogEntry{ + Event: &event, + Attempt: nil, // Missing attempt + } + + mock, msg := newMockMessage(entry) + err = bp.Add(ctx, msg) + require.NoError(t, err) + + // Wait for batch to process + time.Sleep(200 * time.Millisecond) + + assert.False(t, mock.acked, "invalid message should not be acked") + assert.True(t, mock.nacked, "invalid message should be nacked") + + events, attempts := logStore.getInserted() + assert.Empty(t, events, "no events should be inserted for invalid entry") + assert.Empty(t, attempts, "no attempts should be inserted for invalid entry") +} + +func TestBatchProcessor_InvalidEntry_DoesNotBlockBatch(t *testing.T) { + ctx := context.Background() + logger := testutil.CreateTestLogger(t) + logStore := &mockLogStore{} + + bp, err := logmq.NewBatchProcessor(ctx, logger, logStore, logmq.BatchProcessorConfig{ + ItemCountThreshold: 3, // Wait for 3 messages before processing + DelayThreshold: 1 * time.Second, + }) + require.NoError(t, err) + defer bp.Shutdown() + + // Create valid entry 1 + event1 := testutil.EventFactory.Any() + attempt1 := testutil.AttemptFactory.Any() + validEntry1 := models.LogEntry{Event: &event1, Attempt: &attempt1} + mock1, msg1 := newMockMessage(validEntry1) + + // Create invalid entry (missing event) + attempt2 := testutil.AttemptFactory.Any() + invalidEntry := models.LogEntry{Event: nil, Attempt: &attempt2} + mock2, msg2 := newMockMessage(invalidEntry) + + // Create valid entry 2 + event3 := testutil.EventFactory.Any() + attempt3 := testutil.AttemptFactory.Any() + validEntry2 := models.LogEntry{Event: &event3, Attempt: &attempt3} + mock3, msg3 := newMockMessage(validEntry2) + + // Add all messages + require.NoError(t, bp.Add(ctx, msg1)) + require.NoError(t, bp.Add(ctx, msg2)) + require.NoError(t, bp.Add(ctx, msg3)) + + // Wait for batch to process + time.Sleep(200 * time.Millisecond) + + // Valid messages should be acked + assert.True(t, mock1.acked, "valid message 1 should be acked") + assert.False(t, mock1.nacked, "valid message 1 should not be nacked") + + // Invalid message should be nacked + assert.False(t, mock2.acked, "invalid message should not be acked") + assert.True(t, mock2.nacked, "invalid message should be nacked") + + // Valid message 2 should be acked (not blocked by invalid message) + assert.True(t, mock3.acked, "valid message 2 should be acked") + assert.False(t, mock3.nacked, "valid message 2 should not be nacked") + + // Only valid entries should be inserted + events, attempts := logStore.getInserted() + assert.Len(t, events, 2, "only 2 valid events should be inserted") + assert.Len(t, attempts, 2, "only 2 valid attempts should be inserted") +} + +func TestBatchProcessor_MalformedJSON(t *testing.T) { + ctx := context.Background() + logger := testutil.CreateTestLogger(t) + logStore := &mockLogStore{} + + bp, err := logmq.NewBatchProcessor(ctx, logger, logStore, logmq.BatchProcessorConfig{ + ItemCountThreshold: 1, + DelayThreshold: 1 * time.Second, + }) + require.NoError(t, err) + defer bp.Shutdown() + + mock, msg := newMockMessageFromBytes([]byte("not valid json")) + err = bp.Add(ctx, msg) + require.NoError(t, err) + + // Wait for batch to process + time.Sleep(200 * time.Millisecond) + + assert.False(t, mock.acked, "malformed message should not be acked") + assert.True(t, mock.nacked, "malformed message should be nacked") + + events, attempts := logStore.getInserted() + assert.Empty(t, events) + assert.Empty(t, attempts) +} diff --git a/internal/logmq/logmq.go b/internal/logmq/logmq.go index 395824b2..540873a5 100644 --- a/internal/logmq/logmq.go +++ b/internal/logmq/logmq.go @@ -44,8 +44,8 @@ func (q *LogMQ) Init(ctx context.Context) (func(), error) { return q.queue.Init(ctx) } -func (q *LogMQ) Publish(ctx context.Context, event models.DeliveryEvent) error { - return q.queue.Publish(ctx, &event) +func (q *LogMQ) Publish(ctx context.Context, entry models.LogEntry) error { + return q.queue.Publish(ctx, &entry) } func (q *LogMQ) Subscribe(ctx context.Context) (mqs.Subscription, error) { diff --git a/internal/logmq/messagehandler.go b/internal/logmq/messagehandler.go index 6bee0e4f..024e71b0 100644 --- a/internal/logmq/messagehandler.go +++ b/internal/logmq/messagehandler.go @@ -8,27 +8,28 @@ import ( "github.com/hookdeck/outpost/internal/mqs" ) -type batcher interface { +// BatchAdder is the interface for adding messages to a batch processor. +type BatchAdder interface { Add(ctx context.Context, msg *mqs.Message) error } type messageHandler struct { - logger *logging.Logger - batcher batcher + logger *logging.Logger + batchAdder BatchAdder } var _ consumer.MessageHandler = (*messageHandler)(nil) -func NewMessageHandler(logger *logging.Logger, batcher batcher) consumer.MessageHandler { +func NewMessageHandler(logger *logging.Logger, batchAdder BatchAdder) consumer.MessageHandler { return &messageHandler{ - logger: logger, - batcher: batcher, + logger: logger, + batchAdder: batchAdder, } } func (h *messageHandler) Handle(ctx context.Context, msg *mqs.Message) error { logger := h.logger.Ctx(ctx) logger.Info("logmq handler") - h.batcher.Add(ctx, msg) + h.batchAdder.Add(ctx, msg) return nil } diff --git a/internal/logstore/chlogstore/README.md b/internal/logstore/chlogstore/README.md index 360452ce..8c89605a 100644 --- a/internal/logstore/chlogstore/README.md +++ b/internal/logstore/chlogstore/README.md @@ -37,7 +37,7 @@ For most use cases (log viewing), brief duplicates are acceptable. ## Operations -### ListDeliveryEvent +### ListDelivery Direct index scan with cursor-based pagination. @@ -45,7 +45,7 @@ Direct index scan with cursor-based pagination. SELECT event_id, tenant_id, destination_id, topic, eligible_for_retry, event_time, metadata, data, - delivery_id, delivery_event_id, status, delivery_time, code, response_data + delivery_id, status, delivery_time, code, response_data FROM event_log WHERE tenant_id = ? AND [optional filters: destination_id, status, topic, time ranges] @@ -80,7 +80,7 @@ LIMIT 1 With destination filter, adds `AND destination_id = ?`. -### InsertManyDeliveryEvent +### InsertMany Batch insert using ClickHouse's native batch API. @@ -89,11 +89,11 @@ batch, _ := conn.PrepareBatch(ctx, ` INSERT INTO event_log ( event_id, tenant_id, destination_id, topic, eligible_for_retry, event_time, metadata, data, - delivery_id, delivery_event_id, status, delivery_time, code, response_data + delivery_id, status, delivery_time, code, response_data ) `) -for _, de := range deliveryEvents { - batch.Append(...) +for i := range events { + batch.Append(events[i], deliveries[i], ...) } batch.Send() ``` @@ -104,6 +104,6 @@ batch.Send() | Operation | Complexity | Notes | |-----------|------------|-------| -| ListDeliveryEvent | O(limit) | Index scan, stops at LIMIT | +| ListDelivery | O(limit) | Index scan, stops at LIMIT | | RetrieveEvent | O(1) | Single row lookup via bloom filter | -| InsertManyDeliveryEvent | O(batch) | Batch insert, async dedup | +| InsertMany | O(batch) | Batch insert, async dedup | diff --git a/internal/logstore/chlogstore/chlogstore.go b/internal/logstore/chlogstore/chlogstore.go index 1bcdce8e..eadbbd4f 100644 --- a/internal/logstore/chlogstore/chlogstore.go +++ b/internal/logstore/chlogstore/chlogstore.go @@ -16,15 +16,15 @@ import ( ) const ( - cursorResourceEvent = "evt" - cursorResourceDelivery = "dlv" - cursorVersion = 1 + cursorResourceEvent = "evt" + cursorResourceAttempt = "att" + cursorVersion = 1 ) type logStoreImpl struct { - chDB clickhouse.DB - eventsTable string - deliveriesTable string + chDB clickhouse.DB + eventsTable string + attemptsTable string } var _ driver.LogStore = (*logStoreImpl)(nil) @@ -35,9 +35,9 @@ func NewLogStore(chDB clickhouse.DB, deploymentID string) driver.LogStore { prefix = deploymentID + "_" } return &logStoreImpl{ - chDB: chDB, - eventsTable: prefix + "events", - deliveriesTable: prefix + "deliveries", + chDB: chDB, + eventsTable: prefix + "events", + attemptsTable: prefix + "attempts", } } @@ -99,9 +99,9 @@ func (s *logStoreImpl) ListEvent(ctx context.Context, req driver.ListEventReques }, nil } -func buildEventQuery(table string, req driver.ListEventRequest, q pagination.QueryInput) (string, []interface{}) { +func buildEventQuery(table string, req driver.ListEventRequest, q pagination.QueryInput) (string, []any) { var conditions []string - var args []interface{} + var args []any if req.TenantID != "" { conditions = append(conditions, "tenant_id = ?") @@ -201,7 +201,7 @@ func scanEvents(rows clickhouse.Rows) ([]eventWithPosition, error) { } var metadata map[string]string - var data map[string]interface{} + var data map[string]any if metadataStr != "" { if err := json.Unmarshal([]byte(metadataStr), &metadata); err != nil { @@ -236,7 +236,7 @@ func scanEvents(rows clickhouse.Rows) ([]eventWithPosition, error) { return results, nil } -func buildEventCursorCondition(compare, position string) (string, []interface{}) { +func buildEventCursorCondition(compare, position string) (string, []any) { parts := strings.SplitN(position, "::", 2) if len(parts) != 2 { return "1=1", nil // invalid cursor, return always true @@ -252,16 +252,16 @@ func buildEventCursorCondition(compare, position string) (string, []interface{}) OR (event_time = fromUnixTimestamp64Milli(?) AND event_id %s ?) )`, compare, compare) - return condition, []interface{}{eventTimeMs, eventTimeMs, eventID} + return condition, []any{eventTimeMs, eventTimeMs, eventID} } -// deliveryEventWithPosition wraps a delivery event with its cursor position data. -type deliveryEventWithPosition struct { - *models.DeliveryEvent - deliveryTime time.Time +// attemptRecordWithPosition wraps an attempt record with its cursor position data. +type attemptRecordWithPosition struct { + *driver.AttemptRecord + attemptTime time.Time } -func (s *logStoreImpl) ListDeliveryEvent(ctx context.Context, req driver.ListDeliveryEventRequest) (driver.ListDeliveryEventResponse, error) { +func (s *logStoreImpl) ListAttempt(ctx context.Context, req driver.ListAttemptRequest) (driver.ListAttemptResponse, error) { sortOrder := req.SortOrder if sortOrder != "asc" && sortOrder != "desc" { sortOrder = "desc" @@ -272,50 +272,50 @@ func (s *logStoreImpl) ListDeliveryEvent(ctx context.Context, req driver.ListDel limit = 100 } - res, err := pagination.Run(ctx, pagination.Config[deliveryEventWithPosition]{ + res, err := pagination.Run(ctx, pagination.Config[attemptRecordWithPosition]{ Limit: limit, Order: sortOrder, Next: req.Next, Prev: req.Prev, - Fetch: func(ctx context.Context, q pagination.QueryInput) ([]deliveryEventWithPosition, error) { - query, args := buildDeliveryQuery(s.deliveriesTable, req, q) + Fetch: func(ctx context.Context, q pagination.QueryInput) ([]attemptRecordWithPosition, error) { + query, args := buildAttemptQuery(s.attemptsTable, req, q) rows, err := s.chDB.Query(ctx, query, args...) if err != nil { return nil, fmt.Errorf("query failed: %w", err) } defer rows.Close() - return scanDeliveryEvents(rows) + return scanAttemptRecords(rows) }, - Cursor: pagination.Cursor[deliveryEventWithPosition]{ - Encode: func(de deliveryEventWithPosition) string { - position := fmt.Sprintf("%d::%s", de.deliveryTime.UnixMilli(), de.Delivery.ID) - return cursor.Encode(cursorResourceDelivery, cursorVersion, position) + Cursor: pagination.Cursor[attemptRecordWithPosition]{ + Encode: func(ar attemptRecordWithPosition) string { + position := fmt.Sprintf("%d::%s", ar.attemptTime.UnixMilli(), ar.Attempt.ID) + return cursor.Encode(cursorResourceAttempt, cursorVersion, position) }, Decode: func(c string) (string, error) { - return cursor.Decode(c, cursorResourceDelivery, cursorVersion) + return cursor.Decode(c, cursorResourceAttempt, cursorVersion) }, }, }) if err != nil { - return driver.ListDeliveryEventResponse{}, err + return driver.ListAttemptResponse{}, err } - // Extract delivery events from results - data := make([]*models.DeliveryEvent, len(res.Items)) + // Extract attempt records from results + data := make([]*driver.AttemptRecord, len(res.Items)) for i, item := range res.Items { - data[i] = item.DeliveryEvent + data[i] = item.AttemptRecord } - return driver.ListDeliveryEventResponse{ + return driver.ListAttemptResponse{ Data: data, Next: res.Next, Prev: res.Prev, }, nil } -func buildDeliveryQuery(table string, req driver.ListDeliveryEventRequest, q pagination.QueryInput) (string, []interface{}) { +func buildAttemptQuery(table string, req driver.ListAttemptRequest, q pagination.QueryInput) (string, []any) { var conditions []string - var args []interface{} + var args []any if req.TenantID != "" { conditions = append(conditions, "tenant_id = ?") @@ -343,24 +343,24 @@ func buildDeliveryQuery(table string, req driver.ListDeliveryEventRequest, q pag } if req.TimeFilter.GTE != nil { - conditions = append(conditions, "delivery_time >= ?") + conditions = append(conditions, "attempt_time >= ?") args = append(args, *req.TimeFilter.GTE) } if req.TimeFilter.LTE != nil { - conditions = append(conditions, "delivery_time <= ?") + conditions = append(conditions, "attempt_time <= ?") args = append(args, *req.TimeFilter.LTE) } if req.TimeFilter.GT != nil { - conditions = append(conditions, "delivery_time > ?") + conditions = append(conditions, "attempt_time > ?") args = append(args, *req.TimeFilter.GT) } if req.TimeFilter.LT != nil { - conditions = append(conditions, "delivery_time < ?") + conditions = append(conditions, "attempt_time < ?") args = append(args, *req.TimeFilter.LT) } if q.CursorPos != "" { - cursorCond, cursorArgs := buildDeliveryCursorCondition(q.Compare, q.CursorPos) + cursorCond, cursorArgs := buildAttemptCursorCondition(q.Compare, q.CursorPos) conditions = append(conditions, cursorCond) args = append(args, cursorArgs...) } @@ -370,7 +370,7 @@ func buildDeliveryQuery(table string, req driver.ListDeliveryEventRequest, q pag whereClause = "1=1" } - orderByClause := fmt.Sprintf("ORDER BY delivery_time %s, delivery_id %s", + orderByClause := fmt.Sprintf("ORDER BY attempt_time %s, attempt_id %s", strings.ToUpper(q.SortDir), strings.ToUpper(q.SortDir)) query := fmt.Sprintf(` @@ -383,14 +383,13 @@ func buildDeliveryQuery(table string, req driver.ListDeliveryEventRequest, q pag event_time, metadata, data, - delivery_id, - delivery_event_id, + attempt_id, status, - delivery_time, + attempt_time, code, response_data, manual, - attempt + attempt_number FROM %s WHERE %s %s @@ -400,8 +399,8 @@ func buildDeliveryQuery(table string, req driver.ListDeliveryEventRequest, q pag return query, args } -func scanDeliveryEvents(rows clickhouse.Rows) ([]deliveryEventWithPosition, error) { - var results []deliveryEventWithPosition +func scanAttemptRecords(rows clickhouse.Rows) ([]attemptRecordWithPosition, error) { + var results []attemptRecordWithPosition for rows.Next() { var ( eventID string @@ -412,14 +411,13 @@ func scanDeliveryEvents(rows clickhouse.Rows) ([]deliveryEventWithPosition, erro eventTime time.Time metadataStr string dataStr string - deliveryID string - deliveryEventID string + attemptID string status string - deliveryTime time.Time + attemptTime time.Time code string responseDataStr string manual bool - attempt uint32 + attemptNumber uint32 ) err := rows.Scan( @@ -431,22 +429,21 @@ func scanDeliveryEvents(rows clickhouse.Rows) ([]deliveryEventWithPosition, erro &eventTime, &metadataStr, &dataStr, - &deliveryID, - &deliveryEventID, + &attemptID, &status, - &deliveryTime, + &attemptTime, &code, &responseDataStr, &manual, - &attempt, + &attemptNumber, ) if err != nil { return nil, fmt.Errorf("scan failed: %w", err) } var metadata map[string]string - var data map[string]interface{} - var responseData map[string]interface{} + var data map[string]any + var responseData map[string]any if metadataStr != "" { if err := json.Unmarshal([]byte(metadataStr), &metadata); err != nil { @@ -464,13 +461,21 @@ func scanDeliveryEvents(rows clickhouse.Rows) ([]deliveryEventWithPosition, erro } } - results = append(results, deliveryEventWithPosition{ - DeliveryEvent: &models.DeliveryEvent{ - ID: deliveryEventID, - DestinationID: destinationID, - Manual: manual, - Attempt: int(attempt), - Event: models.Event{ + results = append(results, attemptRecordWithPosition{ + AttemptRecord: &driver.AttemptRecord{ + Attempt: &models.Attempt{ + ID: attemptID, + TenantID: tenantID, + EventID: eventID, + DestinationID: destinationID, + AttemptNumber: int(attemptNumber), + Manual: manual, + Status: status, + Time: attemptTime, + Code: code, + ResponseData: responseData, + }, + Event: &models.Event{ ID: eventID, TenantID: tenantID, DestinationID: destinationID, @@ -480,17 +485,8 @@ func scanDeliveryEvents(rows clickhouse.Rows) ([]deliveryEventWithPosition, erro Data: data, Metadata: metadata, }, - Delivery: &models.Delivery{ - ID: deliveryID, - EventID: eventID, - DestinationID: destinationID, - Status: status, - Time: deliveryTime, - Code: code, - ResponseData: responseData, - }, }, - deliveryTime: deliveryTime, + attemptTime: attemptTime, }) } @@ -503,7 +499,7 @@ func scanDeliveryEvents(rows clickhouse.Rows) ([]deliveryEventWithPosition, erro func (s *logStoreImpl) RetrieveEvent(ctx context.Context, req driver.RetrieveEventRequest) (*models.Event, error) { var conditions []string - var args []interface{} + var args []any if req.TenantID != "" { conditions = append(conditions, "tenant_id = ?") @@ -532,7 +528,7 @@ func (s *logStoreImpl) RetrieveEvent(ctx context.Context, req driver.RetrieveEve data FROM %s WHERE %s - LIMIT 1`, s.deliveriesTable, whereClause) + LIMIT 1`, s.attemptsTable, whereClause) rows, err := s.chDB.Query(ctx, query, args...) if err != nil { @@ -574,17 +570,17 @@ func (s *logStoreImpl) RetrieveEvent(ctx context.Context, req driver.RetrieveEve return event, nil } -func (s *logStoreImpl) RetrieveDeliveryEvent(ctx context.Context, req driver.RetrieveDeliveryEventRequest) (*models.DeliveryEvent, error) { +func (s *logStoreImpl) RetrieveAttempt(ctx context.Context, req driver.RetrieveAttemptRequest) (*driver.AttemptRecord, error) { var conditions []string - var args []interface{} + var args []any if req.TenantID != "" { conditions = append(conditions, "tenant_id = ?") args = append(args, req.TenantID) } - conditions = append(conditions, "delivery_id = ?") - args = append(args, req.DeliveryID) + conditions = append(conditions, "attempt_id = ?") + args = append(args, req.AttemptID) whereClause := strings.Join(conditions, " AND ") @@ -598,17 +594,16 @@ func (s *logStoreImpl) RetrieveDeliveryEvent(ctx context.Context, req driver.Ret event_time, metadata, data, - delivery_id, - delivery_event_id, + attempt_id, status, - delivery_time, + attempt_time, code, response_data, manual, - attempt + attempt_number FROM %s WHERE %s - LIMIT 1`, s.deliveriesTable, whereClause) + LIMIT 1`, s.attemptsTable, whereClause) rows, err := s.chDB.Query(ctx, query, args...) if err != nil { @@ -629,14 +624,13 @@ func (s *logStoreImpl) RetrieveDeliveryEvent(ctx context.Context, req driver.Ret eventTime time.Time metadataStr string dataStr string - deliveryID string - deliveryEventID string + attemptID string status string - deliveryTime time.Time + attemptTime time.Time code string responseDataStr string manual bool - attempt uint32 + attemptNumber uint32 ) err = rows.Scan( @@ -648,22 +642,21 @@ func (s *logStoreImpl) RetrieveDeliveryEvent(ctx context.Context, req driver.Ret &eventTime, &metadataStr, &dataStr, - &deliveryID, - &deliveryEventID, + &attemptID, &status, - &deliveryTime, + &attemptTime, &code, &responseDataStr, &manual, - &attempt, + &attemptNumber, ) if err != nil { return nil, fmt.Errorf("scan failed: %w", err) } var metadata map[string]string - var data map[string]interface{} - var responseData map[string]interface{} + var data map[string]any + var responseData map[string]any if metadataStr != "" { if err := json.Unmarshal([]byte(metadataStr), &metadata); err != nil { @@ -681,12 +674,20 @@ func (s *logStoreImpl) RetrieveDeliveryEvent(ctx context.Context, req driver.Ret } } - return &models.DeliveryEvent{ - ID: deliveryEventID, - DestinationID: destinationID, - Manual: manual, - Attempt: int(attempt), - Event: models.Event{ + return &driver.AttemptRecord{ + Attempt: &models.Attempt{ + ID: attemptID, + TenantID: tenantID, + EventID: eventID, + DestinationID: destinationID, + AttemptNumber: int(attemptNumber), + Manual: manual, + Status: status, + Time: attemptTime, + Code: code, + ResponseData: responseData, + }, + Event: &models.Event{ ID: eventID, TenantID: tenantID, DestinationID: destinationID, @@ -696,125 +697,110 @@ func (s *logStoreImpl) RetrieveDeliveryEvent(ctx context.Context, req driver.Ret Data: data, Metadata: metadata, }, - Delivery: &models.Delivery{ - ID: deliveryID, - EventID: eventID, - DestinationID: destinationID, - Status: status, - Time: deliveryTime, - Code: code, - ResponseData: responseData, - }, }, nil } -func (s *logStoreImpl) InsertManyDeliveryEvent(ctx context.Context, deliveryEvents []*models.DeliveryEvent) error { - if len(deliveryEvents) == 0 { +func (s *logStoreImpl) InsertMany(ctx context.Context, entries []*models.LogEntry) error { + if len(entries) == 0 { return nil } - eventBatch, err := s.chDB.PrepareBatch(ctx, - fmt.Sprintf(`INSERT INTO %s ( - event_id, tenant_id, destination_id, topic, eligible_for_retry, event_time, metadata, data - )`, s.eventsTable), - ) - if err != nil { - return fmt.Errorf("prepare events batch failed: %w", err) + // Extract and dedupe events by ID + eventMap := make(map[string]*models.Event) + for _, entry := range entries { + eventMap[entry.Event.ID] = entry.Event } - for _, de := range deliveryEvents { - metadataJSON, err := json.Marshal(de.Event.Metadata) - if err != nil { - return fmt.Errorf("failed to marshal metadata: %w", err) - } - dataJSON, err := json.Marshal(de.Event.Data) + if len(eventMap) > 0 { + eventBatch, err := s.chDB.PrepareBatch(ctx, + fmt.Sprintf(`INSERT INTO %s ( + event_id, tenant_id, destination_id, topic, eligible_for_retry, event_time, metadata, data + )`, s.eventsTable), + ) if err != nil { - return fmt.Errorf("failed to marshal data: %w", err) + return fmt.Errorf("prepare events batch failed: %w", err) } - if err := eventBatch.Append( - de.Event.ID, - de.Event.TenantID, - de.DestinationID, - de.Event.Topic, - de.Event.EligibleForRetry, - de.Event.Time, - string(metadataJSON), - string(dataJSON), - ); err != nil { - return fmt.Errorf("events batch append failed: %w", err) + for _, e := range eventMap { + metadataJSON, err := json.Marshal(e.Metadata) + if err != nil { + return fmt.Errorf("failed to marshal metadata: %w", err) + } + dataJSON, err := json.Marshal(e.Data) + if err != nil { + return fmt.Errorf("failed to marshal data: %w", err) + } + + if err := eventBatch.Append( + e.ID, + e.TenantID, + e.DestinationID, + e.Topic, + e.EligibleForRetry, + e.Time, + string(metadataJSON), + string(dataJSON), + ); err != nil { + return fmt.Errorf("events batch append failed: %w", err) + } } - } - if err := eventBatch.Send(); err != nil { - return fmt.Errorf("events batch send failed: %w", err) + if err := eventBatch.Send(); err != nil { + return fmt.Errorf("events batch send failed: %w", err) + } } - deliveryBatch, err := s.chDB.PrepareBatch(ctx, + // Insert attempts with their paired event data + attemptBatch, err := s.chDB.PrepareBatch(ctx, fmt.Sprintf(`INSERT INTO %s ( event_id, tenant_id, destination_id, topic, eligible_for_retry, event_time, metadata, data, - delivery_id, delivery_event_id, status, delivery_time, code, response_data, manual, attempt - )`, s.deliveriesTable), + attempt_id, status, attempt_time, code, response_data, manual, attempt_number + )`, s.attemptsTable), ) if err != nil { - return fmt.Errorf("prepare deliveries batch failed: %w", err) + return fmt.Errorf("prepare attempts batch failed: %w", err) } - for _, de := range deliveryEvents { - metadataJSON, err := json.Marshal(de.Event.Metadata) + for _, entry := range entries { + event := entry.Event + a := entry.Attempt + + metadataJSON, err := json.Marshal(event.Metadata) if err != nil { return fmt.Errorf("failed to marshal metadata: %w", err) } - dataJSON, err := json.Marshal(de.Event.Data) + dataJSON, err := json.Marshal(event.Data) if err != nil { return fmt.Errorf("failed to marshal data: %w", err) } - - var deliveryID, status, code string - var deliveryTime time.Time - var responseDataJSON []byte - - if de.Delivery != nil { - deliveryID = de.Delivery.ID - status = de.Delivery.Status - deliveryTime = de.Delivery.Time - code = de.Delivery.Code - responseDataJSON, err = json.Marshal(de.Delivery.ResponseData) - if err != nil { - return fmt.Errorf("failed to marshal response_data: %w", err) - } - } else { - deliveryID = de.ID - status = "pending" - deliveryTime = de.Event.Time - code = "" - responseDataJSON = []byte("{}") + responseDataJSON, err := json.Marshal(a.ResponseData) + if err != nil { + return fmt.Errorf("failed to marshal response_data: %w", err) } - if err := deliveryBatch.Append( - de.Event.ID, - de.Event.TenantID, - de.DestinationID, - de.Event.Topic, - de.Event.EligibleForRetry, - de.Event.Time, + if err := attemptBatch.Append( + a.EventID, + event.TenantID, + a.DestinationID, + event.Topic, + event.EligibleForRetry, + event.Time, string(metadataJSON), string(dataJSON), - deliveryID, - de.ID, - status, - deliveryTime, - code, + a.ID, + a.Status, + a.Time, + a.Code, string(responseDataJSON), - de.Manual, - uint32(de.Attempt), + a.Manual, + uint32(a.AttemptNumber), ); err != nil { - return fmt.Errorf("deliveries batch append failed: %w", err) + return fmt.Errorf("attempts batch append failed: %w", err) } } - if err := deliveryBatch.Send(); err != nil { - return fmt.Errorf("deliveries batch send failed: %w", err) + if err := attemptBatch.Send(); err != nil { + return fmt.Errorf("attempts batch send failed: %w", err) } return nil @@ -824,21 +810,21 @@ func parseTimestampMs(s string) (int64, error) { return strconv.ParseInt(s, 10, 64) } -func buildDeliveryCursorCondition(compare, position string) (string, []interface{}) { +func buildAttemptCursorCondition(compare, position string) (string, []any) { parts := strings.SplitN(position, "::", 2) if len(parts) != 2 { return "1=1", nil } - deliveryTimeMs, err := parseTimestampMs(parts[0]) + attemptTimeMs, err := parseTimestampMs(parts[0]) if err != nil { return "1=1", nil // invalid timestamp, return always true } - deliveryID := parts[1] + attemptID := parts[1] condition := fmt.Sprintf(`( - delivery_time %s fromUnixTimestamp64Milli(?) - OR (delivery_time = fromUnixTimestamp64Milli(?) AND delivery_id %s ?) + attempt_time %s fromUnixTimestamp64Milli(?) + OR (attempt_time = fromUnixTimestamp64Milli(?) AND attempt_id %s ?) )`, compare, compare) - return condition, []interface{}{deliveryTimeMs, deliveryTimeMs, deliveryID} + return condition, []any{attemptTimeMs, attemptTimeMs, attemptID} } diff --git a/internal/logstore/chlogstore/chlogstore_test.go b/internal/logstore/chlogstore/chlogstore_test.go index 4eaad64d..827c35c1 100644 --- a/internal/logstore/chlogstore/chlogstore_test.go +++ b/internal/logstore/chlogstore/chlogstore_test.go @@ -77,15 +77,15 @@ func (h *harness) Close() { func (h *harness) FlushWrites(ctx context.Context) error { // Force ClickHouse to merge parts and deduplicate rows on both tables eventsTable := "events" - deliveriesTable := "deliveries" + attemptsTable := "attempts" if h.deploymentID != "" { eventsTable = h.deploymentID + "_events" - deliveriesTable = h.deploymentID + "_deliveries" + attemptsTable = h.deploymentID + "_attempts" } if err := h.chDB.Exec(ctx, "OPTIMIZE TABLE "+eventsTable+" FINAL"); err != nil { return err } - return h.chDB.Exec(ctx, "OPTIMIZE TABLE "+deliveriesTable+" FINAL") + return h.chDB.Exec(ctx, "OPTIMIZE TABLE "+attemptsTable+" FINAL") } func (h *harness) MakeDriver(ctx context.Context) (driver.LogStore, error) { diff --git a/internal/logstore/driver/driver.go b/internal/logstore/driver/driver.go index fe7cf872..ab86fa16 100644 --- a/internal/logstore/driver/driver.go +++ b/internal/logstore/driver/driver.go @@ -18,10 +18,10 @@ type TimeFilter struct { type LogStore interface { ListEvent(context.Context, ListEventRequest) (ListEventResponse, error) - ListDeliveryEvent(context.Context, ListDeliveryEventRequest) (ListDeliveryEventResponse, error) + ListAttempt(context.Context, ListAttemptRequest) (ListAttemptResponse, error) RetrieveEvent(ctx context.Context, request RetrieveEventRequest) (*models.Event, error) - RetrieveDeliveryEvent(ctx context.Context, request RetrieveDeliveryEventRequest) (*models.DeliveryEvent, error) - InsertManyDeliveryEvent(context.Context, []*models.DeliveryEvent) error + RetrieveAttempt(ctx context.Context, request RetrieveAttemptRequest) (*AttemptRecord, error) + InsertMany(context.Context, []*models.LogEntry) error } type ListEventRequest struct { @@ -41,11 +41,11 @@ type ListEventResponse struct { Prev string } -type ListDeliveryEventRequest struct { +type ListAttemptRequest struct { Next string Prev string Limit int - TimeFilter TimeFilter // optional - filter deliveries by time + TimeFilter TimeFilter // optional - filter attempts by time TenantID string // optional - filter by tenant (if empty, returns all tenants) EventID string // optional - filter for specific event DestinationIDs []string // optional @@ -54,8 +54,8 @@ type ListDeliveryEventRequest struct { SortOrder string // optional: "asc", "desc" (default: "desc") } -type ListDeliveryEventResponse struct { - Data []*models.DeliveryEvent +type ListAttemptResponse struct { + Data []*AttemptRecord Next string Prev string } @@ -66,7 +66,13 @@ type RetrieveEventRequest struct { DestinationID string // optional - if provided, scopes to that destination } -type RetrieveDeliveryEventRequest struct { - TenantID string // optional - filter by tenant (if empty, searches all tenants) - DeliveryID string // required +type RetrieveAttemptRequest struct { + TenantID string // optional - filter by tenant (if empty, searches all tenants) + AttemptID string // required +} + +// AttemptRecord represents an attempt query result with optional Event population. +type AttemptRecord struct { + Attempt *models.Attempt + Event *models.Event // optionally populated for query results } diff --git a/internal/logstore/drivertest/crud.go b/internal/logstore/drivertest/crud.go index 45083c55..675b42aa 100644 --- a/internal/logstore/drivertest/crud.go +++ b/internal/logstore/drivertest/crud.go @@ -33,13 +33,13 @@ func testCRUD(t *testing.T, newHarness HarnessMaker) { startTime := baseTime.Add(-48 * time.Hour) // We'll populate these as we insert - var allDeliveryEvents []*models.DeliveryEvent + var allDeliveries []*models.Attempt destinationEvents := map[string][]*models.Event{} topicEvents := map[string][]*models.Event{} - statusDeliveryEvents := map[string][]*models.DeliveryEvent{} + statusDeliveries := map[string][]*models.Attempt{} t.Run("insert and verify", func(t *testing.T) { - t.Run("single delivery event", func(t *testing.T) { + t.Run("single delivery", func(t *testing.T) { destID := destinationIDs[0] topic := testutil.TestTopics[0] event := testutil.EventFactory.AnyPointer( @@ -49,31 +49,26 @@ func testCRUD(t *testing.T, newHarness HarnessMaker) { testutil.EventFactory.WithTopic(topic), testutil.EventFactory.WithTime(baseTime.Add(-30*time.Minute)), ) - delivery := testutil.DeliveryFactory.AnyPointer( - testutil.DeliveryFactory.WithID("single_del"), - testutil.DeliveryFactory.WithEventID(event.ID), - testutil.DeliveryFactory.WithDestinationID(destID), - testutil.DeliveryFactory.WithStatus("success"), - testutil.DeliveryFactory.WithTime(baseTime.Add(-30*time.Minute)), + delivery := testutil.AttemptFactory.AnyPointer( + testutil.AttemptFactory.WithID("single_del"), + testutil.AttemptFactory.WithTenantID(tenantID), + testutil.AttemptFactory.WithEventID(event.ID), + testutil.AttemptFactory.WithDestinationID(destID), + testutil.AttemptFactory.WithStatus("success"), + testutil.AttemptFactory.WithTime(baseTime.Add(-30*time.Minute)), ) - de := &models.DeliveryEvent{ - ID: "single_de", - DestinationID: destID, - Event: *event, - Delivery: delivery, - } - err := logStore.InsertManyDeliveryEvent(ctx, []*models.DeliveryEvent{de}) + err := logStore.InsertMany(ctx, []*models.LogEntry{{Event: event, Attempt: delivery}}) require.NoError(t, err) require.NoError(t, h.FlushWrites(ctx)) // Track in maps for later filter tests destinationEvents[destID] = append(destinationEvents[destID], event) topicEvents[topic] = append(topicEvents[topic], event) - statusDeliveryEvents["success"] = append(statusDeliveryEvents["success"], de) + statusDeliveries["success"] = append(statusDeliveries["success"], delivery) // Verify via List - response, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + response, err := logStore.ListAttempt(ctx, driver.ListAttemptRequest{ TenantID: tenantID, EventID: event.ID, Limit: 10, @@ -82,7 +77,7 @@ func testCRUD(t *testing.T, newHarness HarnessMaker) { require.NoError(t, err) require.Len(t, response.Data, 1) assert.Equal(t, event.ID, response.Data[0].Event.ID) - assert.Equal(t, "success", response.Data[0].Delivery.Status) + assert.Equal(t, "success", response.Data[0].Attempt.Status) // Verify via Retrieve retrieved, err := logStore.RetrieveEvent(ctx, driver.RetrieveEventRequest{ @@ -94,8 +89,10 @@ func testCRUD(t *testing.T, newHarness HarnessMaker) { assert.Equal(t, event.ID, retrieved.ID) }) - t.Run("batch delivery events", func(t *testing.T) { + t.Run("batch deliveries", func(t *testing.T) { // Create 15 events spread across destinations and topics for filter testing + var entries []*models.LogEntry + for i := range 15 { destID := destinationIDs[i%len(destinationIDs)] topic := testutil.TestTopics[i%len(testutil.TestTopics)] @@ -112,32 +109,28 @@ func testCRUD(t *testing.T, newHarness HarnessMaker) { testutil.EventFactory.WithTopic(topic), testutil.EventFactory.WithTime(eventTime), ) - delivery := testutil.DeliveryFactory.AnyPointer( - testutil.DeliveryFactory.WithID(fmt.Sprintf("batch_del_%02d", i)), - testutil.DeliveryFactory.WithEventID(event.ID), - testutil.DeliveryFactory.WithDestinationID(destID), - testutil.DeliveryFactory.WithStatus(status), - testutil.DeliveryFactory.WithTime(eventTime.Add(time.Millisecond)), + delivery := testutil.AttemptFactory.AnyPointer( + testutil.AttemptFactory.WithID(fmt.Sprintf("batch_del_%02d", i)), + testutil.AttemptFactory.WithTenantID(tenantID), + testutil.AttemptFactory.WithEventID(event.ID), + testutil.AttemptFactory.WithDestinationID(destID), + testutil.AttemptFactory.WithStatus(status), + testutil.AttemptFactory.WithTime(eventTime.Add(time.Millisecond)), ) - de := &models.DeliveryEvent{ - ID: fmt.Sprintf("batch_de_%02d", i), - DestinationID: destID, - Event: *event, - Delivery: delivery, - } - allDeliveryEvents = append(allDeliveryEvents, de) + entries = append(entries, &models.LogEntry{Event: event, Attempt: delivery}) + allDeliveries = append(allDeliveries, delivery) destinationEvents[destID] = append(destinationEvents[destID], event) topicEvents[topic] = append(topicEvents[topic], event) - statusDeliveryEvents[status] = append(statusDeliveryEvents[status], de) + statusDeliveries[status] = append(statusDeliveries[status], delivery) } - err := logStore.InsertManyDeliveryEvent(ctx, allDeliveryEvents) + err := logStore.InsertMany(ctx, entries) require.NoError(t, err) require.NoError(t, h.FlushWrites(ctx)) // Verify all inserted - response, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + response, err := logStore.ListAttempt(ctx, driver.ListAttemptRequest{ TenantID: tenantID, Limit: 100, TimeFilter: driver.TimeFilter{GTE: &startTime}, @@ -148,7 +141,7 @@ func testCRUD(t *testing.T, newHarness HarnessMaker) { }) t.Run("empty batch is no-op", func(t *testing.T) { - err := logStore.InsertManyDeliveryEvent(ctx, []*models.DeliveryEvent{}) + err := logStore.InsertMany(ctx, []*models.LogEntry{}) require.NoError(t, err) }) }) @@ -210,50 +203,50 @@ func testCRUD(t *testing.T, newHarness HarnessMaker) { } }) - t.Run("ListDeliveryEvent by destination", func(t *testing.T) { + t.Run("ListAttempt by destination", func(t *testing.T) { destID := destinationIDs[0] - response, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + response, err := logStore.ListAttempt(ctx, driver.ListAttemptRequest{ TenantID: tenantID, DestinationIDs: []string{destID}, Limit: 100, TimeFilter: driver.TimeFilter{GTE: &startTime}, }) require.NoError(t, err) - for _, de := range response.Data { - assert.Equal(t, destID, de.DestinationID) + for _, dr := range response.Data { + assert.Equal(t, destID, dr.Attempt.DestinationID) } }) - t.Run("ListDeliveryEvent by status", func(t *testing.T) { - response, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + t.Run("ListAttempt by status", func(t *testing.T) { + response, err := logStore.ListAttempt(ctx, driver.ListAttemptRequest{ TenantID: tenantID, Status: "success", Limit: 100, TimeFilter: driver.TimeFilter{GTE: &startTime}, }) require.NoError(t, err) - for _, de := range response.Data { - assert.Equal(t, "success", de.Delivery.Status) + for _, dr := range response.Data { + assert.Equal(t, "success", dr.Attempt.Status) } }) - t.Run("ListDeliveryEvent by topic", func(t *testing.T) { + t.Run("ListAttempt by topic", func(t *testing.T) { topic := testutil.TestTopics[0] - response, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + response, err := logStore.ListAttempt(ctx, driver.ListAttemptRequest{ TenantID: tenantID, Topics: []string{topic}, Limit: 100, TimeFilter: driver.TimeFilter{GTE: &startTime}, }) require.NoError(t, err) - for _, de := range response.Data { - assert.Equal(t, topic, de.Event.Topic) + for _, dr := range response.Data { + assert.Equal(t, topic, dr.Event.Topic) } }) - t.Run("ListDeliveryEvent by event ID", func(t *testing.T) { + t.Run("ListAttempt by event ID", func(t *testing.T) { eventID := "batch_evt_00" - response, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + response, err := logStore.ListAttempt(ctx, driver.ListAttemptRequest{ TenantID: tenantID, EventID: eventID, Limit: 100, @@ -268,7 +261,7 @@ func testCRUD(t *testing.T, newHarness HarnessMaker) { t.Run("retrieve", func(t *testing.T) { // Use one of our batch events for retrieve tests knownEventID := "batch_evt_00" - knownDeliveryID := "batch_del_00" + knownAttemptID := "batch_del_00" t.Run("RetrieveEvent existing", func(t *testing.T) { retrieved, err := logStore.RetrieveEvent(ctx, driver.RetrieveEventRequest{ @@ -310,29 +303,29 @@ func testCRUD(t *testing.T, newHarness HarnessMaker) { assert.Nil(t, retrieved) }) - t.Run("RetrieveDeliveryEvent existing", func(t *testing.T) { - retrieved, err := logStore.RetrieveDeliveryEvent(ctx, driver.RetrieveDeliveryEventRequest{ - TenantID: tenantID, - DeliveryID: knownDeliveryID, + t.Run("RetrieveAttempt existing", func(t *testing.T) { + retrieved, err := logStore.RetrieveAttempt(ctx, driver.RetrieveAttemptRequest{ + TenantID: tenantID, + AttemptID: knownAttemptID, }) require.NoError(t, err) require.NotNil(t, retrieved) - assert.Equal(t, knownDeliveryID, retrieved.Delivery.ID) + assert.Equal(t, knownAttemptID, retrieved.Attempt.ID) }) - t.Run("RetrieveDeliveryEvent non-existent returns nil", func(t *testing.T) { - retrieved, err := logStore.RetrieveDeliveryEvent(ctx, driver.RetrieveDeliveryEventRequest{ - TenantID: tenantID, - DeliveryID: "non-existent-delivery", + t.Run("RetrieveAttempt non-existent returns nil", func(t *testing.T) { + retrieved, err := logStore.RetrieveAttempt(ctx, driver.RetrieveAttemptRequest{ + TenantID: tenantID, + AttemptID: "non-existent-delivery", }) require.NoError(t, err) assert.Nil(t, retrieved) }) - t.Run("RetrieveDeliveryEvent wrong tenant returns nil", func(t *testing.T) { - retrieved, err := logStore.RetrieveDeliveryEvent(ctx, driver.RetrieveDeliveryEventRequest{ - TenantID: "wrong-tenant", - DeliveryID: knownDeliveryID, + t.Run("RetrieveAttempt wrong tenant returns nil", func(t *testing.T) { + retrieved, err := logStore.RetrieveAttempt(ctx, driver.RetrieveAttemptRequest{ + TenantID: "wrong-tenant", + AttemptID: knownAttemptID, }) require.NoError(t, err) assert.Nil(t, retrieved) diff --git a/internal/logstore/drivertest/misc.go b/internal/logstore/drivertest/misc.go index dc594ef6..b9dc9cbe 100644 --- a/internal/logstore/drivertest/misc.go +++ b/internal/logstore/drivertest/misc.go @@ -54,12 +54,12 @@ func testIsolation(t *testing.T, ctx context.Context, logStore driver.LogStore, testutil.EventFactory.WithTopic("test.topic"), testutil.EventFactory.WithTime(baseTime.Add(-10*time.Minute)), ) - delivery1 := testutil.DeliveryFactory.AnyPointer( - testutil.DeliveryFactory.WithID("tenant1-delivery"), - testutil.DeliveryFactory.WithEventID(event1.ID), - testutil.DeliveryFactory.WithDestinationID(destinationID), - testutil.DeliveryFactory.WithStatus("success"), - testutil.DeliveryFactory.WithTime(baseTime.Add(-10*time.Minute)), + attempt1 := testutil.AttemptFactory.AnyPointer( + testutil.AttemptFactory.WithID("tenant1-delivery"), + testutil.AttemptFactory.WithEventID(event1.ID), + testutil.AttemptFactory.WithDestinationID(destinationID), + testutil.AttemptFactory.WithStatus("success"), + testutil.AttemptFactory.WithTime(baseTime.Add(-10*time.Minute)), ) event2 := testutil.EventFactory.AnyPointer( @@ -69,23 +69,23 @@ func testIsolation(t *testing.T, ctx context.Context, logStore driver.LogStore, testutil.EventFactory.WithTopic("test.topic"), testutil.EventFactory.WithTime(baseTime.Add(-5*time.Minute)), ) - delivery2 := testutil.DeliveryFactory.AnyPointer( - testutil.DeliveryFactory.WithID("tenant2-delivery"), - testutil.DeliveryFactory.WithEventID(event2.ID), - testutil.DeliveryFactory.WithDestinationID(destinationID), - testutil.DeliveryFactory.WithStatus("failed"), - testutil.DeliveryFactory.WithTime(baseTime.Add(-5*time.Minute)), + attempt2 := testutil.AttemptFactory.AnyPointer( + testutil.AttemptFactory.WithID("tenant2-delivery"), + testutil.AttemptFactory.WithEventID(event2.ID), + testutil.AttemptFactory.WithDestinationID(destinationID), + testutil.AttemptFactory.WithStatus("failed"), + testutil.AttemptFactory.WithTime(baseTime.Add(-5*time.Minute)), ) - require.NoError(t, logStore.InsertManyDeliveryEvent(ctx, []*models.DeliveryEvent{ - {ID: idgen.DeliveryEvent(), DestinationID: destinationID, Event: *event1, Delivery: delivery1}, - {ID: idgen.DeliveryEvent(), DestinationID: destinationID, Event: *event2, Delivery: delivery2}, + require.NoError(t, logStore.InsertMany(ctx, []*models.LogEntry{ + {Event: event1, Attempt: attempt1}, + {Event: event2, Attempt: attempt2}, })) require.NoError(t, h.FlushWrites(ctx)) t.Run("TenantIsolation", func(t *testing.T) { - t.Run("ListDeliveryEvent isolates by tenant", func(t *testing.T) { - response, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + t.Run("ListAttempt isolates by tenant", func(t *testing.T) { + response, err := logStore.ListAttempt(ctx, driver.ListAttemptRequest{ TenantID: tenant1ID, Limit: 100, TimeFilter: driver.TimeFilter{GTE: &startTime}, @@ -94,7 +94,7 @@ func testIsolation(t *testing.T, ctx context.Context, logStore driver.LogStore, require.Len(t, response.Data, 1) assert.Equal(t, "tenant1-event", response.Data[0].Event.ID) - response, err = logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + response, err = logStore.ListAttempt(ctx, driver.ListAttemptRequest{ TenantID: tenant2ID, Limit: 100, TimeFilter: driver.TimeFilter{GTE: &startTime}, @@ -140,8 +140,8 @@ func testIsolation(t *testing.T, ctx context.Context, logStore driver.LogStore, assert.True(t, tenantsSeen[tenant2ID]) }) - t.Run("ListDeliveryEvent returns all tenants when TenantID empty", func(t *testing.T) { - response, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + t.Run("ListAttempt returns all tenants when TenantID empty", func(t *testing.T) { + response, err := logStore.ListAttempt(ctx, driver.ListAttemptRequest{ TenantID: "", DestinationIDs: []string{destinationID}, Limit: 100, @@ -169,18 +169,18 @@ func testIsolation(t *testing.T, ctx context.Context, logStore driver.LogStore, assert.Equal(t, tenant2ID, retrieved2.TenantID) }) - t.Run("RetrieveDeliveryEvent finds delivery across tenants when TenantID empty", func(t *testing.T) { - retrieved1, err := logStore.RetrieveDeliveryEvent(ctx, driver.RetrieveDeliveryEventRequest{ - TenantID: "", - DeliveryID: "tenant1-delivery", + t.Run("RetrieveAttempt finds attempt across tenants when TenantID empty", func(t *testing.T) { + retrieved1, err := logStore.RetrieveAttempt(ctx, driver.RetrieveAttemptRequest{ + TenantID: "", + AttemptID: "tenant1-delivery", }) require.NoError(t, err) require.NotNil(t, retrieved1) assert.Equal(t, tenant1ID, retrieved1.Event.TenantID) - retrieved2, err := logStore.RetrieveDeliveryEvent(ctx, driver.RetrieveDeliveryEventRequest{ - TenantID: "", - DeliveryID: "tenant2-delivery", + retrieved2, err := logStore.RetrieveAttempt(ctx, driver.RetrieveAttemptRequest{ + TenantID: "", + AttemptID: "tenant2-delivery", }) require.NoError(t, err) require.NotNil(t, retrieved2) @@ -195,7 +195,7 @@ func testEdgeCases(t *testing.T, ctx context.Context, logStore driver.LogStore, destinationID := idgen.Destination() baseTime := time.Now().Truncate(time.Second) - var deliveryEvents []*models.DeliveryEvent + var entries []*models.LogEntry for i := range 3 { event := testutil.EventFactory.AnyPointer( testutil.EventFactory.WithID(fmt.Sprintf("sort_evt_%d", i)), @@ -203,25 +203,21 @@ func testEdgeCases(t *testing.T, ctx context.Context, logStore driver.LogStore, testutil.EventFactory.WithDestinationID(destinationID), testutil.EventFactory.WithTime(baseTime.Add(-time.Duration(i)*time.Hour)), ) - delivery := testutil.DeliveryFactory.AnyPointer( - testutil.DeliveryFactory.WithID(fmt.Sprintf("sort_del_%d", i)), - testutil.DeliveryFactory.WithEventID(event.ID), - testutil.DeliveryFactory.WithDestinationID(destinationID), - testutil.DeliveryFactory.WithTime(baseTime.Add(-time.Duration(i)*time.Hour)), + attempt := testutil.AttemptFactory.AnyPointer( + testutil.AttemptFactory.WithID(fmt.Sprintf("sort_del_%d", i)), + testutil.AttemptFactory.WithTenantID(tenantID), + testutil.AttemptFactory.WithEventID(event.ID), + testutil.AttemptFactory.WithDestinationID(destinationID), + testutil.AttemptFactory.WithTime(baseTime.Add(-time.Duration(i)*time.Hour)), ) - deliveryEvents = append(deliveryEvents, &models.DeliveryEvent{ - ID: fmt.Sprintf("sort_de_%d", i), - DestinationID: destinationID, - Event: *event, - Delivery: delivery, - }) + entries = append(entries, &models.LogEntry{Event: event, Attempt: attempt}) } - require.NoError(t, logStore.InsertManyDeliveryEvent(ctx, deliveryEvents)) + require.NoError(t, logStore.InsertMany(ctx, entries)) startTime := baseTime.Add(-48 * time.Hour) t.Run("invalid SortOrder uses default (desc)", func(t *testing.T) { - response, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + response, err := logStore.ListAttempt(ctx, driver.ListAttemptRequest{ TenantID: tenantID, SortOrder: "sideways", TimeFilter: driver.TimeFilter{GTE: &startTime}, @@ -229,8 +225,8 @@ func testEdgeCases(t *testing.T, ctx context.Context, logStore driver.LogStore, }) require.NoError(t, err) require.Len(t, response.Data, 3) - assert.Equal(t, "sort_del_0", response.Data[0].Delivery.ID) - assert.Equal(t, "sort_del_2", response.Data[2].Delivery.ID) + assert.Equal(t, "sort_del_0", response.Data[0].Attempt.ID) + assert.Equal(t, "sort_del_2", response.Data[2].Attempt.ID) }) }) @@ -244,16 +240,15 @@ func testEdgeCases(t *testing.T, ctx context.Context, logStore driver.LogStore, testutil.EventFactory.WithDestinationID(destinationID), testutil.EventFactory.WithTopic("test.topic"), ) - delivery := testutil.DeliveryFactory.AnyPointer( - testutil.DeliveryFactory.WithEventID(event.ID), - testutil.DeliveryFactory.WithDestinationID(destinationID), + attempt := testutil.AttemptFactory.AnyPointer( + testutil.AttemptFactory.WithTenantID(tenantID), + testutil.AttemptFactory.WithEventID(event.ID), + testutil.AttemptFactory.WithDestinationID(destinationID), ) - require.NoError(t, logStore.InsertManyDeliveryEvent(ctx, []*models.DeliveryEvent{ - {ID: idgen.DeliveryEvent(), DestinationID: destinationID, Event: *event, Delivery: delivery}, - })) + require.NoError(t, logStore.InsertMany(ctx, []*models.LogEntry{{Event: event, Attempt: attempt}})) t.Run("nil DestinationIDs equals empty DestinationIDs", func(t *testing.T) { - responseNil, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + responseNil, err := logStore.ListAttempt(ctx, driver.ListAttemptRequest{ TenantID: tenantID, DestinationIDs: nil, TimeFilter: driver.TimeFilter{GTE: &startTime}, @@ -261,7 +256,7 @@ func testEdgeCases(t *testing.T, ctx context.Context, logStore driver.LogStore, }) require.NoError(t, err) - responseEmpty, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + responseEmpty, err := logStore.ListAttempt(ctx, driver.ListAttemptRequest{ TenantID: tenantID, DestinationIDs: []string{}, TimeFilter: driver.TimeFilter{GTE: &startTime}, @@ -301,19 +296,18 @@ func testEdgeCases(t *testing.T, ctx context.Context, logStore driver.LogStore, ) for _, evt := range []*models.Event{eventBefore, eventAt, eventAfter} { - delivery := testutil.DeliveryFactory.AnyPointer( - testutil.DeliveryFactory.WithID(fmt.Sprintf("del_%s", evt.ID)), - testutil.DeliveryFactory.WithEventID(evt.ID), - testutil.DeliveryFactory.WithDestinationID(destinationID), - testutil.DeliveryFactory.WithTime(evt.Time), + attempt := testutil.AttemptFactory.AnyPointer( + testutil.AttemptFactory.WithID(fmt.Sprintf("del_%s", evt.ID)), + testutil.AttemptFactory.WithTenantID(tenantID), + testutil.AttemptFactory.WithEventID(evt.ID), + testutil.AttemptFactory.WithDestinationID(destinationID), + testutil.AttemptFactory.WithTime(evt.Time), ) - require.NoError(t, logStore.InsertManyDeliveryEvent(ctx, []*models.DeliveryEvent{ - {ID: idgen.DeliveryEvent(), DestinationID: destinationID, Event: *evt, Delivery: delivery}, - })) + require.NoError(t, logStore.InsertMany(ctx, []*models.LogEntry{{Event: evt, Attempt: attempt}})) } t.Run("GTE is inclusive (>=)", func(t *testing.T) { - response, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + response, err := logStore.ListAttempt(ctx, driver.ListAttemptRequest{ TenantID: tenantID, TimeFilter: driver.TimeFilter{GTE: &boundaryTime}, Limit: 10, @@ -324,7 +318,7 @@ func testEdgeCases(t *testing.T, ctx context.Context, logStore driver.LogStore, t.Run("LTE is inclusive (<=)", func(t *testing.T) { farPast := boundaryTime.Add(-1 * time.Hour) - response, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + response, err := logStore.ListAttempt(ctx, driver.ListAttemptRequest{ TenantID: tenantID, TimeFilter: driver.TimeFilter{GTE: &farPast, LTE: &boundaryTime}, Limit: 10, @@ -343,16 +337,15 @@ func testEdgeCases(t *testing.T, ctx context.Context, logStore driver.LogStore, testutil.EventFactory.WithTenantID(tenantID), testutil.EventFactory.WithDestinationID(destinationID), ) - delivery := testutil.DeliveryFactory.AnyPointer( - testutil.DeliveryFactory.WithEventID(event.ID), - testutil.DeliveryFactory.WithDestinationID(destinationID), + attempt := testutil.AttemptFactory.AnyPointer( + testutil.AttemptFactory.WithTenantID(tenantID), + testutil.AttemptFactory.WithEventID(event.ID), + testutil.AttemptFactory.WithDestinationID(destinationID), ) - require.NoError(t, logStore.InsertManyDeliveryEvent(ctx, []*models.DeliveryEvent{ - {ID: idgen.DeliveryEvent(), DestinationID: destinationID, Event: *event, Delivery: delivery}, - })) + require.NoError(t, logStore.InsertMany(ctx, []*models.LogEntry{{Event: event, Attempt: attempt}})) - t.Run("modifying ListDeliveryEvent result doesn't affect subsequent queries", func(t *testing.T) { - response1, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + t.Run("modifying ListAttempt result doesn't affect subsequent queries", func(t *testing.T) { + response1, err := logStore.ListAttempt(ctx, driver.ListAttemptRequest{ TenantID: tenantID, Limit: 10, TimeFilter: driver.TimeFilter{GTE: &startTime}, @@ -363,7 +356,7 @@ func testEdgeCases(t *testing.T, ctx context.Context, logStore driver.LogStore, originalID := response1.Data[0].Event.ID response1.Data[0].Event.ID = "MODIFIED" - response2, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + response2, err := logStore.ListAttempt(ctx, driver.ListAttemptRequest{ TenantID: tenantID, Limit: 10, TimeFilter: driver.TimeFilter{GTE: &startTime}, @@ -378,7 +371,7 @@ func testEdgeCases(t *testing.T, ctx context.Context, logStore driver.LogStore, tenantID := idgen.String() destinationID := idgen.Destination() eventTime := time.Now().Add(-30 * time.Minute).Truncate(time.Second) - deliveryTime := eventTime.Add(1 * time.Second) + attemptTime := eventTime.Add(1 * time.Second) startTime := eventTime.Add(-1 * time.Hour) event := testutil.EventFactory.AnyPointer( @@ -386,19 +379,14 @@ func testEdgeCases(t *testing.T, ctx context.Context, logStore driver.LogStore, testutil.EventFactory.WithDestinationID(destinationID), testutil.EventFactory.WithTime(eventTime), ) - delivery := testutil.DeliveryFactory.AnyPointer( - testutil.DeliveryFactory.WithEventID(event.ID), - testutil.DeliveryFactory.WithDestinationID(destinationID), - testutil.DeliveryFactory.WithStatus("success"), - testutil.DeliveryFactory.WithTime(deliveryTime), + attempt := testutil.AttemptFactory.AnyPointer( + testutil.AttemptFactory.WithTenantID(tenantID), + testutil.AttemptFactory.WithEventID(event.ID), + testutil.AttemptFactory.WithDestinationID(destinationID), + testutil.AttemptFactory.WithStatus("success"), + testutil.AttemptFactory.WithTime(attemptTime), ) - de := &models.DeliveryEvent{ - ID: idgen.DeliveryEvent(), - DestinationID: destinationID, - Event: *event, - Delivery: delivery, - } - batch := []*models.DeliveryEvent{de} + entries := []*models.LogEntry{{Event: event, Attempt: attempt}} // Race N goroutines all inserting the same record const numGoroutines = 10 @@ -407,14 +395,14 @@ func testEdgeCases(t *testing.T, ctx context.Context, logStore driver.LogStore, wg.Add(1) go func() { defer wg.Done() - _ = logStore.InsertManyDeliveryEvent(ctx, batch) + _ = logStore.InsertMany(ctx, entries) }() } wg.Wait() require.NoError(t, h.FlushWrites(ctx)) // Assert: still exactly 1 record - response, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + response, err := logStore.ListAttempt(ctx, driver.ListAttemptRequest{ TenantID: tenantID, Limit: 100, TimeFilter: driver.TimeFilter{GTE: &startTime}, @@ -439,7 +427,7 @@ func testCursorValidation(t *testing.T, ctx context.Context, logStore driver.Log for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - _, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + _, err := logStore.ListAttempt(ctx, driver.ListAttemptRequest{ TenantID: tenantID, SortOrder: "desc", Next: tc.cursor, @@ -465,20 +453,19 @@ func testCursorValidation(t *testing.T, ctx context.Context, logStore driver.Log testutil.EventFactory.WithDestinationID(destinationID), testutil.EventFactory.WithTime(baseTime.Add(time.Duration(i)*time.Second)), ) - delivery := testutil.DeliveryFactory.AnyPointer( - testutil.DeliveryFactory.WithID(fmt.Sprintf("cursor_del_%d", i)), - testutil.DeliveryFactory.WithEventID(event.ID), - testutil.DeliveryFactory.WithDestinationID(destinationID), - testutil.DeliveryFactory.WithTime(baseTime.Add(time.Duration(i)*time.Second)), + attempt := testutil.AttemptFactory.AnyPointer( + testutil.AttemptFactory.WithID(fmt.Sprintf("cursor_del_%d", i)), + testutil.AttemptFactory.WithTenantID(tenantID), + testutil.AttemptFactory.WithEventID(event.ID), + testutil.AttemptFactory.WithDestinationID(destinationID), + testutil.AttemptFactory.WithTime(baseTime.Add(time.Duration(i)*time.Second)), ) - require.NoError(t, logStore.InsertManyDeliveryEvent(ctx, []*models.DeliveryEvent{ - {ID: fmt.Sprintf("cursor_de_%d", i), DestinationID: destinationID, Event: *event, Delivery: delivery}, - })) + require.NoError(t, logStore.InsertMany(ctx, []*models.LogEntry{{Event: event, Attempt: attempt}})) } require.NoError(t, h.FlushWrites(ctx)) t.Run("delivery_time desc", func(t *testing.T) { - page1, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + page1, err := logStore.ListAttempt(ctx, driver.ListAttemptRequest{ TenantID: tenantID, SortOrder: "desc", TimeFilter: driver.TimeFilter{GTE: &startTime}, @@ -487,7 +474,7 @@ func testCursorValidation(t *testing.T, ctx context.Context, logStore driver.Log require.NoError(t, err) require.NotEmpty(t, page1.Next) - page2, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + page2, err := logStore.ListAttempt(ctx, driver.ListAttemptRequest{ TenantID: tenantID, SortOrder: "desc", Next: page1.Next, @@ -499,7 +486,7 @@ func testCursorValidation(t *testing.T, ctx context.Context, logStore driver.Log }) t.Run("delivery_time asc", func(t *testing.T) { - page1, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + page1, err := logStore.ListAttempt(ctx, driver.ListAttemptRequest{ TenantID: tenantID, SortOrder: "asc", TimeFilter: driver.TimeFilter{GTE: &startTime}, @@ -508,7 +495,7 @@ func testCursorValidation(t *testing.T, ctx context.Context, logStore driver.Log require.NoError(t, err) require.NotEmpty(t, page1.Next) - page2, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + page2, err := logStore.ListAttempt(ctx, driver.ListAttemptRequest{ TenantID: tenantID, SortOrder: "asc", Next: page1.Next, diff --git a/internal/logstore/drivertest/pagination.go b/internal/logstore/drivertest/pagination.go index 6a48877c..67de5176 100644 --- a/internal/logstore/drivertest/pagination.go +++ b/internal/logstore/drivertest/pagination.go @@ -28,11 +28,11 @@ func testPagination(t *testing.T, newHarness HarnessMaker) { baseTime := time.Now().Truncate(time.Second) farPast := baseTime.Add(-48 * time.Hour) - t.Run("ListDeliveryEvent", func(t *testing.T) { + t.Run("ListAttempt", func(t *testing.T) { var tenantID, destinationID, idPrefix string - suite := paginationtest.Suite[*models.DeliveryEvent]{ - Name: "ListDeliveryEvent", + suite := paginationtest.Suite[*driver.AttemptRecord]{ + Name: "ListAttempt", Cleanup: func(ctx context.Context) error { tenantID = idgen.String() @@ -41,9 +41,9 @@ func testPagination(t *testing.T, newHarness HarnessMaker) { return nil }, - NewItem: func(i int) *models.DeliveryEvent { + NewItem: func(i int) *driver.AttemptRecord { eventTime := baseTime.Add(time.Duration(i) * time.Second) - deliveryTime := eventTime.Add(100 * time.Millisecond) + attemptTime := eventTime.Add(100 * time.Millisecond) event := &models.Event{ ID: fmt.Sprintf("%s_evt_%03d", idPrefix, i), @@ -56,29 +56,32 @@ func testPagination(t *testing.T, newHarness HarnessMaker) { Data: map[string]any{}, } - delivery := &models.Delivery{ + attempt := &models.Attempt{ ID: fmt.Sprintf("%s_del_%03d", idPrefix, i), + TenantID: tenantID, EventID: event.ID, DestinationID: destinationID, Status: "success", - Time: deliveryTime, + Time: attemptTime, Code: "200", } - return &models.DeliveryEvent{ - ID: fmt.Sprintf("%s_de_%03d", idPrefix, i), - DestinationID: destinationID, - Event: *event, - Delivery: delivery, + return &driver.AttemptRecord{ + Event: event, + Attempt: attempt, } }, - InsertMany: func(ctx context.Context, items []*models.DeliveryEvent) error { - return logStore.InsertManyDeliveryEvent(ctx, items) + InsertMany: func(ctx context.Context, items []*driver.AttemptRecord) error { + entries := make([]*models.LogEntry, len(items)) + for i, dr := range items { + entries[i] = &models.LogEntry{Event: dr.Event, Attempt: dr.Attempt} + } + return logStore.InsertMany(ctx, entries) }, - List: func(ctx context.Context, opts paginationtest.ListOpts) (paginationtest.ListResult[*models.DeliveryEvent], error) { - res, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + List: func(ctx context.Context, opts paginationtest.ListOpts) (paginationtest.ListResult[*driver.AttemptRecord], error) { + res, err := logStore.ListAttempt(ctx, driver.ListAttemptRequest{ TenantID: tenantID, Limit: opts.Limit, SortOrder: opts.Order, @@ -87,17 +90,17 @@ func testPagination(t *testing.T, newHarness HarnessMaker) { TimeFilter: driver.TimeFilter{GTE: &farPast}, }) if err != nil { - return paginationtest.ListResult[*models.DeliveryEvent]{}, err + return paginationtest.ListResult[*driver.AttemptRecord]{}, err } - return paginationtest.ListResult[*models.DeliveryEvent]{ + return paginationtest.ListResult[*driver.AttemptRecord]{ Items: res.Data, Next: res.Next, Prev: res.Prev, }, nil }, - GetID: func(de *models.DeliveryEvent) string { - return de.Delivery.ID + GetID: func(dr *driver.AttemptRecord) string { + return dr.Attempt.ID }, AfterInsert: func(ctx context.Context) error { @@ -108,11 +111,11 @@ func testPagination(t *testing.T, newHarness HarnessMaker) { suite.Run(t) }) - t.Run("ListDeliveryEvent_WithDestinationFilter", func(t *testing.T) { + t.Run("ListAttempt_WithDestinationFilter", func(t *testing.T) { var tenantID, targetDestID, otherDestID, idPrefix string - suite := paginationtest.Suite[*models.DeliveryEvent]{ - Name: "ListDeliveryEvent_WithDestinationFilter", + suite := paginationtest.Suite[*driver.AttemptRecord]{ + Name: "ListAttempt_WithDestinationFilter", Cleanup: func(ctx context.Context) error { tenantID = idgen.String() @@ -122,9 +125,9 @@ func testPagination(t *testing.T, newHarness HarnessMaker) { return nil }, - NewItem: func(i int) *models.DeliveryEvent { + NewItem: func(i int) *driver.AttemptRecord { eventTime := baseTime.Add(time.Duration(i) * time.Second) - deliveryTime := eventTime.Add(100 * time.Millisecond) + attemptTime := eventTime.Add(100 * time.Millisecond) destID := targetDestID if i%2 == 1 { @@ -142,29 +145,32 @@ func testPagination(t *testing.T, newHarness HarnessMaker) { Data: map[string]any{}, } - delivery := &models.Delivery{ + attempt := &models.Attempt{ ID: fmt.Sprintf("%s_del_%03d", idPrefix, i), + TenantID: tenantID, EventID: event.ID, DestinationID: destID, Status: "success", - Time: deliveryTime, + Time: attemptTime, Code: "200", } - return &models.DeliveryEvent{ - ID: fmt.Sprintf("%s_de_%03d", idPrefix, i), - DestinationID: destID, - Event: *event, - Delivery: delivery, + return &driver.AttemptRecord{ + Event: event, + Attempt: attempt, } }, - InsertMany: func(ctx context.Context, items []*models.DeliveryEvent) error { - return logStore.InsertManyDeliveryEvent(ctx, items) + InsertMany: func(ctx context.Context, items []*driver.AttemptRecord) error { + entries := make([]*models.LogEntry, len(items)) + for i, dr := range items { + entries[i] = &models.LogEntry{Event: dr.Event, Attempt: dr.Attempt} + } + return logStore.InsertMany(ctx, entries) }, - List: func(ctx context.Context, opts paginationtest.ListOpts) (paginationtest.ListResult[*models.DeliveryEvent], error) { - res, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + List: func(ctx context.Context, opts paginationtest.ListOpts) (paginationtest.ListResult[*driver.AttemptRecord], error) { + res, err := logStore.ListAttempt(ctx, driver.ListAttemptRequest{ TenantID: tenantID, DestinationIDs: []string{targetDestID}, Limit: opts.Limit, @@ -174,21 +180,21 @@ func testPagination(t *testing.T, newHarness HarnessMaker) { TimeFilter: driver.TimeFilter{GTE: &farPast}, }) if err != nil { - return paginationtest.ListResult[*models.DeliveryEvent]{}, err + return paginationtest.ListResult[*driver.AttemptRecord]{}, err } - return paginationtest.ListResult[*models.DeliveryEvent]{ + return paginationtest.ListResult[*driver.AttemptRecord]{ Items: res.Data, Next: res.Next, Prev: res.Prev, }, nil }, - GetID: func(de *models.DeliveryEvent) string { - return de.Delivery.ID + GetID: func(dr *driver.AttemptRecord) string { + return dr.Attempt.ID }, - Matches: func(de *models.DeliveryEvent) bool { - return de.DestinationID == targetDestID + Matches: func(dr *driver.AttemptRecord) bool { + return dr.Attempt.DestinationID == targetDestID }, AfterInsert: func(ctx context.Context) error { @@ -228,24 +234,23 @@ func testPagination(t *testing.T, newHarness HarnessMaker) { }, InsertMany: func(ctx context.Context, items []*models.Event) error { - des := make([]*models.DeliveryEvent, len(items)) + entries := make([]*models.LogEntry, len(items)) for i, evt := range items { - deliveryTime := evt.Time.Add(100 * time.Millisecond) - des[i] = &models.DeliveryEvent{ - ID: fmt.Sprintf("%s_de_%03d", idPrefix, i), - DestinationID: evt.DestinationID, - Event: *evt, - Delivery: &models.Delivery{ + attemptTime := evt.Time.Add(100 * time.Millisecond) + entries[i] = &models.LogEntry{ + Event: evt, + Attempt: &models.Attempt{ ID: fmt.Sprintf("%s_del_%03d", idPrefix, i), + TenantID: evt.TenantID, EventID: evt.ID, DestinationID: evt.DestinationID, Status: "success", - Time: deliveryTime, + Time: attemptTime, Code: "200", }, } } - return logStore.InsertManyDeliveryEvent(ctx, des) + return logStore.InsertMany(ctx, entries) }, List: func(ctx context.Context, opts paginationtest.ListOpts) (paginationtest.ListResult[*models.Event], error) { @@ -314,24 +319,23 @@ func testPagination(t *testing.T, newHarness HarnessMaker) { }, InsertMany: func(ctx context.Context, items []*models.Event) error { - des := make([]*models.DeliveryEvent, len(items)) + entries := make([]*models.LogEntry, len(items)) for i, evt := range items { - deliveryTime := evt.Time.Add(100 * time.Millisecond) - des[i] = &models.DeliveryEvent{ - ID: fmt.Sprintf("%s_de_%03d", idPrefix, i), - DestinationID: evt.DestinationID, - Event: *evt, - Delivery: &models.Delivery{ + attemptTime := evt.Time.Add(100 * time.Millisecond) + entries[i] = &models.LogEntry{ + Event: evt, + Attempt: &models.Attempt{ ID: fmt.Sprintf("%s_del_%03d", idPrefix, i), + TenantID: evt.TenantID, EventID: evt.ID, DestinationID: evt.DestinationID, Status: "success", - Time: deliveryTime, + Time: attemptTime, Code: "200", }, } } - return logStore.InsertManyDeliveryEvent(ctx, des) + return logStore.InsertMany(ctx, entries) }, List: func(ctx context.Context, opts paginationtest.ListOpts) (paginationtest.ListResult[*models.Event], error) { @@ -375,8 +379,8 @@ func testPagination(t *testing.T, newHarness HarnessMaker) { // time-based filters (GTE, LTE, GT, LT), which is critical for // "paginate within a time window" use cases. // - // IMPORTANT: ListDeliveryEvent filters by DELIVERY time, ListEvent filters by EVENT time. - // In this test, delivery_time = event_time + 100ms. + // IMPORTANT: ListAttempt filters by ATTEMPT time, ListEvent filters by EVENT time. + // In this test, attempt_time = event_time + 100ms. t.Run("TimeFilterWithCursor", func(t *testing.T) { tenantID := idgen.String() destinationID := idgen.Destination() @@ -388,15 +392,17 @@ func testPagination(t *testing.T, newHarness HarnessMaker) { // - Events 15-19: far future (should be excluded by LTE filter) // // Event times are spaced 2 minutes apart within the window. - // Delivery times are 1 second after event times (not sub-second) + // Attempt times are 1 second after event times (not sub-second) // to ensure GT/LT tests work consistently across databases. eventWindowStart := baseTime.Add(-10 * time.Minute) eventWindowEnd := baseTime.Add(10 * time.Minute) - // Delivery window accounts for the 1 second offset - deliveryWindowStart := eventWindowStart.Add(time.Second) - deliveryWindowEnd := eventWindowEnd.Add(time.Second) + // Attempt window accounts for the 1 second offset + attemptWindowStart := eventWindowStart.Add(time.Second) + attemptWindowEnd := eventWindowEnd.Add(time.Second) - var allEvents []*models.DeliveryEvent + var allRecords []*driver.AttemptRecord + var allEvents []*models.Event + var allAttempts []*models.Attempt for i := range 20 { var eventTime time.Time switch { @@ -412,7 +418,7 @@ func testPagination(t *testing.T, newHarness HarnessMaker) { eventTime = eventWindowEnd.Add(time.Duration(i-14) * time.Hour) } - deliveryTime := eventTime.Add(time.Second) + attemptTime := eventTime.Add(time.Second) event := &models.Event{ ID: fmt.Sprintf("%s_evt_%03d", idPrefix, i), @@ -424,45 +430,50 @@ func testPagination(t *testing.T, newHarness HarnessMaker) { Metadata: map[string]string{}, Data: map[string]any{}, } - delivery := &models.Delivery{ + attempt := &models.Attempt{ ID: fmt.Sprintf("%s_del_%03d", idPrefix, i), + TenantID: tenantID, EventID: event.ID, DestinationID: destinationID, Status: "success", - Time: deliveryTime, + Time: attemptTime, Code: "200", } - allEvents = append(allEvents, &models.DeliveryEvent{ - ID: fmt.Sprintf("%s_de_%03d", idPrefix, i), - DestinationID: destinationID, - Event: *event, - Delivery: delivery, + allRecords = append(allRecords, &driver.AttemptRecord{ + Event: event, + Attempt: attempt, }) + allEvents = append(allEvents, event) + allAttempts = append(allAttempts, attempt) } - require.NoError(t, logStore.InsertManyDeliveryEvent(ctx, allEvents)) + entries := make([]*models.LogEntry, len(allEvents)) + for i := range allEvents { + entries[i] = &models.LogEntry{Event: allEvents[i], Attempt: allAttempts[i]} + } + require.NoError(t, logStore.InsertMany(ctx, entries)) require.NoError(t, h.FlushWrites(ctx)) t.Run("paginate within time-bounded window", func(t *testing.T) { - // Paginate through deliveries within the window with limit=3 - // ListDeliveryEvent filters by DELIVERY time, not event time. - // Should only see deliveries 5-14 (10 total), not 0-4 or 15-19 + // Paginate through attempts within the window with limit=3 + // ListAttempt filters by ATTEMPT time, not event time. + // Should only see attempts 5-14 (10 total), not 0-4 or 15-19 var collectedIDs []string var nextCursor string pageCount := 0 for { - res, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + res, err := logStore.ListAttempt(ctx, driver.ListAttemptRequest{ TenantID: tenantID, Limit: 3, SortOrder: "asc", Next: nextCursor, - TimeFilter: driver.TimeFilter{GTE: &deliveryWindowStart, LTE: &deliveryWindowEnd}, + TimeFilter: driver.TimeFilter{GTE: &attemptWindowStart, LTE: &attemptWindowEnd}, }) require.NoError(t, err) - for _, de := range res.Data { - collectedIDs = append(collectedIDs, de.Event.ID) + for _, dr := range res.Data { + collectedIDs = append(collectedIDs, dr.Event.ID) } pageCount++ @@ -477,18 +488,18 @@ func testPagination(t *testing.T, newHarness HarnessMaker) { } } - // Should have collected exactly deliveries 5-14 - require.Len(t, collectedIDs, 10, "should have 10 deliveries in window") + // Should have collected exactly attempts 5-14 + require.Len(t, collectedIDs, 10, "should have 10 attempts in window") for i, id := range collectedIDs { expectedID := fmt.Sprintf("%s_evt_%03d", idPrefix, i+5) - require.Equal(t, expectedID, id, "delivery %d mismatch", i) + require.Equal(t, expectedID, id, "attempt %d mismatch", i) } require.Equal(t, 4, pageCount, "should take 4 pages (3+3+3+1)") }) - t.Run("cursor excludes deliveries outside time filter", func(t *testing.T) { - // First page with no time filter gets all deliveries - resAll, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + t.Run("cursor excludes attempts outside time filter", func(t *testing.T) { + // First page with no time filter gets all attempts + resAll, err := logStore.ListAttempt(ctx, driver.ListAttemptRequest{ TenantID: tenantID, Limit: 5, SortOrder: "asc", @@ -498,35 +509,35 @@ func testPagination(t *testing.T, newHarness HarnessMaker) { require.Len(t, resAll.Data, 5) // Use the cursor but add a time filter that excludes some results - // The cursor points to position after delivery 4 (far past deliveries) - // But with deliveryWindowStart filter, we should start from delivery 5 - res, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + // The cursor points to position after attempt 4 (far past attempts) + // But with attemptWindowStart filter, we should start from attempt 5 + res, err := logStore.ListAttempt(ctx, driver.ListAttemptRequest{ TenantID: tenantID, Limit: 5, SortOrder: "asc", Next: resAll.Next, - TimeFilter: driver.TimeFilter{GTE: &deliveryWindowStart, LTE: &deliveryWindowEnd}, + TimeFilter: driver.TimeFilter{GTE: &attemptWindowStart, LTE: &attemptWindowEnd}, }) require.NoError(t, err) - // Results should respect the time filter (on delivery time) - for _, de := range res.Data { - require.True(t, !de.Delivery.Time.Before(deliveryWindowStart), "delivery time should be >= deliveryWindowStart") - require.True(t, !de.Delivery.Time.After(deliveryWindowEnd), "delivery time should be <= deliveryWindowEnd") + // Results should respect the time filter (on attempt time) + for _, dr := range res.Data { + require.True(t, !dr.Attempt.Time.Before(attemptWindowStart), "attempt time should be >= attemptWindowStart") + require.True(t, !dr.Attempt.Time.After(attemptWindowEnd), "attempt time should be <= attemptWindowEnd") } }) - t.Run("delivery time filter with GT/LT operators", func(t *testing.T) { - // Test exclusive bounds (GT/LT instead of GTE/LTE) on delivery time - // Use delivery times slightly after delivery 5 and slightly before delivery 14 - gtTime := allEvents[5].Delivery.Time.Add(time.Second) // After delivery 5, before delivery 6 - ltTime := allEvents[14].Delivery.Time.Add(-time.Second) // Before delivery 14, after delivery 13 + t.Run("attempt time filter with GT/LT operators", func(t *testing.T) { + // Test exclusive bounds (GT/LT instead of GTE/LTE) on attempt time + // Use attempt times slightly after attempt 5 and slightly before attempt 14 + gtTime := allRecords[5].Attempt.Time.Add(time.Second) // After attempt 5, before attempt 6 + ltTime := allRecords[14].Attempt.Time.Add(-time.Second) // Before attempt 14, after attempt 13 var collectedIDs []string var nextCursor string for { - res, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + res, err := logStore.ListAttempt(ctx, driver.ListAttemptRequest{ TenantID: tenantID, Limit: 3, SortOrder: "asc", @@ -535,8 +546,8 @@ func testPagination(t *testing.T, newHarness HarnessMaker) { }) require.NoError(t, err) - for _, de := range res.Data { - collectedIDs = append(collectedIDs, de.Event.ID) + for _, dr := range res.Data { + collectedIDs = append(collectedIDs, dr.Event.ID) } if res.Next == "" { @@ -561,10 +572,10 @@ func testPagination(t *testing.T, newHarness HarnessMaker) { // comparison across databases with different timestamp precision // (PostgreSQL microseconds, ClickHouse DateTime64, etc.). // - // Important: ListDeliveryEvent filters by DELIVERY time, not event time. + // Important: ListAttempt filters by ATTEMPT time, not event time. - // First, retrieve all deliveries to find delivery 10's time - res, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + // First, retrieve all attempts to find attempt 10's time + res, err := logStore.ListAttempt(ctx, driver.ListAttemptRequest{ TenantID: tenantID, Limit: 100, SortOrder: "asc", @@ -573,90 +584,90 @@ func testPagination(t *testing.T, newHarness HarnessMaker) { }, }) require.NoError(t, err) - require.GreaterOrEqual(t, len(res.Data), 11, "need at least 11 deliveries") + require.GreaterOrEqual(t, len(res.Data), 11, "need at least 11 attempts") - // Find delivery 10's stored delivery time, truncated to seconds - var storedDelivery10Time time.Time - for _, de := range res.Data { - if de.Event.ID == allEvents[10].Event.ID { - storedDelivery10Time = de.Delivery.Time.Truncate(time.Second) + // Find attempt 10's stored attempt time, truncated to seconds + var storedAttempt10Time time.Time + for _, dr := range res.Data { + if dr.Event.ID == allRecords[10].Event.ID { + storedAttempt10Time = dr.Attempt.Time.Truncate(time.Second) break } } - require.False(t, storedDelivery10Time.IsZero(), "should find delivery 10") + require.False(t, storedAttempt10Time.IsZero(), "should find attempt 10") - // GT with exact time should exclude delivery 10 - resGT, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + // GT with exact time should exclude attempt 10 + resGT, err := logStore.ListAttempt(ctx, driver.ListAttemptRequest{ TenantID: tenantID, Limit: 100, SortOrder: "asc", - TimeFilter: driver.TimeFilter{GT: &storedDelivery10Time}, + TimeFilter: driver.TimeFilter{GT: &storedAttempt10Time}, }) require.NoError(t, err) - for _, de := range resGT.Data { - deTimeTrunc := de.Delivery.Time.Truncate(time.Second) - require.True(t, deTimeTrunc.After(storedDelivery10Time), - "GT filter should exclude delivery with exact timestamp, got delivery %s with time %v (filter time: %v)", - de.Delivery.ID, deTimeTrunc, storedDelivery10Time) + for _, dr := range resGT.Data { + drTimeTrunc := dr.Attempt.Time.Truncate(time.Second) + require.True(t, drTimeTrunc.After(storedAttempt10Time), + "GT filter should exclude attempt with exact timestamp, got attempt %s with time %v (filter time: %v)", + dr.Attempt.ID, drTimeTrunc, storedAttempt10Time) } - // LT with exact time should exclude delivery 10 - resLT, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + // LT with exact time should exclude attempt 10 + resLT, err := logStore.ListAttempt(ctx, driver.ListAttemptRequest{ TenantID: tenantID, Limit: 100, SortOrder: "asc", - TimeFilter: driver.TimeFilter{LT: &storedDelivery10Time}, + TimeFilter: driver.TimeFilter{LT: &storedAttempt10Time}, }) require.NoError(t, err) - for _, de := range resLT.Data { - deTimeTrunc := de.Delivery.Time.Truncate(time.Second) - require.True(t, deTimeTrunc.Before(storedDelivery10Time), - "LT filter should exclude delivery with exact timestamp, got delivery %s with time %v (filter time: %v)", - de.Delivery.ID, deTimeTrunc, storedDelivery10Time) + for _, dr := range resLT.Data { + drTimeTrunc := dr.Attempt.Time.Truncate(time.Second) + require.True(t, drTimeTrunc.Before(storedAttempt10Time), + "LT filter should exclude attempt with exact timestamp, got attempt %s with time %v (filter time: %v)", + dr.Attempt.ID, drTimeTrunc, storedAttempt10Time) } - // Verify delivery 10 is included with GTE/LTE (inclusive bounds) - resGTE, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + // Verify attempt 10 is included with GTE/LTE (inclusive bounds) + resGTE, err := logStore.ListAttempt(ctx, driver.ListAttemptRequest{ TenantID: tenantID, Limit: 100, SortOrder: "asc", - TimeFilter: driver.TimeFilter{GTE: &storedDelivery10Time, LTE: &storedDelivery10Time}, + TimeFilter: driver.TimeFilter{GTE: &storedAttempt10Time, LTE: &storedAttempt10Time}, }) require.NoError(t, err) - require.GreaterOrEqual(t, len(resGTE.Data), 1, "GTE/LTE with same time should include delivery at that second") + require.GreaterOrEqual(t, len(resGTE.Data), 1, "GTE/LTE with same time should include attempt at that second") }) t.Run("prev cursor respects time filter", func(t *testing.T) { - // Get first page (ListDeliveryEvent filters by delivery time) - res1, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + // Get first page (ListAttempt filters by attempt time) + res1, err := logStore.ListAttempt(ctx, driver.ListAttemptRequest{ TenantID: tenantID, Limit: 3, SortOrder: "asc", - TimeFilter: driver.TimeFilter{GTE: &deliveryWindowStart, LTE: &deliveryWindowEnd}, + TimeFilter: driver.TimeFilter{GTE: &attemptWindowStart, LTE: &attemptWindowEnd}, }) require.NoError(t, err) require.NotEmpty(t, res1.Next) // Get second page - res2, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + res2, err := logStore.ListAttempt(ctx, driver.ListAttemptRequest{ TenantID: tenantID, Limit: 3, SortOrder: "asc", Next: res1.Next, - TimeFilter: driver.TimeFilter{GTE: &deliveryWindowStart, LTE: &deliveryWindowEnd}, + TimeFilter: driver.TimeFilter{GTE: &attemptWindowStart, LTE: &attemptWindowEnd}, }) require.NoError(t, err) require.NotEmpty(t, res2.Prev) // Go back to first page using prev cursor - resPrev, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + resPrev, err := logStore.ListAttempt(ctx, driver.ListAttemptRequest{ TenantID: tenantID, Limit: 3, SortOrder: "asc", Prev: res2.Prev, - TimeFilter: driver.TimeFilter{GTE: &deliveryWindowStart, LTE: &deliveryWindowEnd}, + TimeFilter: driver.TimeFilter{GTE: &attemptWindowStart, LTE: &attemptWindowEnd}, }) require.NoError(t, err) @@ -731,7 +742,7 @@ func testPagination(t *testing.T, newHarness HarnessMaker) { // Find event 10's stored event time, truncated to seconds var storedEvent10Time time.Time for _, e := range res.Data { - if e.ID == allEvents[10].Event.ID { + if e.ID == allRecords[10].Event.ID { storedEvent10Time = e.Time.Truncate(time.Second) break } diff --git a/internal/logstore/logstore.go b/internal/logstore/logstore.go index 84a45314..b0557792 100644 --- a/internal/logstore/logstore.go +++ b/internal/logstore/logstore.go @@ -16,17 +16,19 @@ import ( type TimeFilter = driver.TimeFilter type ListEventRequest = driver.ListEventRequest type ListEventResponse = driver.ListEventResponse -type ListDeliveryEventRequest = driver.ListDeliveryEventRequest -type ListDeliveryEventResponse = driver.ListDeliveryEventResponse +type ListAttemptRequest = driver.ListAttemptRequest +type ListAttemptResponse = driver.ListAttemptResponse type RetrieveEventRequest = driver.RetrieveEventRequest -type RetrieveDeliveryEventRequest = driver.RetrieveDeliveryEventRequest +type RetrieveAttemptRequest = driver.RetrieveAttemptRequest +type AttemptRecord = driver.AttemptRecord +type LogEntry = models.LogEntry type LogStore interface { ListEvent(context.Context, ListEventRequest) (ListEventResponse, error) - ListDeliveryEvent(context.Context, ListDeliveryEventRequest) (ListDeliveryEventResponse, error) + ListAttempt(context.Context, ListAttemptRequest) (ListAttemptResponse, error) RetrieveEvent(ctx context.Context, request RetrieveEventRequest) (*models.Event, error) - RetrieveDeliveryEvent(ctx context.Context, request RetrieveDeliveryEventRequest) (*models.DeliveryEvent, error) - InsertManyDeliveryEvent(context.Context, []*models.DeliveryEvent) error + RetrieveAttempt(ctx context.Context, request RetrieveAttemptRequest) (*AttemptRecord, error) + InsertMany(context.Context, []*models.LogEntry) error } type DriverOpts struct { diff --git a/internal/logstore/memlogstore/memlogstore.go b/internal/logstore/memlogstore/memlogstore.go index fc6c8014..1f0fb0aa 100644 --- a/internal/logstore/memlogstore/memlogstore.go +++ b/internal/logstore/memlogstore/memlogstore.go @@ -14,23 +14,25 @@ import ( ) const ( - cursorResourceEvent = "evt" - cursorResourceDelivery = "dlv" - cursorVersion = 1 + cursorResourceEvent = "evt" + cursorResourceAttempt = "att" + cursorVersion = 1 ) // memLogStore is an in-memory implementation of driver.LogStore. // It serves as a reference implementation and is useful for testing. type memLogStore struct { - mu sync.RWMutex - deliveryEvents []*models.DeliveryEvent + mu sync.RWMutex + events map[string]*models.Event // keyed by event ID + attempts []*models.Attempt // list of all attempts } var _ driver.LogStore = (*memLogStore)(nil) func NewLogStore() driver.LogStore { return &memLogStore{ - deliveryEvents: make([]*models.DeliveryEvent, 0), + events: make(map[string]*models.Event), + attempts: make([]*models.Attempt, 0), } } @@ -48,20 +50,13 @@ func (s *memLogStore) ListEvent(ctx context.Context, req driver.ListEventRequest limit = 100 } - // Dedupe by event ID and filter - eventMap := make(map[string]*models.Event) - for _, de := range s.deliveryEvents { - if !s.matchesEventFilter(&de.Event, req) { + // Filter events + var allEvents []*models.Event + for _, event := range s.events { + if !s.matchesEventFilter(event, req) { continue } - if _, exists := eventMap[de.Event.ID]; !exists { - eventMap[de.Event.ID] = copyEvent(&de.Event) - } - } - - var allEvents []*models.Event - for _, event := range eventMap { - allEvents = append(allEvents, event) + allEvents = append(allEvents, copyEvent(event)) } // eventWithTimeID pairs an event with its sortable time ID for cursor operations. @@ -191,38 +186,35 @@ func (s *memLogStore) matchesEventFilter(event *models.Event, req driver.ListEve return true } -func (s *memLogStore) InsertManyDeliveryEvent(ctx context.Context, deliveryEvents []*models.DeliveryEvent) error { +func (s *memLogStore) InsertMany(ctx context.Context, entries []*models.LogEntry) error { s.mu.Lock() defer s.mu.Unlock() - for _, de := range deliveryEvents { - copied := &models.DeliveryEvent{ - ID: de.ID, - Attempt: de.Attempt, - DestinationID: de.DestinationID, - Event: de.Event, - Delivery: de.Delivery, - Manual: de.Manual, - } + for _, entry := range entries { + // Insert event (dedupe by ID) + s.events[entry.Event.ID] = copyEvent(entry.Event) + + // Insert attempt (idempotent upsert: match on event_id + attempt_id) + a := entry.Attempt + copied := copyAttempt(a) - // Idempotent upsert: match on event_id + delivery_id found := false - for i, existing := range s.deliveryEvents { - if existing.Event.ID == de.Event.ID && existing.Delivery != nil && de.Delivery != nil && existing.Delivery.ID == de.Delivery.ID { - s.deliveryEvents[i] = copied + for i, existing := range s.attempts { + if existing.EventID == a.EventID && existing.ID == a.ID { + s.attempts[i] = copied found = true break } } if !found { - s.deliveryEvents = append(s.deliveryEvents, copied) + s.attempts = append(s.attempts, copied) } } return nil } -func (s *memLogStore) ListDeliveryEvent(ctx context.Context, req driver.ListDeliveryEventRequest) (driver.ListDeliveryEventResponse, error) { +func (s *memLogStore) ListAttempt(ctx context.Context, req driver.ListAttemptRequest) (driver.ListAttemptResponse, error) { s.mu.RLock() defer s.mu.RUnlock() @@ -236,52 +228,59 @@ func (s *memLogStore) ListDeliveryEvent(ctx context.Context, req driver.ListDeli limit = 100 } - // Filter delivery events - var allDeliveryEvents []*models.DeliveryEvent - for _, de := range s.deliveryEvents { - if !s.matchesFilter(de, req) { + // Filter attempts and build records with events + var allRecords []*driver.AttemptRecord + for _, a := range s.attempts { + event := s.events[a.EventID] + if event == nil { + continue // skip orphan attempts + } + if !s.matchesAttemptFilter(a, event, req) { continue } - allDeliveryEvents = append(allDeliveryEvents, de) + allRecords = append(allRecords, &driver.AttemptRecord{ + Attempt: copyAttempt(a), + Event: copyEvent(event), + }) } - // deliveryEventWithTimeID pairs a delivery event with its sortable time ID. - type deliveryEventWithTimeID struct { - de *models.DeliveryEvent + // attemptRecordWithTimeID pairs an attempt record with its sortable time ID. + type attemptRecordWithTimeID struct { + record *driver.AttemptRecord timeID string } - // Build list with time IDs (using delivery time) - deliveryEventsWithTimeID := make([]deliveryEventWithTimeID, len(allDeliveryEvents)) - for i, de := range allDeliveryEvents { - deliveryEventsWithTimeID[i] = deliveryEventWithTimeID{ - de: de, - timeID: makeTimeID(de.Delivery.Time, de.Delivery.ID), + // Build list with time IDs (using attempt time) + recordsWithTimeID := make([]attemptRecordWithTimeID, len(allRecords)) + for i, r := range allRecords { + recordsWithTimeID[i] = attemptRecordWithTimeID{ + record: r, + timeID: makeTimeID(r.Attempt.Time, r.Attempt.ID), } } - res, err := pagination.Run(ctx, pagination.Config[deliveryEventWithTimeID]{ + res, err := pagination.Run(ctx, pagination.Config[attemptRecordWithTimeID]{ Limit: limit, Order: sortOrder, Next: req.Next, Prev: req.Prev, - Fetch: func(_ context.Context, q pagination.QueryInput) ([]deliveryEventWithTimeID, error) { + Fetch: func(_ context.Context, q pagination.QueryInput) ([]attemptRecordWithTimeID, error) { // Sort based on query direction isDesc := q.SortDir == "desc" - sort.Slice(deliveryEventsWithTimeID, func(i, j int) bool { + sort.Slice(recordsWithTimeID, func(i, j int) bool { if isDesc { - return deliveryEventsWithTimeID[i].timeID > deliveryEventsWithTimeID[j].timeID + return recordsWithTimeID[i].timeID > recordsWithTimeID[j].timeID } - return deliveryEventsWithTimeID[i].timeID < deliveryEventsWithTimeID[j].timeID + return recordsWithTimeID[i].timeID < recordsWithTimeID[j].timeID }) // Filter using q.Compare (like SQL WHERE clause) - var filtered []deliveryEventWithTimeID - for _, de := range deliveryEventsWithTimeID { + var filtered []attemptRecordWithTimeID + for _, r := range recordsWithTimeID { // If no cursor, include all items // If cursor exists, filter using Compare operator - if q.CursorPos == "" || compareTimeID(de.timeID, q.Compare, q.CursorPos) { - filtered = append(filtered, de) + if q.CursorPos == "" || compareTimeID(r.timeID, q.Compare, q.CursorPos) { + filtered = append(filtered, r) } } @@ -290,35 +289,38 @@ func (s *memLogStore) ListDeliveryEvent(ctx context.Context, req driver.ListDeli filtered = filtered[:q.Limit] } - result := make([]deliveryEventWithTimeID, len(filtered)) - for i, de := range filtered { - result[i] = deliveryEventWithTimeID{ - de: copyDeliveryEvent(de.de), - timeID: de.timeID, + result := make([]attemptRecordWithTimeID, len(filtered)) + for i, r := range filtered { + result[i] = attemptRecordWithTimeID{ + record: &driver.AttemptRecord{ + Attempt: copyAttempt(r.record.Attempt), + Event: copyEvent(r.record.Event), + }, + timeID: r.timeID, } } return result, nil }, - Cursor: pagination.Cursor[deliveryEventWithTimeID]{ - Encode: func(de deliveryEventWithTimeID) string { - return cursor.Encode(cursorResourceDelivery, cursorVersion, de.timeID) + Cursor: pagination.Cursor[attemptRecordWithTimeID]{ + Encode: func(r attemptRecordWithTimeID) string { + return cursor.Encode(cursorResourceAttempt, cursorVersion, r.timeID) }, Decode: func(c string) (string, error) { - return cursor.Decode(c, cursorResourceDelivery, cursorVersion) + return cursor.Decode(c, cursorResourceAttempt, cursorVersion) }, }, }) if err != nil { - return driver.ListDeliveryEventResponse{}, err + return driver.ListAttemptResponse{}, err } - // Extract delivery events from results - data := make([]*models.DeliveryEvent, len(res.Items)) + // Extract records from results + data := make([]*driver.AttemptRecord, len(res.Items)) for i, item := range res.Items { - data[i] = item.de + data[i] = item.record } - return driver.ListDeliveryEventResponse{ + return driver.ListAttemptResponse{ Data: data, Next: res.Next, Prev: res.Prev, @@ -329,48 +331,56 @@ func (s *memLogStore) RetrieveEvent(ctx context.Context, req driver.RetrieveEven s.mu.RLock() defer s.mu.RUnlock() - for _, de := range s.deliveryEvents { - if de.Event.ID == req.EventID { - if req.TenantID != "" && de.Event.TenantID != req.TenantID { - continue - } - if req.DestinationID != "" && de.Event.DestinationID != req.DestinationID { - continue - } - return copyEvent(&de.Event), nil - } + event := s.events[req.EventID] + if event == nil { + return nil, nil } - return nil, nil + + if req.TenantID != "" && event.TenantID != req.TenantID { + return nil, nil + } + if req.DestinationID != "" && event.DestinationID != req.DestinationID { + return nil, nil + } + return copyEvent(event), nil } -func (s *memLogStore) RetrieveDeliveryEvent(ctx context.Context, req driver.RetrieveDeliveryEventRequest) (*models.DeliveryEvent, error) { +func (s *memLogStore) RetrieveAttempt(ctx context.Context, req driver.RetrieveAttemptRequest) (*driver.AttemptRecord, error) { s.mu.RLock() defer s.mu.RUnlock() - for _, de := range s.deliveryEvents { - if de.Delivery != nil && de.Delivery.ID == req.DeliveryID { - if req.TenantID != "" && de.Event.TenantID != req.TenantID { + for _, a := range s.attempts { + if a.ID == req.AttemptID { + event := s.events[a.EventID] + if event == nil { continue } - return copyDeliveryEvent(de), nil + if req.TenantID != "" && event.TenantID != req.TenantID { + continue + } + return &driver.AttemptRecord{ + Attempt: copyAttempt(a), + Event: copyEvent(event), + }, nil } } return nil, nil } -func (s *memLogStore) matchesFilter(de *models.DeliveryEvent, req driver.ListDeliveryEventRequest) bool { - if req.TenantID != "" && de.Event.TenantID != req.TenantID { +func (s *memLogStore) matchesAttemptFilter(a *models.Attempt, event *models.Event, req driver.ListAttemptRequest) bool { + // Filter by event's tenant ID since attempts don't have tenant_id in the database + if req.TenantID != "" && event.TenantID != req.TenantID { return false } - if req.EventID != "" && de.Event.ID != req.EventID { + if req.EventID != "" && a.EventID != req.EventID { return false } if len(req.DestinationIDs) > 0 { found := false for _, destID := range req.DestinationIDs { - if de.DestinationID == destID { + if a.DestinationID == destID { found = true break } @@ -380,14 +390,14 @@ func (s *memLogStore) matchesFilter(de *models.DeliveryEvent, req driver.ListDel } } - if req.Status != "" && de.Delivery.Status != req.Status { + if req.Status != "" && a.Status != req.Status { return false } if len(req.Topics) > 0 { found := false for _, topic := range req.Topics { - if de.Event.Topic == topic { + if event.Topic == topic { found = true break } @@ -397,33 +407,22 @@ func (s *memLogStore) matchesFilter(de *models.DeliveryEvent, req driver.ListDel } } - if req.TimeFilter.GTE != nil && de.Delivery.Time.Before(*req.TimeFilter.GTE) { + if req.TimeFilter.GTE != nil && a.Time.Before(*req.TimeFilter.GTE) { return false } - if req.TimeFilter.LTE != nil && de.Delivery.Time.After(*req.TimeFilter.LTE) { + if req.TimeFilter.LTE != nil && a.Time.After(*req.TimeFilter.LTE) { return false } - if req.TimeFilter.GT != nil && !de.Delivery.Time.After(*req.TimeFilter.GT) { + if req.TimeFilter.GT != nil && !a.Time.After(*req.TimeFilter.GT) { return false } - if req.TimeFilter.LT != nil && !de.Delivery.Time.Before(*req.TimeFilter.LT) { + if req.TimeFilter.LT != nil && !a.Time.Before(*req.TimeFilter.LT) { return false } return true } -func copyDeliveryEvent(de *models.DeliveryEvent) *models.DeliveryEvent { - return &models.DeliveryEvent{ - ID: de.ID, - Attempt: de.Attempt, - DestinationID: de.DestinationID, - Event: *copyEvent(&de.Event), - Delivery: copyDelivery(de.Delivery), - Manual: de.Manual, - } -} - func copyEvent(e *models.Event) *models.Event { copied := &models.Event{ ID: e.ID, @@ -441,7 +440,7 @@ func copyEvent(e *models.Event) *models.Event { } } if e.Data != nil { - copied.Data = make(map[string]interface{}, len(e.Data)) + copied.Data = make(map[string]any, len(e.Data)) for k, v := range e.Data { copied.Data[k] = v } @@ -450,22 +449,25 @@ func copyEvent(e *models.Event) *models.Event { return copied } -func copyDelivery(d *models.Delivery) *models.Delivery { - if d == nil { +func copyAttempt(a *models.Attempt) *models.Attempt { + if a == nil { return nil } - copied := &models.Delivery{ - ID: d.ID, - EventID: d.EventID, - DestinationID: d.DestinationID, - Status: d.Status, - Time: d.Time, - Code: d.Code, - } - - if d.ResponseData != nil { - copied.ResponseData = make(map[string]interface{}, len(d.ResponseData)) - for k, v := range d.ResponseData { + copied := &models.Attempt{ + ID: a.ID, + TenantID: a.TenantID, + EventID: a.EventID, + DestinationID: a.DestinationID, + AttemptNumber: a.AttemptNumber, + Manual: a.Manual, + Status: a.Status, + Time: a.Time, + Code: a.Code, + } + + if a.ResponseData != nil { + copied.ResponseData = make(map[string]any, len(a.ResponseData)) + for k, v := range a.ResponseData { copied.ResponseData[k] = v } } diff --git a/internal/logstore/pglogstore/README.md b/internal/logstore/pglogstore/README.md index 7d76df6d..58c05e81 100644 --- a/internal/logstore/pglogstore/README.md +++ b/internal/logstore/pglogstore/README.md @@ -14,7 +14,7 @@ All tables are partitioned by time. ## Operations -### ListDeliveryEvent +### ListDelivery Query pattern: **Index → Hydrate** @@ -49,7 +49,7 @@ WHERE tenant_id = $1 AND id = $2 AND EXISTS (SELECT 1 FROM event_delivery_index WHERE event_id = $2 AND destination_id = $3) ``` -### InsertManyDeliveryEvent +### InsertMany Batch insert using `unnest()` arrays in a single transaction across all 3 tables. diff --git a/internal/logstore/pglogstore/pglogstore.go b/internal/logstore/pglogstore/pglogstore.go index 90f7319f..35feeb08 100644 --- a/internal/logstore/pglogstore/pglogstore.go +++ b/internal/logstore/pglogstore/pglogstore.go @@ -15,9 +15,9 @@ import ( ) const ( - cursorResourceEvent = "evt" - cursorResourceDelivery = "dlv" - cursorVersion = 1 + cursorResourceEvent = "evt" + cursorResourceAttempt = "att" + cursorVersion = 1 ) type logStore struct { @@ -140,7 +140,7 @@ func scanEvents(rows pgx.Rows) ([]eventWithTimeID, error) { eventTime time.Time topic string eligibleForRetry bool - data map[string]interface{} + data map[string]any metadata map[string]string timeID string ) @@ -181,13 +181,13 @@ func scanEvents(rows pgx.Rows) ([]eventWithTimeID, error) { return results, nil } -// deliveryEventWithTimeID wraps a delivery event with its time_delivery_id for cursor encoding. -type deliveryEventWithTimeID struct { - *models.DeliveryEvent - TimeDeliveryID string +// attemptRecordWithTimeID wraps an attempt record with its time_attempt_id for cursor encoding. +type attemptRecordWithTimeID struct { + *driver.AttemptRecord + TimeAttemptID string } -func (s *logStore) ListDeliveryEvent(ctx context.Context, req driver.ListDeliveryEventRequest) (driver.ListDeliveryEventResponse, error) { +func (s *logStore) ListAttempt(ctx context.Context, req driver.ListAttemptRequest) (driver.ListAttemptResponse, error) { sortOrder := req.SortOrder if sortOrder != "asc" && sortOrder != "desc" { sortOrder = "desc" @@ -198,80 +198,80 @@ func (s *logStore) ListDeliveryEvent(ctx context.Context, req driver.ListDeliver limit = 100 } - res, err := pagination.Run(ctx, pagination.Config[deliveryEventWithTimeID]{ + res, err := pagination.Run(ctx, pagination.Config[attemptRecordWithTimeID]{ Limit: limit, Order: sortOrder, Next: req.Next, Prev: req.Prev, - Fetch: func(ctx context.Context, q pagination.QueryInput) ([]deliveryEventWithTimeID, error) { - query, args := buildDeliveryQuery(req, q) + Fetch: func(ctx context.Context, q pagination.QueryInput) ([]attemptRecordWithTimeID, error) { + query, args := buildAttemptQuery(req, q) rows, err := s.db.Query(ctx, query, args...) if err != nil { return nil, fmt.Errorf("query failed: %w", err) } defer rows.Close() - return scanDeliveryEvents(rows) + return scanAttemptRecords(rows) }, - Cursor: pagination.Cursor[deliveryEventWithTimeID]{ - Encode: func(de deliveryEventWithTimeID) string { - return cursor.Encode(cursorResourceDelivery, cursorVersion, de.TimeDeliveryID) + Cursor: pagination.Cursor[attemptRecordWithTimeID]{ + Encode: func(ar attemptRecordWithTimeID) string { + return cursor.Encode(cursorResourceAttempt, cursorVersion, ar.TimeAttemptID) }, Decode: func(c string) (string, error) { - return cursor.Decode(c, cursorResourceDelivery, cursorVersion) + return cursor.Decode(c, cursorResourceAttempt, cursorVersion) }, }, }) if err != nil { - return driver.ListDeliveryEventResponse{}, err + return driver.ListAttemptResponse{}, err } - // Extract delivery events from results - data := make([]*models.DeliveryEvent, len(res.Items)) + // Extract attempt records from results + data := make([]*driver.AttemptRecord, len(res.Items)) for i, item := range res.Items { - data[i] = item.DeliveryEvent + data[i] = item.AttemptRecord } - return driver.ListDeliveryEventResponse{ + return driver.ListAttemptResponse{ Data: data, Next: res.Next, Prev: res.Prev, }, nil } -func buildDeliveryQuery(req driver.ListDeliveryEventRequest, q pagination.QueryInput) (string, []any) { - cursorCondition := fmt.Sprintf("AND ($10::text = '' OR idx.time_delivery_id %s $10::text)", q.Compare) - orderByClause := fmt.Sprintf("idx.delivery_time %s, idx.delivery_id %s", strings.ToUpper(q.SortDir), strings.ToUpper(q.SortDir)) +func buildAttemptQuery(req driver.ListAttemptRequest, q pagination.QueryInput) (string, []any) { + cursorCondition := fmt.Sprintf("AND ($10::text = '' OR idx.time_attempt_id %s $10::text)", q.Compare) + orderByClause := fmt.Sprintf("idx.attempt_time %s, idx.attempt_id %s", strings.ToUpper(q.SortDir), strings.ToUpper(q.SortDir)) query := fmt.Sprintf(` SELECT idx.event_id, - idx.delivery_id, + idx.attempt_id, idx.destination_id, idx.event_time, - idx.delivery_time, + idx.attempt_time, idx.topic, idx.status, - idx.time_delivery_id, + idx.time_attempt_id, e.tenant_id, e.eligible_for_retry, e.data, e.metadata, - d.code, - d.response_data, + a.code, + a.response_data, idx.manual, - idx.attempt - FROM event_delivery_index idx + idx.attempt_number + FROM event_attempt_index idx JOIN events e ON e.id = idx.event_id AND e.time = idx.event_time - JOIN deliveries d ON d.id = idx.delivery_id AND d.time = idx.delivery_time + JOIN attempts a ON a.id = idx.attempt_id AND a.time = idx.attempt_time WHERE ($1::text = '' OR idx.tenant_id = $1) AND ($2::text = '' OR idx.event_id = $2) AND (array_length($3::text[], 1) IS NULL OR idx.destination_id = ANY($3)) AND ($4::text = '' OR idx.status = $4) AND (array_length($5::text[], 1) IS NULL OR idx.topic = ANY($5)) - AND ($6::timestamptz IS NULL OR idx.delivery_time >= $6) - AND ($7::timestamptz IS NULL OR idx.delivery_time <= $7) - AND ($8::timestamptz IS NULL OR idx.delivery_time > $8) - AND ($9::timestamptz IS NULL OR idx.delivery_time < $9) + AND ($6::timestamptz IS NULL OR idx.attempt_time >= $6) + AND ($7::timestamptz IS NULL OR idx.attempt_time <= $7) + AND ($8::timestamptz IS NULL OR idx.attempt_time > $8) + AND ($9::timestamptz IS NULL OR idx.attempt_time < $9) %s ORDER BY %s LIMIT $11 @@ -294,37 +294,37 @@ func buildDeliveryQuery(req driver.ListDeliveryEventRequest, q pagination.QueryI return query, args } -func scanDeliveryEvents(rows pgx.Rows) ([]deliveryEventWithTimeID, error) { - var results []deliveryEventWithTimeID +func scanAttemptRecords(rows pgx.Rows) ([]attemptRecordWithTimeID, error) { + var results []attemptRecordWithTimeID for rows.Next() { var ( eventID string - deliveryID string + attemptID string destinationID string eventTime time.Time - deliveryTime time.Time + attemptTime time.Time topic string status string - timeDeliveryID string + timeAttemptID string tenantID string eligibleForRetry bool - data map[string]interface{} + data map[string]any metadata map[string]string code string - responseData map[string]interface{} + responseData map[string]any manual bool - attempt int + attemptNumber int ) if err := rows.Scan( &eventID, - &deliveryID, + &attemptID, &destinationID, &eventTime, - &deliveryTime, + &attemptTime, &topic, &status, - &timeDeliveryID, + &timeAttemptID, &tenantID, &eligibleForRetry, &data, @@ -332,18 +332,26 @@ func scanDeliveryEvents(rows pgx.Rows) ([]deliveryEventWithTimeID, error) { &code, &responseData, &manual, - &attempt, + &attemptNumber, ); err != nil { return nil, fmt.Errorf("scan failed: %w", err) } - results = append(results, deliveryEventWithTimeID{ - DeliveryEvent: &models.DeliveryEvent{ - ID: fmt.Sprintf("%s_%s", eventID, deliveryID), - DestinationID: destinationID, - Manual: manual, - Attempt: attempt, - Event: models.Event{ + results = append(results, attemptRecordWithTimeID{ + AttemptRecord: &driver.AttemptRecord{ + Attempt: &models.Attempt{ + ID: attemptID, + TenantID: tenantID, + EventID: eventID, + DestinationID: destinationID, + AttemptNumber: attemptNumber, + Manual: manual, + Status: status, + Time: attemptTime, + Code: code, + ResponseData: responseData, + }, + Event: &models.Event{ ID: eventID, TenantID: tenantID, DestinationID: destinationID, @@ -353,17 +361,8 @@ func scanDeliveryEvents(rows pgx.Rows) ([]deliveryEventWithTimeID, error) { Data: data, Metadata: metadata, }, - Delivery: &models.Delivery{ - ID: deliveryID, - EventID: eventID, - DestinationID: destinationID, - Status: status, - Time: deliveryTime, - Code: code, - ResponseData: responseData, - }, }, - TimeDeliveryID: timeDeliveryID, + TimeAttemptID: timeAttemptID, }) } @@ -376,7 +375,7 @@ func scanDeliveryEvents(rows pgx.Rows) ([]deliveryEventWithTimeID, error) { func (s *logStore) RetrieveEvent(ctx context.Context, req driver.RetrieveEventRequest) (*models.Event, error) { var query string - var args []interface{} + var args []any if req.DestinationID != "" { query = ` @@ -392,10 +391,10 @@ func (s *logStore) RetrieveEvent(ctx context.Context, req driver.RetrieveEventRe FROM events e WHERE ($1::text = '' OR e.tenant_id = $1) AND e.id = $2 AND EXISTS ( - SELECT 1 FROM event_delivery_index idx + SELECT 1 FROM event_attempt_index idx WHERE ($1::text = '' OR idx.tenant_id = $1) AND idx.event_id = $2 AND idx.destination_id = $3 )` - args = []interface{}{req.TenantID, req.EventID, req.DestinationID} + args = []any{req.TenantID, req.EventID, req.DestinationID} } else { query = ` SELECT @@ -409,7 +408,7 @@ func (s *logStore) RetrieveEvent(ctx context.Context, req driver.RetrieveEventRe e.metadata FROM events e WHERE ($1::text = '' OR e.tenant_id = $1) AND e.id = $2` - args = []interface{}{req.TenantID, req.EventID} + args = []any{req.TenantID, req.EventID} } row := s.db.QueryRow(ctx, query, args...) @@ -435,56 +434,56 @@ func (s *logStore) RetrieveEvent(ctx context.Context, req driver.RetrieveEventRe return event, nil } -func (s *logStore) RetrieveDeliveryEvent(ctx context.Context, req driver.RetrieveDeliveryEventRequest) (*models.DeliveryEvent, error) { +func (s *logStore) RetrieveAttempt(ctx context.Context, req driver.RetrieveAttemptRequest) (*driver.AttemptRecord, error) { query := ` SELECT idx.event_id, - idx.delivery_id, + idx.attempt_id, idx.destination_id, idx.event_time, - idx.delivery_time, + idx.attempt_time, idx.topic, idx.status, e.tenant_id, e.eligible_for_retry, e.data, e.metadata, - d.code, - d.response_data, + a.code, + a.response_data, idx.manual, - idx.attempt - FROM event_delivery_index idx + idx.attempt_number + FROM event_attempt_index idx JOIN events e ON e.id = idx.event_id AND e.time = idx.event_time - JOIN deliveries d ON d.id = idx.delivery_id AND d.time = idx.delivery_time - WHERE ($1::text = '' OR idx.tenant_id = $1) AND idx.delivery_id = $2 + JOIN attempts a ON a.id = idx.attempt_id AND a.time = idx.attempt_time + WHERE ($1::text = '' OR idx.tenant_id = $1) AND idx.attempt_id = $2 LIMIT 1` - row := s.db.QueryRow(ctx, query, req.TenantID, req.DeliveryID) + row := s.db.QueryRow(ctx, query, req.TenantID, req.AttemptID) var ( eventID string - deliveryID string + attemptID string destinationID string eventTime time.Time - deliveryTime time.Time + attemptTime time.Time topic string status string tenantID string eligibleForRetry bool - data map[string]interface{} + data map[string]any metadata map[string]string code string - responseData map[string]interface{} + responseData map[string]any manual bool - attempt int + attemptNumber int ) err := row.Scan( &eventID, - &deliveryID, + &attemptID, &destinationID, &eventTime, - &deliveryTime, + &attemptTime, &topic, &status, &tenantID, @@ -494,7 +493,7 @@ func (s *logStore) RetrieveDeliveryEvent(ctx context.Context, req driver.Retriev &code, &responseData, &manual, - &attempt, + &attemptNumber, ) if err == pgx.ErrNoRows { return nil, nil @@ -503,12 +502,20 @@ func (s *logStore) RetrieveDeliveryEvent(ctx context.Context, req driver.Retriev return nil, fmt.Errorf("scan failed: %w", err) } - return &models.DeliveryEvent{ - ID: fmt.Sprintf("%s_%s", eventID, deliveryID), - DestinationID: destinationID, - Manual: manual, - Attempt: attempt, - Event: models.Event{ + return &driver.AttemptRecord{ + Attempt: &models.Attempt{ + ID: attemptID, + TenantID: tenantID, + EventID: eventID, + DestinationID: destinationID, + AttemptNumber: attemptNumber, + Manual: manual, + Status: status, + Time: attemptTime, + Code: code, + ResponseData: responseData, + }, + Event: &models.Event{ ID: eventID, TenantID: tenantID, DestinationID: destinationID, @@ -518,145 +525,98 @@ func (s *logStore) RetrieveDeliveryEvent(ctx context.Context, req driver.Retriev Data: data, Metadata: metadata, }, - Delivery: &models.Delivery{ - ID: deliveryID, - EventID: eventID, - DestinationID: destinationID, - Status: status, - Time: deliveryTime, - Code: code, - ResponseData: responseData, - }, }, nil } -func (s *logStore) InsertManyDeliveryEvent(ctx context.Context, deliveryEvents []*models.DeliveryEvent) error { - if len(deliveryEvents) == 0 { +func (s *logStore) InsertMany(ctx context.Context, entries []*models.LogEntry) error { + if len(entries) == 0 { return nil } - tx, err := s.db.Begin(ctx) - if err != nil { - return err - } - defer tx.Rollback(ctx) - - events := make([]*models.Event, len(deliveryEvents)) - for i, de := range deliveryEvents { - events[i] = &de.Event + // Extract and dedupe events by ID + eventMap := make(map[string]*models.Event) + for _, entry := range entries { + eventMap[entry.Event.ID] = entry.Event } - _, err = tx.Exec(ctx, ` - INSERT INTO events (id, tenant_id, destination_id, time, topic, eligible_for_retry, data, metadata) - SELECT * FROM unnest($1::text[], $2::text[], $3::text[], $4::timestamptz[], $5::text[], $6::boolean[], $7::jsonb[], $8::jsonb[]) - ON CONFLICT (time, id) DO NOTHING - `, eventArrays(events)...) - if err != nil { - return err + events := make([]*models.Event, 0, len(eventMap)) + for _, e := range eventMap { + events = append(events, e) } - deliveries := make([]*models.Delivery, len(deliveryEvents)) - for i, de := range deliveryEvents { - if de.Delivery == nil { - // Create a pending delivery if none exists - deliveries[i] = &models.Delivery{ - ID: de.ID, - EventID: de.Event.ID, - DestinationID: de.DestinationID, - Status: "pending", - Time: time.Now(), - } - } else { - deliveries[i] = de.Delivery - } - } - _, err = tx.Exec(ctx, ` - INSERT INTO deliveries (id, event_id, destination_id, status, time, code, response_data, manual, attempt) - SELECT * FROM unnest($1::text[], $2::text[], $3::text[], $4::text[], $5::timestamptz[], $6::text[], $7::jsonb[], $8::boolean[], $9::integer[]) - ON CONFLICT (time, id) DO UPDATE SET - status = EXCLUDED.status, - code = EXCLUDED.code, - response_data = EXCLUDED.response_data - `, deliveryArrays(deliveries, deliveryEvents)...) - if err != nil { - return err + // Extract attempts + attempts := make([]*models.Attempt, 0, len(entries)) + for _, entry := range entries { + attempts = append(attempts, entry.Attempt) } - _, err = tx.Exec(ctx, ` - INSERT INTO event_delivery_index ( - event_id, delivery_id, tenant_id, destination_id, - event_time, delivery_time, topic, status, manual, attempt - ) - SELECT * FROM unnest( - $1::text[], $2::text[], $3::text[], $4::text[], - $5::timestamptz[], $6::timestamptz[], $7::text[], $8::text[], - $9::boolean[], $10::integer[] - ) - ON CONFLICT (delivery_time, event_id, delivery_id) DO UPDATE SET - status = EXCLUDED.status - `, eventDeliveryIndexArrays(deliveryEvents)...) + tx, err := s.db.Begin(ctx) if err != nil { return err } + defer tx.Rollback(ctx) - return tx.Commit(ctx) -} + if len(events) > 0 { + _, err = tx.Exec(ctx, ` + INSERT INTO events (id, tenant_id, destination_id, time, topic, eligible_for_retry, data, metadata) + SELECT * FROM unnest($1::text[], $2::text[], $3::text[], $4::timestamptz[], $5::text[], $6::boolean[], $7::jsonb[], $8::jsonb[]) + ON CONFLICT (time, id) DO NOTHING + `, eventArrays(events)...) + if err != nil { + return err + } + } -func eventDeliveryIndexArrays(deliveryEvents []*models.DeliveryEvent) []interface{} { - eventIDs := make([]string, len(deliveryEvents)) - deliveryIDs := make([]string, len(deliveryEvents)) - tenantIDs := make([]string, len(deliveryEvents)) - destinationIDs := make([]string, len(deliveryEvents)) - eventTimes := make([]time.Time, len(deliveryEvents)) - deliveryTimes := make([]time.Time, len(deliveryEvents)) - topics := make([]string, len(deliveryEvents)) - statuses := make([]string, len(deliveryEvents)) - manuals := make([]bool, len(deliveryEvents)) - attempts := make([]int, len(deliveryEvents)) - - for i, de := range deliveryEvents { - eventIDs[i] = de.Event.ID - if de.Delivery != nil { - deliveryIDs[i] = de.Delivery.ID - } else { - deliveryIDs[i] = de.ID + if len(attempts) > 0 { + _, err = tx.Exec(ctx, ` + INSERT INTO attempts (id, event_id, destination_id, status, time, code, response_data, manual, attempt_number) + SELECT * FROM unnest($1::text[], $2::text[], $3::text[], $4::text[], $5::timestamptz[], $6::text[], $7::jsonb[], $8::boolean[], $9::integer[]) + ON CONFLICT (time, id) DO UPDATE SET + status = EXCLUDED.status, + code = EXCLUDED.code, + response_data = EXCLUDED.response_data + `, attemptArrays(attempts)...) + if err != nil { + return err } - tenantIDs[i] = de.Event.TenantID - destinationIDs[i] = de.DestinationID - eventTimes[i] = de.Event.Time - if de.Delivery != nil { - deliveryTimes[i] = de.Delivery.Time - statuses[i] = de.Delivery.Status - } else { - deliveryTimes[i] = time.Now() - statuses[i] = "pending" + + _, err = tx.Exec(ctx, ` + INSERT INTO event_attempt_index ( + event_id, attempt_id, tenant_id, destination_id, + event_time, attempt_time, topic, status, manual, attempt_number + ) + SELECT + a.event_id, + a.id, + e.tenant_id, + a.destination_id, + e.time, + a.time, + e.topic, + a.status, + a.manual, + a.attempt_number + FROM unnest($1::text[], $2::text[], $3::text[], $4::text[], $5::timestamptz[], $6::text[], $7::jsonb[], $8::boolean[], $9::integer[]) + AS a(id, event_id, destination_id, status, time, code, response_data, manual, attempt_number) + JOIN events e ON e.id = a.event_id + ON CONFLICT (attempt_time, event_id, attempt_id) DO UPDATE SET + status = EXCLUDED.status + `, attemptArrays(attempts)...) + if err != nil { + return err } - topics[i] = de.Event.Topic - manuals[i] = de.Manual - attempts[i] = de.Attempt } - return []interface{}{ - eventIDs, - deliveryIDs, - tenantIDs, - destinationIDs, - eventTimes, - deliveryTimes, - topics, - statuses, - manuals, - attempts, - } + return tx.Commit(ctx) } -func eventArrays(events []*models.Event) []interface{} { +func eventArrays(events []*models.Event) []any { ids := make([]string, len(events)) tenantIDs := make([]string, len(events)) destinationIDs := make([]string, len(events)) times := make([]time.Time, len(events)) topics := make([]string, len(events)) eligibleForRetries := make([]bool, len(events)) - datas := make([]map[string]interface{}, len(events)) + datas := make([]map[string]any, len(events)) metadatas := make([]map[string]string, len(events)) for i, e := range events { @@ -670,7 +630,7 @@ func eventArrays(events []*models.Event) []interface{} { metadatas[i] = e.Metadata } - return []interface{}{ + return []any{ ids, tenantIDs, destinationIDs, @@ -682,30 +642,30 @@ func eventArrays(events []*models.Event) []interface{} { } } -func deliveryArrays(deliveries []*models.Delivery, deliveryEvents []*models.DeliveryEvent) []interface{} { - ids := make([]string, len(deliveries)) - eventIDs := make([]string, len(deliveries)) - destinationIDs := make([]string, len(deliveries)) - statuses := make([]string, len(deliveries)) - times := make([]time.Time, len(deliveries)) - codes := make([]string, len(deliveries)) - responseDatas := make([]map[string]interface{}, len(deliveries)) - manuals := make([]bool, len(deliveries)) - attempts := make([]int, len(deliveries)) - - for i, d := range deliveries { - ids[i] = d.ID - eventIDs[i] = d.EventID - destinationIDs[i] = d.DestinationID - statuses[i] = d.Status - times[i] = d.Time - codes[i] = d.Code - responseDatas[i] = d.ResponseData - manuals[i] = deliveryEvents[i].Manual - attempts[i] = deliveryEvents[i].Attempt - } - - return []interface{}{ +func attemptArrays(attempts []*models.Attempt) []any { + ids := make([]string, len(attempts)) + eventIDs := make([]string, len(attempts)) + destinationIDs := make([]string, len(attempts)) + statuses := make([]string, len(attempts)) + times := make([]time.Time, len(attempts)) + codes := make([]string, len(attempts)) + responseDatas := make([]map[string]any, len(attempts)) + manuals := make([]bool, len(attempts)) + attemptNumbers := make([]int, len(attempts)) + + for i, a := range attempts { + ids[i] = a.ID + eventIDs[i] = a.EventID + destinationIDs[i] = a.DestinationID + statuses[i] = a.Status + times[i] = a.Time + codes[i] = a.Code + responseDatas[i] = a.ResponseData + manuals[i] = a.Manual + attemptNumbers[i] = a.AttemptNumber + } + + return []any{ ids, eventIDs, destinationIDs, @@ -714,6 +674,6 @@ func deliveryArrays(deliveries []*models.Delivery, deliveryEvents []*models.Deli codes, responseDatas, manuals, - attempts, + attemptNumbers, } } diff --git a/internal/migrator/migrations/clickhouse/000001_init.down.sql b/internal/migrator/migrations/clickhouse/000001_init.down.sql index d0465d4e..70b20422 100644 --- a/internal/migrator/migrations/clickhouse/000001_init.down.sql +++ b/internal/migrator/migrations/clickhouse/000001_init.down.sql @@ -1,2 +1,2 @@ -DROP TABLE IF EXISTS {deployment_prefix}deliveries; +DROP TABLE IF EXISTS {deployment_prefix}attempts; DROP TABLE IF EXISTS {deployment_prefix}events; diff --git a/internal/migrator/migrations/clickhouse/000001_init.up.sql b/internal/migrator/migrations/clickhouse/000001_init.up.sql index 13f18332..ac077085 100644 --- a/internal/migrator/migrations/clickhouse/000001_init.up.sql +++ b/internal/migrator/migrations/clickhouse/000001_init.up.sql @@ -21,11 +21,11 @@ CREATE TABLE IF NOT EXISTS {deployment_prefix}events ( PARTITION BY toYYYYMM(event_time) ORDER BY (event_time, event_id); --- Deliveries table for delivery queries --- Each row represents a delivery attempt for an event +-- Attempts table for attempt queries +-- Each row represents an attempt for an event -- Stateless queries: no GROUP BY, no aggregation, direct row access -CREATE TABLE IF NOT EXISTS {deployment_prefix}deliveries ( +CREATE TABLE IF NOT EXISTS {deployment_prefix}attempts ( -- Event fields event_id String, tenant_id String, @@ -36,23 +36,22 @@ CREATE TABLE IF NOT EXISTS {deployment_prefix}deliveries ( metadata String, -- JSON serialized data String, -- JSON serialized - -- Delivery fields - delivery_id String, - delivery_event_id String, + -- Attempt fields + attempt_id String, status String, -- 'success', 'failed' - delivery_time DateTime64(3), + attempt_time DateTime64(3), code String, response_data String, -- JSON serialized manual Bool DEFAULT false, - attempt UInt32 DEFAULT 0, + attempt_number UInt32 DEFAULT 0, -- Indexes for filtering (bloom filters help skip granules) INDEX idx_tenant_id tenant_id TYPE bloom_filter GRANULARITY 1, INDEX idx_destination_id destination_id TYPE bloom_filter GRANULARITY 1, INDEX idx_event_id event_id TYPE bloom_filter GRANULARITY 1, - INDEX idx_delivery_id delivery_id TYPE bloom_filter GRANULARITY 1, + INDEX idx_attempt_id attempt_id TYPE bloom_filter GRANULARITY 1, INDEX idx_topic topic TYPE bloom_filter GRANULARITY 1, INDEX idx_status status TYPE set(100) GRANULARITY 1 ) ENGINE = ReplacingMergeTree -PARTITION BY toYYYYMM(delivery_time) -ORDER BY (delivery_time, delivery_id); +PARTITION BY toYYYYMM(attempt_time) +ORDER BY (attempt_time, attempt_id); diff --git a/internal/migrator/migrations/postgres/000001_init.down.sql b/internal/migrator/migrations/postgres/000001_init.down.sql index f55d529e..e0dca179 100644 --- a/internal/migrator/migrations/postgres/000001_init.down.sql +++ b/internal/migrator/migrations/postgres/000001_init.down.sql @@ -3,4 +3,4 @@ BEGIN; DROP TABLE IF EXISTS events CASCADE; DROP TABLE IF EXISTS deliveries CASCADE; -COMMIT; \ No newline at end of file +COMMIT; diff --git a/internal/migrator/migrations/postgres/000001_init.up.sql b/internal/migrator/migrations/postgres/000001_init.up.sql index f9e5234c..884dc93b 100644 --- a/internal/migrator/migrations/postgres/000001_init.up.sql +++ b/internal/migrator/migrations/postgres/000001_init.up.sql @@ -43,4 +43,4 @@ CREATE TABLE deliveries_default PARTITION OF deliveries DEFAULT; CREATE INDEX ON deliveries (event_id); CREATE INDEX ON deliveries (event_id, status); -COMMIT; \ No newline at end of file +COMMIT; diff --git a/internal/migrator/migrations/postgres/000002_delivery_response.down.sql b/internal/migrator/migrations/postgres/000002_delivery_response.down.sql index 0ecc7d93..4699a411 100644 --- a/internal/migrator/migrations/postgres/000002_delivery_response.down.sql +++ b/internal/migrator/migrations/postgres/000002_delivery_response.down.sql @@ -3,4 +3,4 @@ BEGIN; ALTER TABLE deliveries DROP COLUMN code, DROP COLUMN response_data; -COMMIT; \ No newline at end of file +COMMIT; diff --git a/internal/migrator/migrations/postgres/000002_delivery_response.up.sql b/internal/migrator/migrations/postgres/000002_delivery_response.up.sql index 24e0d728..7a224676 100644 --- a/internal/migrator/migrations/postgres/000002_delivery_response.up.sql +++ b/internal/migrator/migrations/postgres/000002_delivery_response.up.sql @@ -4,4 +4,4 @@ ALTER TABLE deliveries ADD COLUMN code TEXT, ADD COLUMN response_data JSONB; -COMMIT; \ No newline at end of file +COMMIT; diff --git a/internal/migrator/migrations/postgres/000003_event_delivery_index.down.sql b/internal/migrator/migrations/postgres/000003_event_delivery_index.down.sql index 93623032..2218227c 100644 --- a/internal/migrator/migrations/postgres/000003_event_delivery_index.down.sql +++ b/internal/migrator/migrations/postgres/000003_event_delivery_index.down.sql @@ -3,4 +3,4 @@ BEGIN; DROP TABLE IF EXISTS event_delivery_index_default; DROP TABLE IF EXISTS event_delivery_index CASCADE; -COMMIT; \ No newline at end of file +COMMIT; diff --git a/internal/migrator/migrations/postgres/000003_event_delivery_index.up.sql b/internal/migrator/migrations/postgres/000003_event_delivery_index.up.sql index 23fc772f..bba2ad01 100644 --- a/internal/migrator/migrations/postgres/000003_event_delivery_index.up.sql +++ b/internal/migrator/migrations/postgres/000003_event_delivery_index.up.sql @@ -50,4 +50,4 @@ CREATE INDEX IF NOT EXISTS idx_event_delivery_index_main ON event_delivery_index time_delivery_id ); -COMMIT; \ No newline at end of file +COMMIT; diff --git a/internal/migrator/migrations/postgres/000005_rename_delivery_to_attempt.down.sql b/internal/migrator/migrations/postgres/000005_rename_delivery_to_attempt.down.sql new file mode 100644 index 00000000..21f8575d --- /dev/null +++ b/internal/migrator/migrations/postgres/000005_rename_delivery_to_attempt.down.sql @@ -0,0 +1,48 @@ +BEGIN; + +-- Drop new index and restore old one +DROP INDEX IF EXISTS idx_event_attempt_index_main; + +-- Restore generated column with old name +ALTER TABLE event_attempt_index DROP COLUMN time_attempt_id; +ALTER TABLE event_attempt_index ADD COLUMN time_delivery_id text GENERATED ALWAYS AS ( + LPAD( + CAST( + EXTRACT( + EPOCH + FROM attempt_time AT TIME ZONE 'UTC' + ) AS BIGINT + )::text, + 10, + '0' + ) || '_' || attempt_id +) STORED; + +-- Rename columns back in event_attempt_index +ALTER TABLE event_attempt_index RENAME COLUMN attempt_number TO attempt; +ALTER TABLE event_attempt_index RENAME COLUMN attempt_time TO delivery_time; +ALTER TABLE event_attempt_index RENAME COLUMN attempt_id TO delivery_id; + +-- Rename tables back +ALTER TABLE event_attempt_index RENAME TO event_delivery_index; +ALTER TABLE event_attempt_index_default RENAME TO event_delivery_index_default; + +-- Rename column back in attempts: attempt_number -> attempt +ALTER TABLE attempts RENAME COLUMN attempt_number TO attempt; + +ALTER TABLE attempts RENAME TO deliveries; +ALTER TABLE attempts_default RENAME TO deliveries_default; + +-- Recreate old index +CREATE INDEX IF NOT EXISTS idx_event_delivery_index_main ON event_delivery_index( + tenant_id, + destination_id, + topic, + status, + event_time DESC, + delivery_time DESC, + time_event_id, + time_delivery_id +); + +COMMIT; diff --git a/internal/migrator/migrations/postgres/000005_rename_delivery_to_attempt.up.sql b/internal/migrator/migrations/postgres/000005_rename_delivery_to_attempt.up.sql new file mode 100644 index 00000000..ea143623 --- /dev/null +++ b/internal/migrator/migrations/postgres/000005_rename_delivery_to_attempt.up.sql @@ -0,0 +1,47 @@ +BEGIN; + +-- Rename deliveries table to attempts +ALTER TABLE deliveries RENAME TO attempts; +ALTER TABLE deliveries_default RENAME TO attempts_default; + +-- Rename column in attempts: attempt -> attempt_number +ALTER TABLE attempts RENAME COLUMN attempt TO attempt_number; + +-- Rename event_delivery_index table to event_attempt_index +ALTER TABLE event_delivery_index RENAME TO event_attempt_index; +ALTER TABLE event_delivery_index_default RENAME TO event_attempt_index_default; + +-- Rename columns in event_attempt_index +ALTER TABLE event_attempt_index RENAME COLUMN delivery_id TO attempt_id; +ALTER TABLE event_attempt_index RENAME COLUMN delivery_time TO attempt_time; +ALTER TABLE event_attempt_index RENAME COLUMN attempt TO attempt_number; + +-- Drop and recreate generated column with new name +ALTER TABLE event_attempt_index DROP COLUMN time_delivery_id; +ALTER TABLE event_attempt_index ADD COLUMN time_attempt_id text GENERATED ALWAYS AS ( + LPAD( + CAST( + EXTRACT( + EPOCH + FROM attempt_time AT TIME ZONE 'UTC' + ) AS BIGINT + )::text, + 10, + '0' + ) || '_' || attempt_id +) STORED; + +-- Drop old index and create new one with updated column names +DROP INDEX IF EXISTS idx_event_delivery_index_main; +CREATE INDEX IF NOT EXISTS idx_event_attempt_index_main ON event_attempt_index( + tenant_id, + destination_id, + topic, + status, + event_time DESC, + attempt_time DESC, + time_event_id, + time_attempt_id +); + +COMMIT; diff --git a/internal/migrator/migrator_test.go b/internal/migrator/migrator_test.go index 51762911..8edeb651 100644 --- a/internal/migrator/migrator_test.go +++ b/internal/migrator/migrator_test.go @@ -342,9 +342,9 @@ func TestMigrator_DeploymentID_TableNaming(t *testing.T) { assert.Equal(t, uint64(1), count, "testdeploy_events table should exist") err = chDB.QueryRow(ctx, "SELECT count() FROM system.tables WHERE database = ? AND name = ?", - chConfig.Database, "testdeploy_deliveries").Scan(&count) + chConfig.Database, "testdeploy_attempts").Scan(&count) require.NoError(t, err) - assert.Equal(t, uint64(1), count, "testdeploy_deliveries table should exist") + assert.Equal(t, uint64(1), count, "testdeploy_attempts table should exist") } // TestMigrator_DeploymentID_Isolation tests that multiple deployments are isolated. @@ -390,8 +390,8 @@ func TestMigrator_DeploymentID_Isolation(t *testing.T) { defer chDB.Close() tables := []string{ - "deploy_a_events", "deploy_a_deliveries", - "deploy_b_events", "deploy_b_deliveries", + "deploy_a_events", "deploy_a_attempts", + "deploy_b_events", "deploy_b_attempts", } for _, table := range tables { var count uint64 @@ -466,9 +466,9 @@ func TestMigrator_NoDeploymentID_DefaultTables(t *testing.T) { assert.Equal(t, uint64(1), count, "events table should exist") err = chDB.QueryRow(ctx, "SELECT count() FROM system.tables WHERE database = ? AND name = ?", - chConfig.Database, "deliveries").Scan(&count) + chConfig.Database, "attempts").Scan(&count) require.NoError(t, err) - assert.Equal(t, uint64(1), count, "deliveries table should exist") + assert.Equal(t, uint64(1), count, "attempts table should exist") } func setupClickHouseConfig(t *testing.T) clickhouse.ClickHouseConfig { diff --git a/internal/models/event.go b/internal/models/event.go index 73c0ea85..f6f6e3b0 100644 --- a/internal/models/event.go +++ b/internal/models/event.go @@ -6,7 +6,6 @@ import ( "fmt" "time" - "github.com/hookdeck/outpost/internal/idgen" "github.com/hookdeck/outpost/internal/mqs" ) @@ -67,75 +66,105 @@ func (e *Event) ToMessage() (*mqs.Message, error) { return &mqs.Message{Body: data}, nil } -type DeliveryEventTelemetry struct { +type DeliveryTelemetry struct { TraceID string SpanID string } -type DeliveryEvent struct { - ID string - Attempt int - DestinationID string - Event Event - Delivery *Delivery - Telemetry *DeliveryEventTelemetry - Manual bool // Indicates if this is a manual retry +// DeliveryTask represents a task to deliver an event to a destination. +// This is a message type (no ID) used by: publishmq -> deliverymq, retry -> deliverymq +type DeliveryTask struct { + Event Event `json:"event"` + DestinationID string `json:"destination_id"` + Attempt int `json:"attempt"` + Manual bool `json:"manual"` + Telemetry *DeliveryTelemetry `json:"telemetry,omitempty"` } -var _ mqs.IncomingMessage = &DeliveryEvent{} +var _ mqs.IncomingMessage = &DeliveryTask{} -func (e *DeliveryEvent) FromMessage(msg *mqs.Message) error { - return json.Unmarshal(msg.Body, e) +func (t *DeliveryTask) FromMessage(msg *mqs.Message) error { + return json.Unmarshal(msg.Body, t) } -func (e *DeliveryEvent) ToMessage() (*mqs.Message, error) { - data, err := json.Marshal(e) +func (t *DeliveryTask) ToMessage() (*mqs.Message, error) { + data, err := json.Marshal(t) if err != nil { return nil, err } return &mqs.Message{Body: data}, nil } -// GetRetryID returns the ID used for scheduling retries. -// -// We use Event.ID + DestinationID (not DeliveryEvent.ID) because: -// 1. Multiple destinations: The same event can be delivered to multiple destinations. -// Each needs its own retry, so we include DestinationID to avoid collisions. -// 2. Manual retry cancellation: When a manual retry succeeds, it must cancel any -// pending automatic retry. Manual retries create a NEW DeliveryEvent with a NEW ID, -// but share the same Event.ID + DestinationID. This allows Cancel() to find and -// remove the pending automatic retry. -func (e *DeliveryEvent) GetRetryID() string { - return e.Event.ID + ":" + e.DestinationID -} - -func NewDeliveryEvent(event Event, destinationID string) DeliveryEvent { - return DeliveryEvent{ - ID: idgen.DeliveryEvent(), - DestinationID: destinationID, +// IdempotencyKey returns the key used for idempotency checks. +// Uses Event.ID + DestinationID + Manual flag. +// Manual retries get a different key so they can bypass idempotency of failed automatic deliveries. +func (t *DeliveryTask) IdempotencyKey() string { + if t.Manual { + return t.Event.ID + ":" + t.DestinationID + ":manual" + } + return t.Event.ID + ":" + t.DestinationID +} + +// RetryID returns the ID used for scheduling and canceling retries. +// Uses event_id:destination_id to allow manual retries to cancel pending automatic retries. +func RetryID(eventID, destinationID string) string { + return eventID + ":" + destinationID +} + +// NewDeliveryTask creates a new DeliveryTask for an event and destination. +func NewDeliveryTask(event Event, destinationID string) DeliveryTask { + return DeliveryTask{ Event: event, + DestinationID: destinationID, Attempt: 0, } } -func NewManualDeliveryEvent(event Event, destinationID string) DeliveryEvent { - deliveryEvent := NewDeliveryEvent(event, destinationID) - deliveryEvent.Manual = true - return deliveryEvent +// NewManualDeliveryTask creates a new DeliveryTask for a manual retry. +func NewManualDeliveryTask(event Event, destinationID string) DeliveryTask { + task := NewDeliveryTask(event, destinationID) + task.Manual = true + return task } const ( - DeliveryStatusSuccess = "success" - DeliveryStatusFailed = "failed" + AttemptStatusSuccess = "success" + AttemptStatusFailed = "failed" ) -type Delivery struct { - ID string `json:"id"` - DeliveryEventID string `json:"delivery_event_id"` - EventID string `json:"event_id"` - DestinationID string `json:"destination_id"` - Status string `json:"status"` - Time time.Time `json:"time"` - Code string `json:"code"` - ResponseData map[string]interface{} `json:"response_data"` +// LogEntry represents a message for the log queue. +// +// IMPORTANT: Both Event and Attempt are REQUIRED. The logstore requires both +// to exist for proper data consistency. The logmq consumer validates this +// requirement and rejects entries missing either field. +type LogEntry struct { + Event *Event `json:"event"` + Attempt *Attempt `json:"attempt"` +} + +var _ mqs.IncomingMessage = &LogEntry{} + +func (e *LogEntry) FromMessage(msg *mqs.Message) error { + return json.Unmarshal(msg.Body, e) +} + +func (e *LogEntry) ToMessage() (*mqs.Message, error) { + data, err := json.Marshal(e) + if err != nil { + return nil, err + } + return &mqs.Message{Body: data}, nil +} + +type Attempt struct { + ID string `json:"id"` + TenantID string `json:"tenant_id"` + EventID string `json:"event_id"` + DestinationID string `json:"destination_id"` + AttemptNumber int `json:"attempt_number"` + Manual bool `json:"manual"` + Status string `json:"status"` + Time time.Time `json:"time"` + Code string `json:"code"` + ResponseData map[string]interface{} `json:"response_data"` } diff --git a/internal/mqinfra/awssqs.go b/internal/mqinfra/awssqs.go index 6c6b0d65..4e990397 100644 --- a/internal/mqinfra/awssqs.go +++ b/internal/mqinfra/awssqs.go @@ -30,7 +30,6 @@ func (infra *infraAWSSQS) Exist(ctx context.Context) (bool, error) { return false, err } - // Check if main queue exists _, err = awsutil.RetrieveQueueURL(ctx, sqsClient, infra.cfg.AWSSQS.Topic) if err != nil { var apiErr smithy.APIError @@ -45,7 +44,6 @@ func (infra *infraAWSSQS) Exist(ctx context.Context) (bool, error) { return false, err } - // Check if DLQ exists dlqName := infra.cfg.AWSSQS.Topic + "-dlq" _, err = awsutil.RetrieveQueueURL(ctx, sqsClient, dlqName) if err != nil { diff --git a/internal/mqinfra/azureservicebus.go b/internal/mqinfra/azureservicebus.go index 5aff2c28..61abd331 100644 --- a/internal/mqinfra/azureservicebus.go +++ b/internal/mqinfra/azureservicebus.go @@ -25,7 +25,6 @@ func (infra *infraAzureServiceBus) Exist(ctx context.Context) (bool, error) { return true, nil } - // Create credential for authentication cred, err := azidentity.NewClientSecretCredential( cfg.TenantID, cfg.ClientID, @@ -36,7 +35,6 @@ func (infra *infraAzureServiceBus) Exist(ctx context.Context) (bool, error) { return false, fmt.Errorf("failed to create credential: %w", err) } - // Create clients for topic and subscription management topicClient, err := armservicebus.NewTopicsClient(cfg.SubscriptionID, cred, nil) if err != nil { return false, fmt.Errorf("failed to create topic client: %w", err) @@ -47,7 +45,6 @@ func (infra *infraAzureServiceBus) Exist(ctx context.Context) (bool, error) { return false, fmt.Errorf("failed to create subscription client: %w", err) } - // Check if topic exists _, err = topicClient.Get(ctx, cfg.ResourceGroup, cfg.Namespace, cfg.Topic, nil) if err != nil { if isNotFoundError(err) { @@ -56,7 +53,6 @@ func (infra *infraAzureServiceBus) Exist(ctx context.Context) (bool, error) { return false, fmt.Errorf("failed to check topic existence: %w", err) } - // Check if subscription exists _, err = subClient.Get(ctx, cfg.ResourceGroup, cfg.Namespace, cfg.Topic, cfg.Subscription, nil) if err != nil { if isNotFoundError(err) { @@ -79,7 +75,6 @@ func (infra *infraAzureServiceBus) Declare(ctx context.Context) error { return nil } - // Create credential for authentication cred, err := azidentity.NewClientSecretCredential( cfg.TenantID, cfg.ClientID, @@ -90,7 +85,6 @@ func (infra *infraAzureServiceBus) Declare(ctx context.Context) error { return fmt.Errorf("failed to create credential: %w", err) } - // Create clients for topic and subscription management topicClient, err := armservicebus.NewTopicsClient(cfg.SubscriptionID, cred, nil) if err != nil { return fmt.Errorf("failed to create topic client: %w", err) @@ -101,7 +95,6 @@ func (infra *infraAzureServiceBus) Declare(ctx context.Context) error { return fmt.Errorf("failed to create subscription client: %w", err) } - // Declare main topic (upsert) topicName := cfg.Topic err = infra.declareTopic(ctx, topicClient, cfg.ResourceGroup, cfg.Namespace, topicName) if err != nil { @@ -158,7 +151,6 @@ func (infra *infraAzureServiceBus) TearDown(ctx context.Context) error { return nil } - // Create credential for authentication cred, err := azidentity.NewClientSecretCredential( cfg.TenantID, cfg.ClientID, @@ -169,7 +161,6 @@ func (infra *infraAzureServiceBus) TearDown(ctx context.Context) error { return fmt.Errorf("failed to create credential: %w", err) } - // Create clients for topic and subscription management topicClient, err := armservicebus.NewTopicsClient(cfg.SubscriptionID, cred, nil) if err != nil { return fmt.Errorf("failed to create topic client: %w", err) @@ -182,13 +173,11 @@ func (infra *infraAzureServiceBus) TearDown(ctx context.Context) error { topicName := cfg.Topic - // Delete main subscription err = infra.deleteSubscription(ctx, subClient, cfg.ResourceGroup, cfg.Namespace, topicName, cfg.Subscription) if err != nil { return fmt.Errorf("failed to delete subscription: %w", err) } - // Delete main topic err = infra.deleteTopic(ctx, topicClient, cfg.ResourceGroup, cfg.Namespace, topicName) if err != nil { return fmt.Errorf("failed to delete topic: %w", err) diff --git a/internal/mqinfra/gcppubsub.go b/internal/mqinfra/gcppubsub.go index 29a238c0..0be4aab9 100644 --- a/internal/mqinfra/gcppubsub.go +++ b/internal/mqinfra/gcppubsub.go @@ -21,20 +21,17 @@ func (infra *infraGCPPubSub) Exist(ctx context.Context) (bool, error) { return false, errors.New("failed assertion: cfg.GCPPubSub != nil") // IMPOSSIBLE } - // Create client options var opts []option.ClientOption if infra.cfg.GCPPubSub.ServiceAccountCredentials != "" { opts = append(opts, option.WithCredentialsJSON([]byte(infra.cfg.GCPPubSub.ServiceAccountCredentials))) } - // Create client client, err := pubsub.NewClient(ctx, infra.cfg.GCPPubSub.ProjectID, opts...) if err != nil { return false, fmt.Errorf("failed to create pubsub client: %w", err) } defer client.Close() - // Check if main topic exists topicID := infra.cfg.GCPPubSub.TopicID topic := client.Topic(topicID) topicExists, err := topic.Exists(ctx) @@ -45,7 +42,6 @@ func (infra *infraGCPPubSub) Exist(ctx context.Context) (bool, error) { return false, nil } - // Check if DLQ topic exists dlqTopicID := topicID + "-dlq" dlqTopic := client.Topic(dlqTopicID) dlqTopicExists, err := dlqTopic.Exists(ctx) @@ -56,7 +52,6 @@ func (infra *infraGCPPubSub) Exist(ctx context.Context) (bool, error) { return false, nil } - // Check if DLQ subscription exists dlqSubID := dlqTopicID + "-sub" dlqSub := client.Subscription(dlqSubID) dlqSubExists, err := dlqSub.Exists(ctx) @@ -67,7 +62,6 @@ func (infra *infraGCPPubSub) Exist(ctx context.Context) (bool, error) { return false, nil } - // Check if main subscription exists subID := infra.cfg.GCPPubSub.SubscriptionID sub := client.Subscription(subID) subExists, err := sub.Exists(ctx) @@ -86,20 +80,17 @@ func (infra *infraGCPPubSub) Declare(ctx context.Context) error { return errors.New("failed assertion: cfg.GCPPubSub != nil") // IMPOSSIBLE } - // Create client options var opts []option.ClientOption if infra.cfg.GCPPubSub.ServiceAccountCredentials != "" { opts = append(opts, option.WithCredentialsJSON([]byte(infra.cfg.GCPPubSub.ServiceAccountCredentials))) } - // Create client client, err := pubsub.NewClient(ctx, infra.cfg.GCPPubSub.ProjectID, opts...) if err != nil { return fmt.Errorf("failed to create pubsub client: %w", err) } defer client.Close() - // Create topic (if not exists) topicID := infra.cfg.GCPPubSub.TopicID topic := client.Topic(topicID) topicExists, err := topic.Exists(ctx) @@ -119,7 +110,6 @@ func (infra *infraGCPPubSub) Declare(ctx context.Context) error { } } - // Create DLQ topic (if not exists) dlqTopicID := topicID + "-dlq" dlqTopic := client.Topic(dlqTopicID) dlqTopicExists, err := dlqTopic.Exists(ctx) @@ -139,7 +129,6 @@ func (infra *infraGCPPubSub) Declare(ctx context.Context) error { } } - // Create DLQ subscription (if not exists) dlqSubID := dlqTopicID + "-sub" dlqSub := client.Subscription(dlqSubID) dlqSubExists, err := dlqSub.Exists(ctx) @@ -212,20 +201,17 @@ func (infra *infraGCPPubSub) TearDown(ctx context.Context) error { return errors.New("failed assertion: cfg.GCPPubSub != nil") // IMPOSSIBLE } - // Create client options var opts []option.ClientOption if infra.cfg.GCPPubSub.ServiceAccountCredentials != "" { opts = append(opts, option.WithCredentialsJSON([]byte(infra.cfg.GCPPubSub.ServiceAccountCredentials))) } - // Create client client, err := pubsub.NewClient(ctx, infra.cfg.GCPPubSub.ProjectID, opts...) if err != nil { return fmt.Errorf("failed to create pubsub client: %w", err) } defer client.Close() - // Delete main subscription subID := infra.cfg.GCPPubSub.SubscriptionID sub := client.Subscription(subID) subExists, err := sub.Exists(ctx) @@ -238,7 +224,6 @@ func (infra *infraGCPPubSub) TearDown(ctx context.Context) error { } } - // Delete DLQ subscription dlqTopicID := infra.cfg.GCPPubSub.TopicID + "-dlq" dlqSubID := dlqTopicID + "-sub" dlqSub := client.Subscription(dlqSubID) @@ -252,7 +237,6 @@ func (infra *infraGCPPubSub) TearDown(ctx context.Context) error { } } - // Delete main topic topicID := infra.cfg.GCPPubSub.TopicID topic := client.Topic(topicID) topicExists, err := topic.Exists(ctx) @@ -265,7 +249,6 @@ func (infra *infraGCPPubSub) TearDown(ctx context.Context) error { } } - // Delete DLQ topic dlqTopic := client.Topic(dlqTopicID) dlqTopicExists, err := dlqTopic.Exists(ctx) if err != nil { diff --git a/internal/mqinfra/rabbitmq.go b/internal/mqinfra/rabbitmq.go index e5463a74..49a86dbc 100644 --- a/internal/mqinfra/rabbitmq.go +++ b/internal/mqinfra/rabbitmq.go @@ -29,7 +29,6 @@ func (infra *infraRabbitMQ) Exist(ctx context.Context) (bool, error) { dlq := infra.cfg.RabbitMQ.Queue + ".dlq" - // Check if exchange exists using passive declare if err := ch.ExchangeDeclarePassive( infra.cfg.RabbitMQ.Exchange, // name "topic", // type @@ -46,7 +45,6 @@ func (infra *infraRabbitMQ) Exist(ctx context.Context) (bool, error) { return false, err } - // Check if main queue exists using passive declare if _, err := ch.QueueDeclarePassive( infra.cfg.RabbitMQ.Queue, // name true, // durable @@ -62,7 +60,6 @@ func (infra *infraRabbitMQ) Exist(ctx context.Context) (bool, error) { return false, err } - // Check if DLQ exists using passive declare if _, err := ch.QueueDeclarePassive( dlq, // name true, // durable @@ -99,7 +96,6 @@ func (infra *infraRabbitMQ) Declare(ctx context.Context) error { dlq := infra.cfg.RabbitMQ.Queue + ".dlq" - // Declare target exchange & queue if err := ch.ExchangeDeclare( infra.cfg.RabbitMQ.Exchange, // name "topic", // type @@ -136,7 +132,6 @@ func (infra *infraRabbitMQ) Declare(ctx context.Context) error { return err } - // Declare dead-letter queue if _, err := ch.QueueDeclare( dlq, // name true, // durable diff --git a/internal/portal/src/common/RetryDeliveryButton/RetryDeliveryButton.tsx b/internal/portal/src/common/RetryDeliveryButton/RetryDeliveryButton.tsx index 8746eaca..c46d48d0 100644 --- a/internal/portal/src/common/RetryDeliveryButton/RetryDeliveryButton.tsx +++ b/internal/portal/src/common/RetryDeliveryButton/RetryDeliveryButton.tsx @@ -5,7 +5,7 @@ import { showToast } from "../Toast/Toast"; import { ApiContext, formatError } from "../../app"; interface RetryDeliveryButtonProps { - deliveryId: string; + attemptId: string; disabled: boolean; loading: boolean; completed: (success: boolean) => void; @@ -14,7 +14,7 @@ interface RetryDeliveryButtonProps { } const RetryDeliveryButton: React.FC = ({ - deliveryId, + attemptId, disabled, loading, completed, @@ -29,7 +29,7 @@ const RetryDeliveryButton: React.FC = ({ e.stopPropagation(); setRetrying(true); try { - await apiClient.fetch(`deliveries/${deliveryId}/retry`, { + await apiClient.fetch(`attempts/${attemptId}/retry`, { method: "POST", }); showToast("success", "Retry successful."); @@ -41,7 +41,7 @@ const RetryDeliveryButton: React.FC = ({ setRetrying(false); }, - [apiClient, deliveryId, completed], + [apiClient, attemptId, completed], ); return ( diff --git a/internal/portal/src/scenes/Destination/Destination.tsx b/internal/portal/src/scenes/Destination/Destination.tsx index d6029b90..268864f3 100644 --- a/internal/portal/src/scenes/Destination/Destination.tsx +++ b/internal/portal/src/scenes/Destination/Destination.tsx @@ -14,19 +14,17 @@ import { } from "../../typings/Destination"; import getLogo from "../../utils/logo"; import DestinationSettings from "./DestinationSettings/DestinationSettings"; -import { DeliveryRoutes } from "./Events/Deliveries"; +import { AttemptRoutes } from "./Events/Attempts"; -// Define the tab interface interface Tab { label: string; path: string; } -// Define available tabs const tabs: Tab[] = [ { label: "Overview", path: "" }, { label: "Settings", path: "/settings" }, - { label: "Deliveries", path: "/deliveries" }, + { label: "Event Deliveries", path: "/deliveries" }, ]; const Destination = () => { @@ -134,7 +132,7 @@ const Destination = () => { /> } + element={} /> void; + navigateAttempt: (path: string, params?: any) => void; }) => { - const { delivery_id: deliveryId } = useParams(); + const { attempt_id: attemptId } = useParams(); - const { data: delivery } = useSWR( - `deliveries/${deliveryId}?include=event.data,response_data`, + const { data: attempt } = useSWR( + `attempts/${attemptId}?include=event.data,response_data`, ); - if (!delivery) { + if (!attempt) { return
Loading...
; } const event = - typeof delivery.event === "object" ? (delivery.event as EventFull) : null; + typeof attempt.event === "object" ? (attempt.event as EventFull) : null; return (

- {event?.topic || "Delivery"} + {event?.topic || "Attempt"}

{}} @@ -45,7 +45,7 @@ const DeliveryDetails = ({ icon iconLabel="Close" minimal - onClick={() => navigateDelivery("/")} + onClick={() => navigateAttempt("/")} > @@ -53,30 +53,30 @@ const DeliveryDetails = ({
-
-
+
+
Status
- {delivery.code && ( + {attempt.code && (
Response Code
-
{delivery.code}
+
{attempt.code}
)}
Attempt
-
{delivery.attempt}
+
{attempt.attempt_number}
{event && (
@@ -87,7 +87,7 @@ const DeliveryDetails = ({
Delivered at
- {new Date(delivery.delivered_at).toLocaleString("en-US", { + {new Date(attempt.delivered_at).toLocaleString("en-US", { year: "numeric", month: "numeric", day: "numeric", @@ -99,10 +99,10 @@ const DeliveryDetails = ({
-
Delivery ID
+
Attempt ID
- {delivery.id} - + {attempt.id} +
{event && ( @@ -118,7 +118,7 @@ const DeliveryDetails = ({
{event?.data && ( -
+

Data

                 {JSON.stringify(event.data, null, 2)}
@@ -127,7 +127,7 @@ const DeliveryDetails = ({
           )}
 
           {event?.metadata && Object.keys(event.metadata).length > 0 && (
-            

+            

               
Metadata

               
                 {JSON.stringify(event.metadata, null, 2)}
@@ -135,11 +135,11 @@ const DeliveryDetails = ({
             

           )}
 
-          {delivery.response_data && (
-            

+          {attempt.response_data && (
+            

               
Response

               
-                {JSON.stringify(delivery.response_data, null, 2)}
+                {JSON.stringify(attempt.response_data, null, 2)}
               

             

           )}
@@ -149,4 +149,4 @@ const DeliveryDetails = ({
   );
 };
 
-export default DeliveryDetails;
+export default AttemptDetails;
diff --git a/internal/portal/src/scenes/Destination/Events/Deliveries.scss b/internal/portal/src/scenes/Destination/Events/Attempts.scss
similarity index 98%
rename from internal/portal/src/scenes/Destination/Events/Deliveries.scss
rename to internal/portal/src/scenes/Destination/Events/Attempts.scss
index fff9146c..5d37d6d2 100644
--- a/internal/portal/src/scenes/Destination/Events/Deliveries.scss
+++ b/internal/portal/src/scenes/Destination/Events/Attempts.scss
@@ -1,4 +1,4 @@
-.destination-deliveries {
+.destination-attempts {
   margin-top: var(--spacing-5);
   margin-bottom: var(--spacing-20);
 
@@ -32,7 +32,7 @@
     display: grid;
     min-height: 713px;
 
-    .delivery-time-cell {
+    .attempt-time-cell {
       text-transform: uppercase;
     }
 
@@ -117,7 +117,7 @@
   }
 }
 
-.delivery-data {
+.attempt-data {
   height: 100%;
   box-sizing: border-box;
 
diff --git a/internal/portal/src/scenes/Destination/Events/Deliveries.tsx b/internal/portal/src/scenes/Destination/Events/Attempts.tsx
similarity index 79%
rename from internal/portal/src/scenes/Destination/Events/Deliveries.tsx
rename to internal/portal/src/scenes/Destination/Events/Attempts.tsx
index 3cfb8b8b..49299775 100644
--- a/internal/portal/src/scenes/Destination/Events/Deliveries.tsx
+++ b/internal/portal/src/scenes/Destination/Events/Attempts.tsx
@@ -1,9 +1,9 @@
 import { useCallback, useMemo, useState } from "react";
 import Badge from "../../../common/Badge/Badge";
 import Button from "../../../common/Button/Button";
-import "./Deliveries.scss";
+import "./Attempts.scss";
 import Table from "../../../common/Table/Table";
-import { DeliveryListResponse, EventSummary } from "../../../typings/Event";
+import { AttemptListResponse, EventSummary } from "../../../typings/Event";
 import useSWR from "swr";
 import Dropdown from "../../../common/Dropdown/Dropdown";
 import {
@@ -24,20 +24,20 @@ import {
   useParams,
 } from "react-router-dom";
 import CONFIGS from "../../../config";
-import DeliveryDetails from "./DeliveryDetails";
+import AttemptDetails from "./AttemptDetails";
 
-interface DeliveriesProps {
+interface AttemptsProps {
   destination: any;
-  navigateDelivery: (path: string, state?: any) => void;
+  navigateAttempt: (path: string, state?: any) => void;
 }
 
-const Deliveries: React.FC = ({
+const Attempts: React.FC = ({
   destination,
-  navigateDelivery,
+  navigateAttempt,
 }) => {
   const [timeRange, setTimeRange] = useState("24h");
-  const { delivery_id: deliveryId } = useParams<{ delivery_id: string }>();
-  const { status, topics, pagination, urlSearchParams } = useDeliveryFilter();
+  const { attempt_id: attemptId } = useParams<{ attempt_id: string }>();
+  const { status, topics, pagination, urlSearchParams } = useAttemptFilter();
 
   const queryUrl = useMemo(() => {
     const searchParams = new URLSearchParams(urlSearchParams);
@@ -70,31 +70,31 @@ const Deliveries: React.FC = ({
     searchParams.set("destination_id", destination.id);
     searchParams.set("include", "event");
 
-    return `deliveries?${searchParams.toString()}`;
+    return `attempts?${searchParams.toString()}`;
   }, [destination.id, timeRange, urlSearchParams]);
 
   const {
-    data: deliveriesList,
+    data: attemptsList,
     mutate,
     isValidating,
-  } = useSWR(queryUrl, {
+  } = useSWR(queryUrl, {
     revalidateOnFocus: false,
   });
 
   const topicsList = CONFIGS.TOPICS.split(",");
 
-  const table_rows = deliveriesList?.models
-    ? deliveriesList.models.map((delivery) => {
+  const table_rows = attemptsList?.models
+    ? attemptsList.models.map((attempt) => {
         const event =
-          typeof delivery.event === "object"
-            ? (delivery.event as EventSummary)
+          typeof attempt.event === "object"
+            ? (attempt.event as EventSummary)
             : null;
         return {
-          id: delivery.id,
-          active: delivery.id === (deliveryId || ""),
+          id: attempt.id,
+          active: attempt.id === (attemptId || ""),
           entries: [
-            
-              {new Date(delivery.delivered_at).toLocaleString("en-US", {
+            
+              {new Date(attempt.delivered_at).toLocaleString("en-US", {
                 month: "short",
                 day: "numeric",
                 hour: "numeric",
@@ -103,13 +103,13 @@ const Deliveries: React.FC = ({
               })}
             ,
             
-              {delivery.status === "success" ? (
+              {attempt.status === "success" ? (
                 
               ) : (
                 
               )}
                {
@@ -120,21 +120,21 @@ const Deliveries: React.FC = ({
               />
             ,
             {event?.topic || "-"},
-            {delivery.id},
+            {attempt.id},
           ],
-          onClick: () => navigateDelivery(`/${delivery.id}`),
+          onClick: () => navigateAttempt(`/${attempt.id}`),
         };
       })
     : [];
 
   return (
-    

-      

-        

-          Deliveries{" "}
-          
+    

+      

+        

+          Event Deliveries{" "}
+          
         

-        

+        

           }
             trigger={`Last ${timeRange}`}
@@ -230,8 +230,8 @@ const Deliveries: React.FC = ({
       

 
       

          = ({
               header: "Topic",
             },
             {
-              header: "Delivery ID",
+              header: "Attempt ID",
             },
           ]}
           rows={table_rows}
@@ -256,7 +256,7 @@ const Deliveries: React.FC = ({
             

               

                 
-                  {deliveriesList?.models.length ?? 0} deliveries
+                  {attemptsList?.models.length ?? 0} attempts
                 
               

 
@@ -264,9 +264,9 @@ const Deliveries: React.FC = ({