chore: expect emails e2e (#10881)

* refactor: emails e2e

* fix: email subject with nameless organizer

* fix: comment on NEXT_PUBLIC_IS_E2E

* chore: E2E_TEST_MAILHOG_ENABLED env

* feat: IS_MAILHOG_ENABLED

* doc: E2E_TEST_MAILHOG_ENABLED

* fix: comment

* fix: env order

* chore: add E2E_TEST_MAILHOG_ENABLED to e2e env

.env.example

@@ -126,8 +126,7 @@ TWILIO_WHATSAPP_PHONE_NUMBER=
 NEXT_PUBLIC_SENDER_ID=
 TWILIO_VERIFY_SID=
 
-# This is used so we can bypass emails in auth flows for E2E testing
-# Set it to "1" if you need to run E2E tests locally
+# Set it to "1" if you need to run E2E tests locally. 
 NEXT_PUBLIC_IS_E2E=
 
 # Used for internal billing system
@@ -172,6 +171,12 @@ EMAIL_SERVER_PORT=1025
 ## You will need to provision an App Password.
 ## @see https://support.google.com/accounts/answer/185833
 # EMAIL_SERVER_PASSWORD='<gmail_app_password>'
+
+# Used for E2E for email testing
+# Set it to "1" if you need to email checks in E2E tests locally
+# Make sure to run mailhog container manually or with `yarn dx`
+E2E_TEST_MAILHOG_ENABLED=
+
 # **********************************************************************************************************
 
 # Set the following value to true if you wish to enable Team Impersonation

.github/workflows/e2e-app-store.yml

@@ -43,6 +43,7 @@ jobs:
           DEPLOYSENTINEL_API_KEY: ${{ secrets.DEPLOYSENTINEL_API_KEY }}
           E2E_TEST_APPLE_CALENDAR_EMAIL: ${{ secrets.E2E_TEST_APPLE_CALENDAR_EMAIL }}
           E2E_TEST_APPLE_CALENDAR_PASSWORD: ${{ secrets.E2E_TEST_APPLE_CALENDAR_PASSWORD }}
+          E2E_TEST_MAILHOG_ENABLED: ${{ vars.E2E_TEST_MAILHOG_ENABLED }}
           GOOGLE_API_CREDENTIALS: ${{ secrets.CI_GOOGLE_API_CREDENTIALS }}
           GOOGLE_LOGIN_ENABLED: ${{ vars.CI_GOOGLE_LOGIN_ENABLED }}
           NEXTAUTH_SECRET: ${{ secrets.CI_NEXTAUTH_SECRET }}

.github/workflows/e2e-embed.yml

@@ -43,6 +43,7 @@ jobs:
           DEPLOYSENTINEL_API_KEY: ${{ secrets.DEPLOYSENTINEL_API_KEY }}
           E2E_TEST_APPLE_CALENDAR_EMAIL: ${{ secrets.E2E_TEST_APPLE_CALENDAR_EMAIL }}
           E2E_TEST_APPLE_CALENDAR_PASSWORD: ${{ secrets.E2E_TEST_APPLE_CALENDAR_PASSWORD }}
+          E2E_TEST_MAILHOG_ENABLED: ${{ vars.E2E_TEST_MAILHOG_ENABLED }}
           GOOGLE_API_CREDENTIALS: ${{ secrets.CI_GOOGLE_API_CREDENTIALS }}
           GOOGLE_LOGIN_ENABLED: ${{ vars.CI_GOOGLE_LOGIN_ENABLED }}
           NEXTAUTH_SECRET: ${{ secrets.CI_NEXTAUTH_SECRET }}

.github/workflows/e2e.yml

@@ -42,6 +42,7 @@ jobs:
           DEPLOYSENTINEL_API_KEY: ${{ secrets.DEPLOYSENTINEL_API_KEY }}
           E2E_TEST_APPLE_CALENDAR_EMAIL: ${{ secrets.E2E_TEST_APPLE_CALENDAR_EMAIL }}
           E2E_TEST_APPLE_CALENDAR_PASSWORD: ${{ secrets.E2E_TEST_APPLE_CALENDAR_PASSWORD }}
+          E2E_TEST_MAILHOG_ENABLED: ${{ vars.E2E_TEST_MAILHOG_ENABLED }}
           GOOGLE_API_CREDENTIALS: ${{ secrets.CI_GOOGLE_API_CREDENTIALS }}
           EMAIL_SERVER_HOST: ${{ secrets.CI_EMAIL_SERVER_HOST }}
           EMAIL_SERVER_PORT: ${{ secrets.CI_EMAIL_SERVER_PORT }}

README.md

@@ -221,6 +221,7 @@ echo 'NEXT_PUBLIC_DEBUG=1' >> .env
    ```
 
 1. Run [mailhog](https://github.com/mailhog/MailHog) to view emails sent during development
+   > **_NOTE:_** Required when `E2E_TEST_MAILHOG_ENABLED` is "1"
 
    ```sh
    docker pull mailhog/mailhog

apps/web/playwright/booking-pages.e2e.ts

@@ -1,5 +1,4 @@
 import { expect } from "@playwright/test";
-import type { Messages } from "mailhog";
 
 import { randomString } from "@calcom/lib/random";
 
@@ -8,6 +7,7 @@ import {
   bookFirstEvent,
   bookOptinEvent,
   bookTimeSlot,
+  expectEmailsToHaveSubject,
   selectFirstAvailableTimeSlotNextMonth,
   testEmail,
   testName,
@@ -41,19 +41,13 @@ test.describe("free user", () => {
     // Make sure we're navigated to the success page
     await expect(page.locator("[data-testid=success-page]")).toBeVisible();
     const { title: eventTitle } = await user.getFirstEventAsOwner();
-    // TODO: follow DRY
-    const emailsOrganiserReceived = await emails.search(user.email, "to");
-    const emailsBookerReceived = await emails.search(bookerObj.email, "to");
-    expect(emailsOrganiserReceived?.total).toBe(1);
-    expect(emailsBookerReceived?.total).toBe(1);
-
-    const [organizerFirstEmail] = (emailsOrganiserReceived as Messages).items;
-    const [bookerFirstEmail] = (emailsBookerReceived as Messages).items;
-    const emailSubject = `${eventTitle} between ${user.name} and ${bookerObj.name}`;
-
-    expect(organizerFirstEmail.subject).toBe(emailSubject);
-    expect(bookerFirstEmail.subject).toBe(emailSubject);
 
+    await expectEmailsToHaveSubject({
+      emails,
+      organizer: user,
+      booker: bookerObj,
+      eventTitle,
+    });
     await page.goto(bookingUrl);
 
     // book same time spot again

apps/web/playwright/fixtures/users.ts

@@ -118,7 +118,7 @@ const createTeamAndAddUser = async (
 };
 
 // creates a user fixture instance and stores the collection
-export const createUsersFixture = (page: Page, emails: API, workerInfo: WorkerInfo) => {
+export const createUsersFixture = (page: Page, emails: API | undefined, workerInfo: WorkerInfo) => {
   const store = { users: [], page } as { users: UserFixture[]; page: typeof page };
   return {
     create: async (
@@ -332,8 +332,9 @@ export const createUsersFixture = (page: Page, emails: API, workerInfo: WorkerIn
       await page.goto("/auth/logout");
     },
     deleteAll: async () => {
-      const emailMessageIds: string[] = [];
       const ids = store.users.map((u) => u.id);
+      if (emails) {
+        const emailMessageIds: string[] = [];
         for (const user of store.users) {
           const emailMessages = await emails.search(user.email);
           if (emailMessages && emailMessages.count > 0) {
@@ -345,6 +346,8 @@ export const createUsersFixture = (page: Page, emails: API, workerInfo: WorkerIn
         for (const id of emailMessageIds) {
           await emails.deleteMessage(id);
         }
+      }
+
       await prisma.user.deleteMany({ where: { id: { in: ids } } });
       store.users = [];
     },

apps/web/playwright/lib/fixtures.ts

@@ -3,6 +3,7 @@ import { test as base } from "@playwright/test";
 import type { API } from "mailhog";
 import mailhog from "mailhog";
 
+import { IS_MAILHOG_ENABLED } from "@calcom/lib/constants";
 import prisma from "@calcom/prisma";
 
 import type { ExpectedUrlDetails } from "../../../../playwright.config";
@@ -21,7 +22,7 @@ export interface Fixtures {
   getActionFiredDetails: ReturnType<typeof createGetActionFiredDetails>;
   servers: ReturnType<typeof createServersFixture>;
   prisma: typeof prisma;
-  emails: API;
+  emails?: API;
 }
 
 declare global {
@@ -71,7 +72,11 @@ export const test = base.extend<Fixtures>({
     await use(prisma);
   },
   emails: async ({}, use) => {
+    if (IS_MAILHOG_ENABLED) {
       const mailhogAPI = mailhog();
       await use(mailhogAPI);
+    } else {
+      await use(undefined);
+    }
   },
 });

apps/web/playwright/lib/testUtils.ts

@@ -4,6 +4,7 @@ import type { IncomingMessage, ServerResponse } from "http";
 import { createServer } from "http";
 // eslint-disable-next-line no-restricted-imports
 import { noop } from "lodash";
+import type { API, Messages } from "mailhog";
 
 import { test } from "./fixtures";
 
@@ -191,3 +192,39 @@ export async function installAppleCalendar(page: Page) {
   await page.waitForURL("/apps/apple-calendar");
   await page.click('[data-testid="install-app-button"]');
 }
+export async function getEmailsReceivedByUser({
+  emails,
+  userEmail,
+}: {
+  emails?: API;
+  userEmail: string;
+}): Promise<Messages | null> {
+  if (!emails) return null;
+  return emails.search(userEmail, "to");
+}
+
+export async function expectEmailsToHaveSubject({
+  emails,
+  organizer,
+  booker,
+  eventTitle,
+}: {
+  emails?: API;
+  organizer: { name?: string | null; email: string };
+  booker: { name: string; email: string };
+  eventTitle: string;
+}) {
+  if (!emails) return null;
+  const emailsOrganizerReceived = await getEmailsReceivedByUser({ emails, userEmail: organizer.email });
+  const emailsBookerReceived = await getEmailsReceivedByUser({ emails, userEmail: booker.email });
+
+  expect(emailsOrganizerReceived?.total).toBe(1);
+  expect(emailsBookerReceived?.total).toBe(1);
+
+  const [organizerFirstEmail] = (emailsOrganizerReceived as Messages).items;
+  const [bookerFirstEmail] = (emailsBookerReceived as Messages).items;
+  const emailSubject = `${eventTitle} between ${organizer.name ?? "Nameless"} and ${booker.name}`;
+
+  expect(organizerFirstEmail.subject).toBe(emailSubject);
+  expect(bookerFirstEmail.subject).toBe(emailSubject);
+}

packages/lib/constants.ts

@@ -94,4 +94,7 @@ export const ALLOWED_HOSTNAMES = JSON.parse(`[${process.env.ALLOWED_HOSTNAMES ||
 export const RESERVED_SUBDOMAINS = JSON.parse(`[${process.env.RESERVED_SUBDOMAINS || ""}]`) as string[];
 
 export const ORGANIZATION_MIN_SEATS = 30;
+
+// Needed for emails in E2E
+export const IS_MAILHOG_ENABLED = process.env.E2E_TEST_MAILHOG_ENABLED === "1";
 export const CALCOM_VERSION = process.env.NEXT_PUBLIC_CALCOM_VERSION as string;

packages/types/environment.d.ts

@@ -45,10 +45,13 @@ declare namespace NodeJS {
     /** The URL of the deployment. Example: my-site-7q03y4pi5.vercel.app. */
     readonly VERCEL_URL: string | undefined;
     /**
-     * This is used so we can bypass emails in auth flows for E2E testing.
      * Set it to "1" if you need to run E2E tests locally
      **/
     readonly NEXT_PUBLIC_IS_E2E: "1" | undefined;
+    /**
+     * This is used so we can enable Mailhog in E2E tests.
+     */
+    readonly E2E_TEST_MAILHOG_ENABLED: "1" | undefined;
     readonly NEXT_PUBLIC_APP_NAME: string | "Cal";
     readonly NEXT_PUBLIC_SUPPORT_MAIL_ADDRESS: string | "help@cal.com";
     readonly NEXT_PUBLIC_COMPANY_NAME: string | "Cal.com, Inc.";

turbo.json

@@ -198,6 +198,7 @@
     "DEBUG",
     "E2E_TEST_APPLE_CALENDAR_EMAIL",
     "E2E_TEST_APPLE_CALENDAR_PASSWORD",
+    "E2E_TEST_MAILHOG_ENABLED",
     "EMAIL_FROM",
     "EMAIL_SERVER_HOST",
     "EMAIL_SERVER_PASSWORD",