send transcript and recording to cal ai

.env.example

@@ -42,7 +42,7 @@ NEXT_PUBLIC_WEBAPP_URL='http://localhost:3000'
 NEXT_PUBLIC_WEBSITE_URL='http://localhost:3000'
 NEXT_PUBLIC_CONSOLE_URL='http://localhost:3004'
 NEXT_PUBLIC_EMBED_LIB_URL='http://localhost:3000/embed/embed.js'
-
+CAL_AI_PARSE_KEY=''
 # To enable SAML login, set both these variables
 # @see https://github.com/calcom/cal.com/tree/main/packages/features/ee#setting-up-saml-login
 # SAML_DATABASE_URL="postgresql://postgres:@localhost:5450/cal-saml"

apps/ai/src/app/api/sumarize/route.ts

@@ -1,5 +0,0 @@
-import type { NextRequest } from "next/server";
-
-export const POST = async (request: NextRequest) => {
-  console.log("req", request);
-};

apps/ai/src/app/api/transcribe/route.ts

@@ -0,0 +1,94 @@
+import type { NextRequest } from "next/server";
+import { NextResponse } from "next/server";
+import { z } from "zod";
+
+import type { Booking } from "@calcom/prisma/client";
+
+import sendEmail from "../../../utils/sendEmail";
+import { verifyParseKey } from "../../../utils/verifyParseKey";
+
+const dailyUrl = "https://api.daily.co/v1";
+
+const requestJobSchema = z.object({
+  id: z.string().min(1),
+});
+/** API Reference @link https://docs.daily.co/private/batch-processor/reference/submit-job */
+const submitJob = async (
+  recordingId: string,
+  preset: "transcript" | "summarize"
+): Promise<{ data: { jobId: string } | null; error: string | null }> => {
+  const data = {
+    preset: preset,
+    inParams: { sourceType: "recordingId", recordingId: recordingId },
+    outParams: { s3Config: { s3KeyTemplate: preset == "transcript" ? "transcript" : "summary" } },
+  };
+  try {
+    const response = await fetch(`${dailyUrl}/batch-processor`, {
+      headers: {
+        "Content-type": "application/json",
+        Authorization: `Bearer ${process.env.DAILY_API_KEY}`,
+      },
+      method: "POST",
+      body: JSON.stringify(data),
+    });
+    if (response.status == 200) {
+      const { id } = requestJobSchema.parse(response.body);
+      return { data: { jobId: id }, error: null };
+    } else {
+      return { data: null, error: "Could not request transcript" };
+    }
+  } catch {
+    return { data: null, error: "Could not reach Daily API" };
+  }
+};
+const corsHeaders = {
+  "Access-Control-Allow-Origin": "*",
+  "Access-Control-Allow-Headers": "authorization, x-client-info, apikey, content-type",
+};
+
+export const POST = async (request: NextRequest) => {
+  if (request.method === "OPTIONS") {
+    return new NextResponse("ok", { headers: corsHeaders });
+  }
+
+  try {
+    const verified = verifyParseKey(request.url);
+
+    if (!verified) {
+      return new NextResponse("Unauthorized", { status: 401 });
+    }
+
+    const {
+      recordingId,
+      transcriptBody,
+      booking,
+      organizerEmail,
+    }: { recordingId: string; transcriptBody: string; booking: Booking; organizerEmail: string } =
+      await request.json();
+
+    if (!transcriptBody || !booking || !organizerEmail) {
+      return new NextResponse("No data received", { status: 400 });
+    }
+    //Approach 1 - send Transcript directly
+    if (transcriptBody) {
+      console.log(`transcript: ${transcriptBody}`);
+      await sendEmail({
+        subject: `Re: ${booking.title}`,
+        text: `Thanks for using Cal.ai! Here's your transcript for the meeting`,
+        to: organizerEmail, // TODO - do we want to send transcript to all attendees?
+        from: "", //TODO ,
+      });
+    }
+
+    //Approach 2 - use Batch Processor API
+    if (recordingId) {
+      const { data: transcriptJobId, error: transcriptError } = await submitJob(recordingId, "transcript");
+      const { data: summaryJobId, error: summaryError } = await submitJob(recordingId, "summarize");
+      //TODO add cron to check for job status
+      /** API reference @link https://docs.daily.co/private/batch-processor/reference/get-job-access-link */
+    }
+    return new NextResponse("ok");
+  } catch {
+    return new NextResponse("Could not process the request", { status: 500 });
+  }
+};

apps/web/pages/api/recorded-daily-video.ts

@@ -113,7 +113,7 @@ async function handler(req: NextApiRequest, res: NextApiResponse) {
       },
     });
 
-    if (!booking || booking.location !== DailyLocationType) {
+    if (!booking || booking.location !== DailyLocationType || !booking.user) {
       return res.status(404).send({
         message: `Booking of uid ${bookingUID} does not exist or does not contain daily video as location`,
       });
@@ -154,6 +154,28 @@ async function handler(req: NextApiRequest, res: NextApiResponse) {
       },
     });
 
+    const aiAppInstalled = await prisma.credential.findFirst({
+      where: {
+        appId: "cal-ai",
+        userId: booking.user.id,
+      },
+    });
+    if (aiAppInstalled) {
+      console.log(`calAIKey ${process.env.CAL_AI_PARSE_KEY}`);
+      const transcribeData = {
+        recordingId: recordingId,
+        attendeesList: attendeesList,
+        organiserEmail: booking.user.email,
+      };
+      const calAiUrl = process.env.CAL_AI_URL;
+      //TODO replace with CAL_AI URL
+      await fetch(`${calAiUrl}/api/transcribe?parseKey=${process.env.CAL_AI_PARSE_KEY}`, {
+        method: "POST",
+        body: JSON.stringify(transcribeData),
+      }).catch((err) => {
+        console.log(err);
+      });
+    }
     const response = await getDownloadLinkOfCalVideoByRecordingId(recordingId);
     const downloadLinkResponse = downloadLinkSchema.parse(response);
     const downloadLink = downloadLinkResponse.download_link;

apps/web/pages/video/[uid].tsx

@@ -10,7 +10,7 @@ import type { GetServerSidePropsContext } from "next";
 import Head from "next/head";
 import { useRouter } from "next/navigation";
 import { useState, useEffect, useRef } from "react";
-import showToast, { toast } from "react-hot-toast";
+import { toast } from "react-hot-toast";
 import z from "zod";
 
 import dayjs from "@calcom/dayjs";
@@ -21,7 +21,6 @@ import { formatToLocalizedDate, formatToLocalizedTime } from "@calcom/lib/date-f
 import { useLocale } from "@calcom/lib/hooks/useLocale";
 import { markdownToSafeHTML } from "@calcom/lib/markdownToSafeHTML";
 import prisma, { bookingMinimalSelect } from "@calcom/prisma";
-import { trpc } from "@calcom/trpc/react";
 import type { inferSSRProps } from "@calcom/types/inferSSRProps";
 import { ChevronRight } from "@calcom/ui/components/icon";
 
@@ -39,22 +38,13 @@ export type JoinCallPageProps = inferSSRProps<typeof getServerSideProps>;
 const md = new MarkdownIt("default", { html: true, breaks: true, linkify: true });
 
 export default function JoinCall(props: JoinCallPageProps) {
-  const utils = trpc.useContext();
   const { t } = useLocale();
   const router = useRouter();
   const { meetingUrl, meetingPassword, booking } = props;
   const recordingId = useRef<string | null>(null);
   const isTranscribing = useRef<boolean>(false);
   const transcript = useRef<string>("");
-  const createWebhookMutation = trpc.viewer.webhook.create.useMutation({
-    async onSuccess() {
-      await utils.viewer.webhook.list.invalidate();
-      router.back();
-    },
-    onError(error) {
-      showToast(`${error.message}`);
-    },
-  });
+
   useEffect(() => {
     /** adding a custom tray button @link https://docs.daily.co/reference/daily-js/daily-iframe-class/properties#customTrayButtons* */
     const callFrame = DailyIframe.createFrame({
@@ -89,7 +79,7 @@ export default function JoinCall(props: JoinCallPageProps) {
       url: meetingUrl,
       ...(typeof meetingPassword === "string" && { token: meetingPassword }),
     });
-    //TODO - add handlers for `transcription-started` and `transcription-ended` @link https://docs.daily.co/reference/rn-daily-js/events/transcription-events
+    /** API reference @link https://docs.daily.co/reference/rn-daily-js/events/transcription-events */
     /** handling custom-button-click @link  https://docs.daily.co/reference/daily-js/events/meeting-events#custom-button-click */
     const onCustomButtonClick = async (event?: DailyEventObjectCustomButtonClick | undefined) => {
       if (!props.calAiCredential) {
@@ -111,7 +101,7 @@ export default function JoinCall(props: JoinCallPageProps) {
       if (!!isTranscribing.current) {
         return;
       }
-      console.log("starting tradncription");
+      console.log("starting transcription");
       isTranscribing.current = true;
       await callFrame.startTranscription(transcriptionOptions);
     }
@@ -154,7 +144,7 @@ export default function JoinCall(props: JoinCallPageProps) {
 
   const onRecordingStopped = () => {
     const data = { recordingId: recordingId.current, bookingUID: booking.uid };
-
+    console.log(`data when recording stopped ${JSON.stringify(data)}`);
     fetch("/api/recorded-daily-video", {
       method: "POST",
       body: JSON.stringify(data),
@@ -175,16 +165,22 @@ export default function JoinCall(props: JoinCallPageProps) {
   const handleTranscriptionStopped = async () => {
     //
   };
-  const handleLeftMeeting = () => {
-    //TODO send transcript by email
-    console.log(`Full Transcript: ${transcript.current}`);
-    createWebhookMutation.mutate({
-      subscriberUrl: "",
-      eventTriggers: ["MEETING_ENDED"],
-      active: true,
-      payloadTemplate: "",
-      secret: "",
-    });
+  const handleLeftMeeting = async () => {
+    if (props.calAiCredential) {
+      console.log(`Full Transcript: ${transcript.current}`);
+      const transcribeData = {
+        recordingId: recordingId,
+        bookingL: booking,
+        organiserEmail: booking.user?.email,
+      };
+      //TODO need to replace the `CAL_AI_PARSE_KEY` with a key that can be exposed to client or bypass this altogether
+      await fetch(`${calAiUrl}/api/transcribe?parseKey=${process.env.CAL_AI_PARSE_KEY}`, {
+        method: "POST",
+        body: JSON.stringify(transcribeData),
+      }).catch((err) => {
+        console.log(err);
+      });
+    }
   };
   const handleTranscriptionError = () => {
     toast.error("Could not run transcription service for this meeting");

packages/app-store/dailyvideo/lib/VideoApiAdapter.ts

@@ -1,6 +1,7 @@
 import { z } from "zod";
 
 import { handleErrorsJson } from "@calcom/lib/errors";
+import logger from "@calcom/lib/logger";
 import { prisma } from "@calcom/prisma";
 import type { GetRecordingsResponseSchema, GetAccessLinkResponseSchema } from "@calcom/prisma/zod-utils";
 import { getRecordingsResponseSchema, getAccessLinkResponseSchema } from "@calcom/prisma/zod-utils";
@@ -92,6 +93,7 @@ const DailyVideoApiAdapter = (): VideoApiAdapter => {
       throw new Error("We need need the booking uid to create the Daily reference in DB");
     }
     const body = await translateEvent(event);
+    console.log(`meeting body: ${JSON.stringify(body)}`);
     const dailyEvent = await postToDailyAPI(endpoint, body).then(dailyReturnTypeSchema.parse);
     const meetingToken = await postToDailyAPI("/meeting-tokens", {
       properties: {
@@ -116,6 +118,7 @@ const DailyVideoApiAdapter = (): VideoApiAdapter => {
         userId: event.organizer.id,
       },
     });
+    logger.log(0, `checkCalAiInstalled: ${JSON.stringify(checkCalAiInstalled)}`);
     // Documentation at: https://docs.daily.co/reference#list-rooms
     // added a 1 hour buffer for room expiration
     const exp = Math.round(new Date(event.endTime).getTime() / 1000) + 60 * 60;
@@ -161,6 +164,7 @@ const DailyVideoApiAdapter = (): VideoApiAdapter => {
         privacy: "public",
         properties: {
           ...options,
+          enable_recording: "cloud",
           enable_transcription: `deepgram:${process.env.DEEPGRAM_API_KEY}`,
           permissions: { canAdmin: ["transcription"] },
         },

turbo.json

@@ -200,6 +200,8 @@
     "BASECAMP3_USER_AGENT",
     "AUTH_BEARER_TOKEN_VERCEL",
     "BUILD_ID",
+    "CAL_AI_URL",
+    "CAL_AI_PARSE_KEY",
     "CALCOM_APP_CREDENTIAL_ENCRYPTION_KEY",
     "CALCOM_CREDENTIAL_SYNC_ENDPOINT",
     "CALCOM_ENV",