From 8eccd3658e8a67ee60d446edaa0c43338ec15745 Mon Sep 17 00:00:00 2001 From: sean-brydon <55134778+sean-brydon@users.noreply.github.com> Date: Mon, 19 Jun 2023 11:01:06 +0100 Subject: [PATCH] feat: Upstash implementation for rate limiting/redis (#9514) * Introduce rate limiting that works on the edge * typo * Log once on init * Update rateLimit.ts --------- Co-authored-by: zomars --- .env.example | 2 + apps/web/package.json | 2 + apps/web/pages/api/auth/forgot-password.ts | 18 ++--- .../features/auth/lib/next-auth-options.ts | 13 ++-- packages/features/auth/lib/verifyEmail.ts | 13 ++-- packages/lib/getIP.ts | 10 +++ packages/lib/rateLimit.ts | 66 ++++++++++++------- turbo.json | 5 +- yarn.lock | 39 +++++++++++ 9 files changed, 124 insertions(+), 44 deletions(-) diff --git a/.env.example b/.env.example index 798c774414..31b3c60756 100644 --- a/.env.example +++ b/.env.example @@ -23,6 +23,8 @@ CALCOM_LICENSE_KEY= # - DATABASE ************************************************************************************************ DATABASE_URL="postgresql://postgres:@localhost:5450/calendso" +UPSTASH_REDIS_REST_URL= +UPSTASH_REDIS_REST_TOKEN= # Uncomment to enable a dedicated connection pool for Prisma using Prisma Data Proxy # Cold boots will be faster and you'll be able to scale your DB independently of your app. diff --git a/apps/web/package.json b/apps/web/package.json index e4a0282b8c..4fef7ecdce 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -61,6 +61,8 @@ "@tanstack/react-query": "^4.3.9", "@tremor/react": "^2.0.0", "@types/turndown": "^5.0.1", + "@upstash/ratelimit": "^0.4.3", + "@upstash/redis": "^1.21.0", "@vercel/edge-config": "^0.1.1", "@vercel/edge-functions-ui": "^0.2.1", "@vercel/og": "^0.5.0", diff --git a/apps/web/pages/api/auth/forgot-password.ts b/apps/web/pages/api/auth/forgot-password.ts index 1d671e5b14..d31e1c0b20 100644 --- a/apps/web/pages/api/auth/forgot-password.ts +++ b/apps/web/pages/api/auth/forgot-password.ts @@ -5,15 +5,12 @@ import { z } from "zod"; import dayjs from "@calcom/dayjs"; import { sendPasswordResetEmail } from "@calcom/emails"; import { PASSWORD_RESET_EXPIRY_HOURS } from "@calcom/emails/templates/forgot-password-email"; -import rateLimit from "@calcom/lib/rateLimit"; +import { ErrorCode } from "@calcom/features/auth/lib/ErrorCode"; +import rateLimiter from "@calcom/lib/rateLimit"; import { defaultHandler } from "@calcom/lib/server"; import { getTranslation } from "@calcom/lib/server/i18n"; import prisma from "@calcom/prisma"; -const limiter = rateLimit({ - intervalInMs: 60 * 1000, // 1 minute -}); - async function handler(req: NextApiRequest, res: NextApiResponse) { const t = await getTranslation(req.body.language ?? "en", "common"); @@ -37,10 +34,13 @@ async function handler(req: NextApiRequest, res: NextApiResponse) { // 10 requests per minute - try { - limiter.check(10, ip); - } catch (e) { - return res.status(429).json({ message: "Too Many Requests." }); + const limiter = await rateLimiter(); + const limit = await limiter({ + identifier: ip, + }); + + if (!limit.success) { + throw new Error(ErrorCode.RateLimitExceeded); } try { diff --git a/packages/features/auth/lib/next-auth-options.ts b/packages/features/auth/lib/next-auth-options.ts index f717ad2f1a..a7b336c826 100644 --- a/packages/features/auth/lib/next-auth-options.ts +++ b/packages/features/auth/lib/next-auth-options.ts @@ -14,7 +14,7 @@ import { symmetricDecrypt } from "@calcom/lib/crypto"; import { defaultCookies } from "@calcom/lib/default-cookies"; import { isENVDev } from "@calcom/lib/env"; import { randomString } from "@calcom/lib/random"; -import rateLimit from "@calcom/lib/rateLimit"; +import rateLimiter from "@calcom/lib/rateLimit"; import slugify from "@calcom/lib/slugify"; import prisma from "@calcom/prisma"; import { IdentityProvider } from "@calcom/prisma/enums"; @@ -102,11 +102,14 @@ const providers: Provider[] = [ if (!user) { throw new Error(ErrorCode.IncorrectUsernamePassword); } - - const limiter = rateLimit({ - intervalInMs: 60 * 1000, // 1 minute + const limiter = await rateLimiter(); + const rateLimit = await limiter({ + identifier: user.email, }); - await limiter.check(10, user.email); // 10 requests per minute + + if (!rateLimit.success) { + throw new Error(ErrorCode.RateLimitExceeded); + } if (user.identityProvider !== IdentityProvider.CAL && !credentials.totpCode) { throw new Error(ErrorCode.ThirdPartyIdentityProviderEnabled); diff --git a/packages/features/auth/lib/verifyEmail.ts b/packages/features/auth/lib/verifyEmail.ts index a1c9b01ba9..454efe7d45 100644 --- a/packages/features/auth/lib/verifyEmail.ts +++ b/packages/features/auth/lib/verifyEmail.ts @@ -4,17 +4,13 @@ import { sendEmailVerificationLink } from "@calcom/emails/email-manager"; import { getFeatureFlagMap } from "@calcom/features/flags/server/utils"; import { WEBAPP_URL } from "@calcom/lib/constants"; import logger from "@calcom/lib/logger"; -import rateLimit from "@calcom/lib/rateLimit"; +import rateLimiter from "@calcom/lib/rateLimit"; import { getTranslation } from "@calcom/lib/server/i18n"; import { prisma } from "@calcom/prisma"; import { TRPCError } from "@calcom/trpc/server"; const log = logger.getChildLogger({ prefix: [`[[Auth] `] }); -const limiter = rateLimit({ - intervalInMs: 60 * 1000, // 1 minute -}); - interface VerifyEmailType { username?: string; email: string; @@ -43,9 +39,12 @@ export const sendEmailVerification = async ({ email, language, username }: Verif token, }); - const { isRateLimited } = limiter.check(10, email); // 10 requests per minute + const limiter = await rateLimiter(); + const rateLimit = await limiter({ + identifier: email, + }); - if (isRateLimited) { + if (!rateLimit.success) { throw new TRPCError({ code: "TOO_MANY_REQUESTS", message: "An unexpected error occurred, please try again later.", diff --git a/packages/lib/getIP.ts b/packages/lib/getIP.ts index 342670d5ab..31c18803aa 100644 --- a/packages/lib/getIP.ts +++ b/packages/lib/getIP.ts @@ -28,3 +28,13 @@ export function isIpInBanlist(request: Request | NextApiRequest) { } return false; } + +export function isIpInBanListString(identifer: string) { + const rawBanListJson = process.env.IP_BANLIST || "[]"; + const banList = banlistSchema.parse(JSON.parse(rawBanListJson)); + if (banList.includes(identifer)) { + console.log(`Found banned IP: ${identifer} in IP_BANLIST`); + return true; + } + return false; +} diff --git a/packages/lib/rateLimit.ts b/packages/lib/rateLimit.ts index 4941842c55..e56fbf7869 100644 --- a/packages/lib/rateLimit.ts +++ b/packages/lib/rateLimit.ts @@ -1,27 +1,49 @@ -/* eslint-disable @typescript-eslint/no-explicit-any */ -import cache from "memory-cache"; +import { Ratelimit } from "@upstash/ratelimit"; +import { Redis } from "@upstash/redis"; -import { ErrorCode } from "@calcom/features/auth/lib/ErrorCode"; +import { isIpInBanListString } from "./getIP"; +import logger from "./logger"; -const rateLimit = (options: { intervalInMs: number }) => { - return { - check: (requestLimit: number, uniqueIdentifier: string) => { - const count = cache.get(uniqueIdentifier) || [0]; - if (count[0] === 0) { - cache.put(uniqueIdentifier, count, options.intervalInMs); - } - count[0] += 1; +const log = logger.getChildLogger({ prefix: ["RateLimit"] }); - const currentUsage = count[0]; - const isRateLimited = currentUsage >= requestLimit; - - if (isRateLimited) { - throw new Error(ErrorCode.RateLimitExceeded); - } - - return { isRateLimited, requestLimit, remaining: isRateLimited ? 0 : requestLimit - currentUsage }; - }, - }; +type RateLimitHelper = { + rateLimitingType?: "core" | "forcedSlowMode"; + identifier: string; }; -export default rateLimit; +function rateLimiter() { + const UPSATCH_ENV_FOUND = process.env.UPSTASH_REDIS_REST_URL && process.env.UPSTASH_REDIS_REST_TOKEN; + + if (!UPSATCH_ENV_FOUND) { + log.warn("Disabled due to not finding UPSTASH env variables"); + return () => ({ success: true }); + } + + const redis = Redis.fromEnv(); + const limiter = { + core: new Ratelimit({ + redis, + analytics: true, + prefix: "ratelimit", + limiter: Ratelimit.fixedWindow(10, "60s"), + }), + forcedSlowMode: new Ratelimit({ + redis, + analytics: true, + prefix: "ratelimit:slowmode", + limiter: Ratelimit.fixedWindow(1, "30s"), + }), + }; + + async function rateLimit({ rateLimitingType = "core", identifier }: RateLimitHelper) { + if (isIpInBanListString(identifier)) { + return await limiter.forcedSlowMode.limit(identifier); + } + + return await limiter[rateLimitingType].limit(identifier); + } + + return rateLimit; +} + +export default rateLimiter; diff --git a/turbo.json b/turbo.json index e8696b66c2..f4b2e877f6 100644 --- a/turbo.json +++ b/turbo.json @@ -175,9 +175,9 @@ "globalDependencies": ["yarn.lock"], "globalEnv": [ "ANALYZE", - "AUTH_BEARER_TOKEN_VERCEL", "API_KEY_PREFIX", "APP_USER_NAME", + "AUTH_BEARER_TOKEN_VERCEL", "BUILD_ID", "CALCOM_LICENSE_KEY", "CALCOM_TELEMETRY_DISABLED", @@ -204,6 +204,7 @@ "HUBSPOT_CLIENT_SECRET", "INTEGRATION_TEST_MODE", "INTERCOM_SECRET", + "INTERCOM_SECRET", "IP_BANLIST", "LARK_OPEN_APP_ID", "LARK_OPEN_APP_SECRET", @@ -272,6 +273,8 @@ "TWILIO_SID", "TWILIO_TOKEN", "TWILIO_VERIFY_SID", + "UPSTASH_REDIS_REST_TOKEN", + "UPSTASH_REDIS_REST_URL", "VERCEL_ENV", "VERCEL_URL", "VITAL_API_KEY", diff --git a/yarn.lock b/yarn.lock index 06f8d763fe..f99f7846a4 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4896,6 +4896,8 @@ __metadata: "@types/stripe": ^8.0.417 "@types/turndown": ^5.0.1 "@types/uuid": 8.3.1 + "@upstash/ratelimit": ^0.4.3 + "@upstash/redis": ^1.21.0 "@vercel/edge-config": ^0.1.1 "@vercel/edge-functions-ui": ^0.2.1 "@vercel/og": ^0.5.0 @@ -13331,6 +13333,33 @@ __metadata: languageName: node linkType: hard +"@upstash/core-analytics@npm:^0.0.6": + version: 0.0.6 + resolution: "@upstash/core-analytics@npm:0.0.6" + dependencies: + "@upstash/redis": ^1.19.3 + checksum: 4d952984b1a7dd6c9b7d2ed6e597b6d64909ab3ed822088a0e014f6598f0c8cf25ac39ec6b72481c67ba9ea27afce1596c18ecb6e5f5f0837811cc600660f137 + languageName: node + linkType: hard + +"@upstash/ratelimit@npm:^0.4.3": + version: 0.4.3 + resolution: "@upstash/ratelimit@npm:0.4.3" + dependencies: + "@upstash/core-analytics": ^0.0.6 + checksum: d75c154abad949d90a82a62109e8c8511660fce22564e5de42b806a695f7a41526de91604df561b104b44edf4896fb4cb589d795781a427f31c8943b78c20b61 + languageName: node + linkType: hard + +"@upstash/redis@npm:^1.19.3, @upstash/redis@npm:^1.21.0": + version: 1.21.0 + resolution: "@upstash/redis@npm:1.21.0" + dependencies: + isomorphic-fetch: ^3.0.0 + checksum: ba2ba971c1f6d0297afddf73aba0817a39fcf8d71e51131030a24541e4564138a11bae50b78da7dd0eca5a11baa1322888688cb206f078603ea3a8d0a639a30e + languageName: node + linkType: hard + "@vercel/analytics@npm:^0.1.6": version: 0.1.11 resolution: "@vercel/analytics@npm:0.1.11" @@ -23963,6 +23992,16 @@ __metadata: languageName: node linkType: hard +"isomorphic-fetch@npm:^3.0.0": + version: 3.0.0 + resolution: "isomorphic-fetch@npm:3.0.0" + dependencies: + node-fetch: ^2.6.1 + whatwg-fetch: ^3.4.1 + checksum: e5ab79a56ce5af6ddd21265f59312ad9a4bc5a72cebc98b54797b42cb30441d5c5f8d17c5cd84a99e18101c8af6f90c081ecb8d12fd79e332be1778d58486d75 + languageName: node + linkType: hard + "isomorphic-unfetch@npm:^3.1.0": version: 3.1.0 resolution: "isomorphic-unfetch@npm:3.1.0"