feat: Upstash implementation for rate limiting/redis (#9514)

* Introduce rate limiting that works on the edge

* typo

* Log once on init

* Update rateLimit.ts

---------

Co-authored-by: zomars <zomars@me.com>
pull/9618/head^2
sean-brydon 2023-06-19 11:01:06 +01:00 committed by GitHub
parent ed65b2a3ab
commit 8eccd3658e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 124 additions and 44 deletions

View File

@ -23,6 +23,8 @@ CALCOM_LICENSE_KEY=
# - DATABASE ************************************************************************************************
DATABASE_URL="postgresql://postgres:@localhost:5450/calendso"
UPSTASH_REDIS_REST_URL=
UPSTASH_REDIS_REST_TOKEN=
# Uncomment to enable a dedicated connection pool for Prisma using Prisma Data Proxy
# Cold boots will be faster and you'll be able to scale your DB independently of your app.

View File

@ -61,6 +61,8 @@
"@tanstack/react-query": "^4.3.9",
"@tremor/react": "^2.0.0",
"@types/turndown": "^5.0.1",
"@upstash/ratelimit": "^0.4.3",
"@upstash/redis": "^1.21.0",
"@vercel/edge-config": "^0.1.1",
"@vercel/edge-functions-ui": "^0.2.1",
"@vercel/og": "^0.5.0",

View File

@ -5,15 +5,12 @@ import { z } from "zod";
import dayjs from "@calcom/dayjs";
import { sendPasswordResetEmail } from "@calcom/emails";
import { PASSWORD_RESET_EXPIRY_HOURS } from "@calcom/emails/templates/forgot-password-email";
import rateLimit from "@calcom/lib/rateLimit";
import { ErrorCode } from "@calcom/features/auth/lib/ErrorCode";
import rateLimiter from "@calcom/lib/rateLimit";
import { defaultHandler } from "@calcom/lib/server";
import { getTranslation } from "@calcom/lib/server/i18n";
import prisma from "@calcom/prisma";
const limiter = rateLimit({
intervalInMs: 60 * 1000, // 1 minute
});
async function handler(req: NextApiRequest, res: NextApiResponse) {
const t = await getTranslation(req.body.language ?? "en", "common");
@ -37,10 +34,13 @@ async function handler(req: NextApiRequest, res: NextApiResponse) {
// 10 requests per minute
try {
limiter.check(10, ip);
} catch (e) {
return res.status(429).json({ message: "Too Many Requests." });
const limiter = await rateLimiter();
const limit = await limiter({
identifier: ip,
});
if (!limit.success) {
throw new Error(ErrorCode.RateLimitExceeded);
}
try {

View File

@ -14,7 +14,7 @@ import { symmetricDecrypt } from "@calcom/lib/crypto";
import { defaultCookies } from "@calcom/lib/default-cookies";
import { isENVDev } from "@calcom/lib/env";
import { randomString } from "@calcom/lib/random";
import rateLimit from "@calcom/lib/rateLimit";
import rateLimiter from "@calcom/lib/rateLimit";
import slugify from "@calcom/lib/slugify";
import prisma from "@calcom/prisma";
import { IdentityProvider } from "@calcom/prisma/enums";
@ -102,11 +102,14 @@ const providers: Provider[] = [
if (!user) {
throw new Error(ErrorCode.IncorrectUsernamePassword);
}
const limiter = rateLimit({
intervalInMs: 60 * 1000, // 1 minute
const limiter = await rateLimiter();
const rateLimit = await limiter({
identifier: user.email,
});
await limiter.check(10, user.email); // 10 requests per minute
if (!rateLimit.success) {
throw new Error(ErrorCode.RateLimitExceeded);
}
if (user.identityProvider !== IdentityProvider.CAL && !credentials.totpCode) {
throw new Error(ErrorCode.ThirdPartyIdentityProviderEnabled);

View File

@ -4,17 +4,13 @@ import { sendEmailVerificationLink } from "@calcom/emails/email-manager";
import { getFeatureFlagMap } from "@calcom/features/flags/server/utils";
import { WEBAPP_URL } from "@calcom/lib/constants";
import logger from "@calcom/lib/logger";
import rateLimit from "@calcom/lib/rateLimit";
import rateLimiter from "@calcom/lib/rateLimit";
import { getTranslation } from "@calcom/lib/server/i18n";
import { prisma } from "@calcom/prisma";
import { TRPCError } from "@calcom/trpc/server";
const log = logger.getChildLogger({ prefix: [`[[Auth] `] });
const limiter = rateLimit({
intervalInMs: 60 * 1000, // 1 minute
});
interface VerifyEmailType {
username?: string;
email: string;
@ -43,9 +39,12 @@ export const sendEmailVerification = async ({ email, language, username }: Verif
token,
});
const { isRateLimited } = limiter.check(10, email); // 10 requests per minute
const limiter = await rateLimiter();
const rateLimit = await limiter({
identifier: email,
});
if (isRateLimited) {
if (!rateLimit.success) {
throw new TRPCError({
code: "TOO_MANY_REQUESTS",
message: "An unexpected error occurred, please try again later.",

View File

@ -28,3 +28,13 @@ export function isIpInBanlist(request: Request | NextApiRequest) {
}
return false;
}
export function isIpInBanListString(identifer: string) {
const rawBanListJson = process.env.IP_BANLIST || "[]";
const banList = banlistSchema.parse(JSON.parse(rawBanListJson));
if (banList.includes(identifer)) {
console.log(`Found banned IP: ${identifer} in IP_BANLIST`);
return true;
}
return false;
}

View File

@ -1,27 +1,49 @@
/* eslint-disable @typescript-eslint/no-explicit-any */
import cache from "memory-cache";
import { Ratelimit } from "@upstash/ratelimit";
import { Redis } from "@upstash/redis";
import { ErrorCode } from "@calcom/features/auth/lib/ErrorCode";
import { isIpInBanListString } from "./getIP";
import logger from "./logger";
const rateLimit = (options: { intervalInMs: number }) => {
return {
check: (requestLimit: number, uniqueIdentifier: string) => {
const count = cache.get(uniqueIdentifier) || [0];
if (count[0] === 0) {
cache.put(uniqueIdentifier, count, options.intervalInMs);
}
count[0] += 1;
const log = logger.getChildLogger({ prefix: ["RateLimit"] });
const currentUsage = count[0];
const isRateLimited = currentUsage >= requestLimit;
if (isRateLimited) {
throw new Error(ErrorCode.RateLimitExceeded);
}
return { isRateLimited, requestLimit, remaining: isRateLimited ? 0 : requestLimit - currentUsage };
},
};
type RateLimitHelper = {
rateLimitingType?: "core" | "forcedSlowMode";
identifier: string;
};
export default rateLimit;
function rateLimiter() {
const UPSATCH_ENV_FOUND = process.env.UPSTASH_REDIS_REST_URL && process.env.UPSTASH_REDIS_REST_TOKEN;
if (!UPSATCH_ENV_FOUND) {
log.warn("Disabled due to not finding UPSTASH env variables");
return () => ({ success: true });
}
const redis = Redis.fromEnv();
const limiter = {
core: new Ratelimit({
redis,
analytics: true,
prefix: "ratelimit",
limiter: Ratelimit.fixedWindow(10, "60s"),
}),
forcedSlowMode: new Ratelimit({
redis,
analytics: true,
prefix: "ratelimit:slowmode",
limiter: Ratelimit.fixedWindow(1, "30s"),
}),
};
async function rateLimit({ rateLimitingType = "core", identifier }: RateLimitHelper) {
if (isIpInBanListString(identifier)) {
return await limiter.forcedSlowMode.limit(identifier);
}
return await limiter[rateLimitingType].limit(identifier);
}
return rateLimit;
}
export default rateLimiter;

View File

@ -175,9 +175,9 @@
"globalDependencies": ["yarn.lock"],
"globalEnv": [
"ANALYZE",
"AUTH_BEARER_TOKEN_VERCEL",
"API_KEY_PREFIX",
"APP_USER_NAME",
"AUTH_BEARER_TOKEN_VERCEL",
"BUILD_ID",
"CALCOM_LICENSE_KEY",
"CALCOM_TELEMETRY_DISABLED",
@ -204,6 +204,7 @@
"HUBSPOT_CLIENT_SECRET",
"INTEGRATION_TEST_MODE",
"INTERCOM_SECRET",
"INTERCOM_SECRET",
"IP_BANLIST",
"LARK_OPEN_APP_ID",
"LARK_OPEN_APP_SECRET",
@ -272,6 +273,8 @@
"TWILIO_SID",
"TWILIO_TOKEN",
"TWILIO_VERIFY_SID",
"UPSTASH_REDIS_REST_TOKEN",
"UPSTASH_REDIS_REST_URL",
"VERCEL_ENV",
"VERCEL_URL",
"VITAL_API_KEY",

View File

@ -4896,6 +4896,8 @@ __metadata:
"@types/stripe": ^8.0.417
"@types/turndown": ^5.0.1
"@types/uuid": 8.3.1
"@upstash/ratelimit": ^0.4.3
"@upstash/redis": ^1.21.0
"@vercel/edge-config": ^0.1.1
"@vercel/edge-functions-ui": ^0.2.1
"@vercel/og": ^0.5.0
@ -13331,6 +13333,33 @@ __metadata:
languageName: node
linkType: hard
"@upstash/core-analytics@npm:^0.0.6":
version: 0.0.6
resolution: "@upstash/core-analytics@npm:0.0.6"
dependencies:
"@upstash/redis": ^1.19.3
checksum: 4d952984b1a7dd6c9b7d2ed6e597b6d64909ab3ed822088a0e014f6598f0c8cf25ac39ec6b72481c67ba9ea27afce1596c18ecb6e5f5f0837811cc600660f137
languageName: node
linkType: hard
"@upstash/ratelimit@npm:^0.4.3":
version: 0.4.3
resolution: "@upstash/ratelimit@npm:0.4.3"
dependencies:
"@upstash/core-analytics": ^0.0.6
checksum: d75c154abad949d90a82a62109e8c8511660fce22564e5de42b806a695f7a41526de91604df561b104b44edf4896fb4cb589d795781a427f31c8943b78c20b61
languageName: node
linkType: hard
"@upstash/redis@npm:^1.19.3, @upstash/redis@npm:^1.21.0":
version: 1.21.0
resolution: "@upstash/redis@npm:1.21.0"
dependencies:
isomorphic-fetch: ^3.0.0
checksum: ba2ba971c1f6d0297afddf73aba0817a39fcf8d71e51131030a24541e4564138a11bae50b78da7dd0eca5a11baa1322888688cb206f078603ea3a8d0a639a30e
languageName: node
linkType: hard
"@vercel/analytics@npm:^0.1.6":
version: 0.1.11
resolution: "@vercel/analytics@npm:0.1.11"
@ -23963,6 +23992,16 @@ __metadata:
languageName: node
linkType: hard
"isomorphic-fetch@npm:^3.0.0":
version: 3.0.0
resolution: "isomorphic-fetch@npm:3.0.0"
dependencies:
node-fetch: ^2.6.1
whatwg-fetch: ^3.4.1
checksum: e5ab79a56ce5af6ddd21265f59312ad9a4bc5a72cebc98b54797b42cb30441d5c5f8d17c5cd84a99e18101c8af6f90c081ecb8d12fd79e332be1778d58486d75
languageName: node
linkType: hard
"isomorphic-unfetch@npm:^3.1.0":
version: 3.1.0
resolution: "isomorphic-unfetch@npm:3.1.0"