Merge main into next

.changeset/blue-moons-travel.md

@@ -0,0 +1,33 @@
+---
+"@comet/cms-api": minor
+---
+
+Set content scopes in request object
+
+This allows accessing the affected content scopes inside a block's transformer service.
+
+**Example**
+
+```ts
+import { Inject, Injectable } from "@nestjs/common";
+import { CONTEXT } from "@nestjs/graphql";
+
+/* ... */
+
+@Injectable()
+export class PixelImageBlockTransformerService implements BlockTransformerServiceInterface<PixelImageBlockData, TransformResponse> {
+    constructor(
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        @Inject(CONTEXT) private readonly context: any,
+    ) {}
+
+    async transformToPlain(block: PixelImageBlockData) {
+        // Get the affected content scopes
+        const contentScopes = this.context.req.contentScopes;
+
+        // Do something with the content scopes
+
+        /* ... */
+    }
+}
+```

.changeset/soft-spoons-deliver.md

@@ -0,0 +1,7 @@
+---
+"@comet/cms-site": minor
+---
+
+gql: Handle non-string variables in GraphQL documents
+
+Non-string variables were incorrectly converted to strings, e.g., `'[object Object]'`. This error usually occurred when trying to import a GraphQL fragment from a React Client Component. The `gql` helper now throws an error for non-string variables.

.changeset/spotty-pigs-reply.md

@@ -0,0 +1,5 @@
+---
+"@comet/cms-site": patch
+---
+
+GraphQLFetch: Correctly report GraphQL schema validation errors

.env

@@ -78,3 +78,9 @@ TRACING_ENABLED=1
 
 # file uploads
 FILE_UPLOADS_DOWNLOAD_SECRET=gPM8DTrrAdCMPYaNM99sH6hgtJfPWuEV
+
+# redis
+REDIS_ENABLED=true
+REDIS_PORT=6379
+REDIS_HOST=localhost
+REDIS_PASSWORD=vivid

demo/site/cache-handler.ts

@@ -0,0 +1,136 @@
+/* eslint-disable no-console */
+import { Redis } from "ioredis";
+import { LRUCache } from "lru-cache";
+import { CacheHandler as NextCacheHandler } from "next/dist/server/lib/incremental-cache";
+
+const REDIS_HOST = process.env.REDIS_HOST;
+if (!REDIS_HOST) {
+    throw new Error("REDIS_HOST is required");
+}
+
+const REDIS_PORT = parseInt(process.env.REDIS_PORT || "6379", 10);
+
+const REDIS_PASSWORD = process.env.REDIS_PASSWORD;
+if (!REDIS_PASSWORD) {
+    throw new Error("REDIS_PASSWORD is required");
+}
+
+const REDIS_KEY_PREFIX = process.env.REDIS_KEY_PREFIX || "";
+
+const REDIS_ENABLE_AUTOPIPELINING = process.env.REDIS_ENABLE_AUTOPIPELINING === "true";
+
+const CACHE_HANDLER_DEBUG = process.env.CACHE_HANDLER_DEBUG === "true";
+
+const CACHE_TTL_IN_S = 24 * 60 * 60; // 1 day
+
+const redis = new Redis({
+    commandTimeout: 1000,
+    enableOfflineQueue: false,
+    host: REDIS_HOST,
+    keyPrefix: REDIS_KEY_PREFIX,
+    password: REDIS_PASSWORD,
+    port: REDIS_PORT,
+    socketTimeout: 1000,
+    enableAutoPipelining: REDIS_ENABLE_AUTOPIPELINING, // https://github.com/redis/ioredis?tab=readme-ov-file#autopipelining
+});
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+const fallbackCache = new LRUCache<string, any>({
+    maxSize: 50 * 1024 * 1024, // 50MB
+    ttl: CACHE_TTL_IN_S * 1000,
+    ttlAutopurge: true,
+});
+
+let isFallbackInUse = false;
+
+function parseBodyForGqlError(body: string) {
+    try {
+        const decodedBody = Buffer.from(body, "base64").toString("utf-8");
+        if (!decodedBody.startsWith("{")) return null; // Not a JSON response, ignore
+        return JSON.parse(decodedBody);
+    } catch (error) {
+        console.error("CacheHandler.parseBodyForGqlError error", error);
+        return null;
+    }
+}
+
+export default class CacheHandler extends NextCacheHandler {
+    //constructor(_ctx: NextCacheHandlerContext) {}
+
+    async get(
+        key: string,
+        //ctx: Parameters<NextCacheHandler["get"]>[1]
+    ): ReturnType<NextCacheHandler["get"]> {
+        if (redis.status === "ready") {
+            try {
+                if (CACHE_HANDLER_DEBUG) {
+                    console.log("CacheHandler.get redis", key);
+                }
+
+                const redisResponse = await redis.get(key);
+                if (isFallbackInUse) {
+                    isFallbackInUse = false;
+                    console.info(`${new Date().toISOString()} [${REDIS_HOST} up] Switching back to redis cache`);
+                }
+                if (!redisResponse) {
+                    return null;
+                }
+                return JSON.parse(redisResponse);
+            } catch (e) {
+                console.error("CacheHandler.get error", e);
+            }
+        }
+
+        if (CACHE_HANDLER_DEBUG) {
+            console.log("CacheHandler.get fallbackCache", key);
+        }
+
+        // fallback to in-memory cache
+        if (!isFallbackInUse) {
+            console.warn(`${new Date().toISOString()} | [${REDIS_HOST} down] switching to fallback in-memory cache`);
+            isFallbackInUse = true;
+        }
+
+        return fallbackCache.get(key) ?? null;
+    }
+
+    async set(
+        key: string,
+        value: Parameters<NextCacheHandler["set"]>[1],
+        // ctx: Parameters<NextCacheHandler["set"]>[2],
+    ): Promise<void> {
+        if (value?.kind === "FETCH") {
+            const responseBody = parseBodyForGqlError(value.data.body);
+            if (responseBody?.errors) {
+                // Must not cache GraphQL errors
+                console.error("CacheHandler.set GraphQL Error: ", responseBody.error);
+                return;
+            }
+        }
+
+        const stringData = JSON.stringify({
+            lastModified: Date.now(),
+            value,
+        });
+
+        if (redis.status === "ready") {
+            try {
+                if (CACHE_HANDLER_DEBUG) {
+                    console.log("CacheHandler.set redis", key);
+                }
+                await redis.set(key, stringData, "EX", CACHE_TTL_IN_S);
+            } catch (e) {
+                console.error("CacheHandler.set error", e);
+            }
+        }
+        if (CACHE_HANDLER_DEBUG) {
+            console.log("CacheHandler.set fallbackCache", key);
+        }
+        fallbackCache.set(key, value, { size: stringData.length });
+    }
+
+    async revalidateTag(tags: string | string[]): Promise<void> {
+        console.log("CacheHandler.revalidateTag", tags);
+        throw new Error("unsupported");
+    }
+}

demo/site/next.config.js

@@ -34,6 +34,8 @@ const nextConfig = {
     experimental: {
         optimizePackageImports: ["@comet/cms-site"],
     },
+    cacheHandler: process.env.REDIS_ENABLED === "true" ? require.resolve("./dist/cache-handler.js") : undefined,
+    cacheMaxMemorySize: process.env.REDIS_ENABLED === "true" ? 0 : undefined, // disable default in-memory caching
 };
 
 module.exports = withBundleAnalyzer(nextConfig);

demo/site/package.json

@@ -4,7 +4,7 @@
     "private": true,
     "scripts": {
         "build": "run-s intl:compile && run-p gql:types generate-block-types && tsc --project tsconfig.server.json && next build",
-        "dev": "run-s intl:compile && run-p gql:types generate-block-types && NODE_OPTIONS='--inspect=localhost:9230' ts-node --project tsconfig.server.json server.ts",
+        "dev": "run-s intl:compile && run-p gql:types generate-block-types && tsc --project tsconfig.server.json && NODE_OPTIONS='--inspect=localhost:9230' node dist/server.js",
         "export": "next export",
         "generate-block-types": "comet generate-block-types",
         "generate-block-types:watch": "chokidar -s \"**/block-meta.json\" -c \"$npm_execpath generate-block-types\"",
@@ -33,6 +33,8 @@
         "fs-extra": "^9.0.0",
         "graphql": "^15.0.0",
         "graphql-tag": "^2.12.6",
+        "ioredis": "^5.4.1",
+        "lru-cache": "^11.0.1",
         "next": "^14.2.12",
         "pure-react-carousel": "^1.0.0",
         "react": "^18.2.0",

demo/site/tsconfig.server.json

@@ -8,5 +8,5 @@
         "isolatedModules": false,
         "noEmit": false
     },
-    "include": ["server.ts", "tracing.ts"]
+    "include": ["server.ts", "tracing.ts", "cache-handler.ts"]
 }

docker-compose.yml

@@ -43,9 +43,19 @@ services:
             COLLECTOR_OTLP_ENABLED: "true"
             COLLECTOR_OTLP_HTTP_HOST_PORT: 0.0.0.0:4318
 
+    redis:
+        image: redis:7
+        command: redis-server --maxmemory 256M --maxmemory-policy allkeys-lru --loglevel warning --requirepass ${REDIS_PASSWORD}
+        ports:
+            - ${REDIS_PORT}:6379
+        networks:
+            - redis
+
 networks:
     postgres:
         driver: bridge
+    redis:
+        driver: bridge
 
 volumes:
     postgres:

packages/api/cms-api/src/common/decorators/utils.ts

@@ -1,8 +1,7 @@
 import { ExecutionContext } from "@nestjs/common";
 import { GqlExecutionContext } from "@nestjs/graphql";
-import { Request } from "express";
 
-export const getRequestFromExecutionContext = (ctx: ExecutionContext): Request => {
+export const getRequestFromExecutionContext = (ctx: ExecutionContext) => {
     if (ctx.getType().toString() === "graphql") {
         return GqlExecutionContext.create(ctx).getContext().req;
     }

packages/api/cms-api/src/user-permissions/auth/user-permissions.guard.ts

@@ -2,6 +2,7 @@ import { CanActivate, ExecutionContext, Inject, Injectable } from "@nestjs/commo
 import { Reflector } from "@nestjs/core";
 import { GqlContextType, GqlExecutionContext } from "@nestjs/graphql";
 
+import { getRequestFromExecutionContext } from "../../common/decorators/utils";
 import { ContentScopeService } from "../content-scope.service";
 import { DisablePermissionCheck, RequiredPermissionMetadata } from "../decorators/required-permission.decorator";
 import { CurrentUser } from "../dto/current-user";
@@ -20,6 +21,14 @@ export class UserPermissionsGuard implements CanActivate {
     async canActivate(context: ExecutionContext): Promise<boolean> {
         const location = `${context.getClass().name}::${context.getHandler().name}()`;
 
+        const requiredContentScopes = await this.contentScopeService.getScopesForPermissionCheck(context);
+
+        // Ignore field resolvers as they have no scopes and would overwrite the scopes of the root query.
+        if (!this.isResolvingGraphQLField(context)) {
+            const request = getRequestFromExecutionContext(context);
+            request.contentScopes = this.contentScopeService.getUniqueScopes(requiredContentScopes);
+        }
+
         if (this.getDecorator(context, "disableCometGuards")) return true;
 
         const user = this.getUser(context);
@@ -38,7 +47,6 @@ export class UserPermissionsGuard implements CanActivate {
             // At least one permission is required
             return requiredPermissions.some((permission) => this.accessControlService.isAllowed(user, permission));
         } else {
-            const requiredContentScopes = await this.contentScopeService.getScopesForPermissionCheck(context);
             if (requiredContentScopes.length === 0)
                 throw new Error(
                     `Could not get content scope. Either pass a scope-argument or add an @AffectedEntity()-decorator or enable skipScopeCheck in the @RequiredPermission()-decorator of ${location}`,

packages/api/cms-api/src/user-permissions/content-scope.service.ts

@@ -43,7 +43,19 @@ export class ContentScopeService {
     }
 
     async inferScopesFromExecutionContext(context: ExecutionContext): Promise<ContentScope[]> {
-        return [...new Set((await this.getScopesForPermissionCheck(context)).flat())];
+        return this.getUniqueScopes(await this.getScopesForPermissionCheck(context));
+    }
+
+    getUniqueScopes(scopes: ContentScope[][]): ContentScope[] {
+        const uniqueScopes: ContentScope[] = [];
+
+        scopes.flat().forEach((incomingScope) => {
+            if (!uniqueScopes.some((existingScope) => this.scopesAreEqual(existingScope, incomingScope))) {
+                uniqueScopes.push(incomingScope);
+            }
+        });
+
+        return uniqueScopes;
     }
 
     private async getContentScopesFromEntity(affectedEntity: AffectedEntityMeta, args: Record<string, string>): Promise<ContentScope[][]> {

packages/site/cms-site/src/graphQLFetch/graphQLFetch.ts

@@ -30,9 +30,32 @@ function graphQLHeaders(previewData?: SitePreviewData) {
     return headers;
 }
 
-//from graphql-request https://github.com/jasonkuhrt/graphql-request/blob/main/src/raw/functions/gql.ts
+// Adapted from https://github.com/jasonkuhrt/graffle/blob/main/src/layers/6_helpers/gql.ts
 export const gql = (chunks: TemplateStringsArray, ...variables: unknown[]): string => {
-    return chunks.reduce((acc, chunk, index) => `${acc}${chunk}${index in variables ? String(variables[index]) : ``}`, ``);
+    return chunks.reduce((acc, chunk, index) => {
+        let variable;
+
+        if (index in variables) {
+            if (typeof variables[index] !== "string") {
+                let errorMessage =
+                    `Non-string variable in the GraphQL document\n\n` +
+                    `This is most likely due to importing a GraphQL document from a React Client Component.\n` +
+                    `All GraphQL documents need to be imported from React Server Components (i.e. no "use client" notation).`;
+
+                if (chunk.trim().length > 0) {
+                    errorMessage += `\n\nThe error occurred in the following GraphQL document:\n${chunk}`;
+                }
+
+                throw new Error(errorMessage);
+            } else {
+                variable = variables[index];
+            }
+        } else {
+            variable = "";
+        }
+
+        return `${acc}${chunk}${variable}`;
+    }, ``);
 };
 
 export function createFetchWithPreviewHeaders(fetch: Fetch, previewData?: SitePreviewData): Fetch {
@@ -79,13 +102,28 @@ export function createGraphQLFetch(fetch: Fetch, url: string): GraphQLFetch {
             });
         }
         if (!response.ok) {
-            throw new Error(`Network response was not ok. Status: ${response.status}`);
+            let errorMessage = `Network response was not ok. Status: ${response.status}`;
+            const body = await response.text();
+
+            try {
+                const json = JSON.parse(body);
+
+                const { errors } = json;
+                if (errors) {
+                    errorMessage += `\n\nGraphQL error(s):\n- ${errors.map((error: { message: string }) => error.message).join("\n- ")}`;
+                }
+            } catch (error) {
+                errorMessage += `\n${body}`;
+            }
+
+            throw new Error(errorMessage);
         }
+
         const { data, errors } = await response.json();
+
         if (errors) {
             // eslint-disable-next-line @typescript-eslint/no-explicit-any
-            const errorMessage = errors.map((error: any) => error.message).join("\n");
-            throw new Error(`GraphQL Error: ${errorMessage}`);
+            throw new Error(`GraphQL error(s):\n- ${errors.map((error: any) => error.message).join("\n- ")}`);
         }
 
         return data;