Implement legacy collections using glob

.changeset/quick-onions-leave.md

@@ -0,0 +1,34 @@
+---
+'astro': patch
+---
+
+Implements legacy content and data collections using `glob()` loader
+
+:warning: **BREAKING CHANGE FOR LEGACY CONTENT COLLECTIONS** :warning:
+
+By default, collections that use the the old types (content or data) are now implemented using the glob loader, with extra backward-compat handling. This includes any collection without a `loader` defined.
+
+Any legacy content collections are handled like this:
+
+- a `glob` loader collection is defined, with patterns that match the previous handling (matches `src/content/<collection name>/**/*.md` and other content extensions depending on installed integrations, with underscore-prefixed files and folders ignored)
+- When used in the runtime, the entries have an ID based on the filename in the same format as legacy collections
+- A `slug` field is added with the same format as before
+- A `render()` method is added to the entry, so they can be called using `entry.render()`
+- `getEntryBySlug` is supported
+
+Legacy data collections are handled like this:
+
+- a `glob` loader collection is defined, with patterns that match the previous handling (matches `src/content/<collection name>/**/*{.json,.yaml}` and other data extensions, with underscore-prefixed files and folders ignored)
+- Entries have an ID that is not slugified
+- `getDataEntryById` is supported
+
+While these emulate most of the features of legacy collections, they have these differences:
+
+- No implicit collections. In order to be generated, a collection must be defined in `config.ts`. For legacy collections these can just be empty declarations: e.g.`const blog = defineCollection({})`. Removing implicit collections means that we can allow content layer collections in `src/content`.
+- The `layout` field is not supported in Markdown
+- Experimental content collection cache is not supported
+- Sort order of generated collections is non-deterministic and platform-dependent.
+- `image().refine()` is not supported
+- the key for `getEntry` is typed as `string`, rather than having types for every entry.
+
+A new config flag `legacy.legacyContentCollections` is added for users that need the old behavior. When set, collections in `src/content` are processed in the same way as before rather than being implemented with glob - including implicit collections. When set, content layer collections are forbidden in `src/content`, and will fail a build if defined.

examples/with-markdoc/src/content/config.ts

@@ -0,0 +1,5 @@
+import { defineCollection } from 'astro:content';
+
+export const collections = {
+	docs: defineCollection({})
+};

packages/astro/e2e/fixtures/content-collections/src/content/config.ts

@@ -0,0 +1,6 @@
+import { defineCollection } from "astro:content";
+
+
+const posts = defineCollection({});
+
+export const collections = { posts };

packages/astro/src/content/data-store.ts

@@ -34,6 +34,8 @@ export interface DataEntry<TData extends Record<string, unknown> = Record<string
 	 */
 	deferredRender?: boolean;
 	assetImports?: Array<string>;
+	/** @deprecated */
+	legacyId?: string;
 }
 
 /**

packages/astro/src/content/loaders/glob.ts

@@ -35,7 +35,7 @@ function generateIdDefault({ entry, base, data }: GenerateIdOptions): string {
 	if (data.slug) {
 		return data.slug as string;
 	}
-	const entryURL = new URL(entry, base);
+	const entryURL = new URL(encodeURI(entry), base);
 	const { slug } = getContentEntryIdAndSlug({
 		entry: entryURL,
 		contentDir: base,
@@ -55,6 +55,15 @@ function checkPrefix(pattern: string | Array<string>, prefix: string) {
  * Loads multiple entries, using a glob pattern to match files.
  * @param pattern A glob pattern to match files, relative to the content directory.
  */
+export function glob(globOptions: GlobOptions): Loader;
+/** @private */
+export function glob(
+	globOptions: GlobOptions & {
+		/** @deprecated */
+		_legacy?: true;
+	},
+): Loader;
+
 export function glob(globOptions: GlobOptions): Loader {
 	if (checkPrefix(globOptions.pattern, '../')) {
 		throw new Error(
@@ -80,19 +89,21 @@ export function glob(globOptions: GlobOptions): Loader {
 			>();
 
 			const untouchedEntries = new Set(store.keys());
-
+			const isLegacy = (globOptions as any)._legacy;
+			// If legacy mode is *not* enabled then we use emulate legacy collections instead
+			const emulateLegacyCollections = !config.legacy.legacyContentCollections;
 			async function syncData(entry: string, base: URL, entryType?: ContentEntryType) {
 				if (!entryType) {
 					logger.warn(`No entry type found for ${entry}`);
 					return;
 				}
-				const fileUrl = new URL(entry, base);
+				const fileUrl = new URL(encodeURI(entry), base);
 				const contents = await fs.readFile(fileUrl, 'utf-8').catch((err) => {
 					logger.error(`Error reading ${entry}: ${err.message}`);
 					return;
 				});
 
-				if (!contents) {
+				if (!contents && contents !== '') {
 					logger.warn(`No contents found for ${entry}`);
 					return;
 				}
@@ -103,6 +114,17 @@ export function glob(globOptions: GlobOptions): Loader {
 				});
 
 				const id = generateId({ entry, base, data });
+				let legacyId: string | undefined;
+
+				if (isLegacy) {
+					const entryURL = new URL(encodeURI(entry), base);
+					const legacyOptions = getContentEntryIdAndSlug({
+						entry: entryURL,
+						contentDir: base,
+						collection: '',
+					});
+					legacyId = legacyOptions.id;
+				}
 				untouchedEntries.delete(id);
 
 				const existingEntry = store.get(id);
@@ -132,6 +154,12 @@ export function glob(globOptions: GlobOptions): Loader {
 					filePath,
 				});
 				if (entryType.getRenderFunction) {
+					if (isLegacy && data.layout) {
+						logger.error(
+							`The Markdown "layout" field is not supported in content collections in Astro 5. Ignoring layout for ${JSON.stringify(entry)}. Enable "legacy.legacyContentCollections" if you need to use the layout field.`,
+						);
+					}
+
 					let render = renderFunctionByContentType.get(entryType);
 					if (!render) {
 						render = await entryType.getRenderFunction(config);
@@ -160,6 +188,7 @@ export function glob(globOptions: GlobOptions): Loader {
 						digest,
 						rendered,
 						assetImports: rendered?.metadata?.imagePaths,
+						legacyId,
 					});
 
 					// todo: add an explicit way to opt in to deferred rendering
@@ -171,9 +200,10 @@ export function glob(globOptions: GlobOptions): Loader {
 						filePath: relativePath,
 						digest,
 						deferredRender: true,
+						legacyId,
 					});
 				} else {
-					store.set({ id, data: parsedData, body, filePath: relativePath, digest });
+					store.set({ id, data: parsedData, body, filePath: relativePath, digest, legacyId });
 				}
 
 				fileToIdMap.set(filePath, id);
@@ -222,7 +252,7 @@ export function glob(globOptions: GlobOptions): Loader {
 					if (isConfigFile(entry)) {
 						return;
 					}
-					if (isInContentDir(entry)) {
+					if (!emulateLegacyCollections && isInContentDir(entry)) {
 						skippedFiles.push(entry);
 						return;
 					}
@@ -240,7 +270,9 @@ export function glob(globOptions: GlobOptions): Loader {
 					? globOptions.pattern.join(', ')
 					: globOptions.pattern;
 
-				logger.warn(`The glob() loader cannot be used for files in ${bold('src/content')}.`);
+				logger.warn(
+					`The glob() loader cannot be used for files in ${bold('src/content')} when legacy mode is enabled.`,
+				);
 				if (skipCount > 10) {
 					logger.warn(
 						`Skipped ${green(skippedFiles.length)} files that matched ${green(patternList)}.`,

packages/astro/src/content/mutable-data-store.ts

@@ -4,7 +4,7 @@ import { Traverse } from 'neotraverse/modern';
 import { imageSrcToImportId, importIdToSymbolName } from '../assets/utils/resolveImports.js';
 import { AstroError, AstroErrorData } from '../core/errors/index.js';
 import { IMAGE_IMPORT_PREFIX } from './consts.js';
-import { type DataEntry, ImmutableDataStore, type RenderedContent } from './data-store.js';
+import { type DataEntry, ImmutableDataStore } from './data-store.js';
 import { contentModuleToId } from './utils.js';
 
 const SAVE_DEBOUNCE_MS = 500;
@@ -197,7 +197,17 @@ export default new Map([\n${lines.join(',\n')}]);
 			entries: () => this.entries(collectionName),
 			values: () => this.values(collectionName),
 			keys: () => this.keys(collectionName),
-			set: ({ id: key, data, body, filePath, deferredRender, digest, rendered, assetImports }) => {
+			set: ({
+				id: key,
+				data,
+				body,
+				filePath,
+				deferredRender,
+				digest,
+				rendered,
+				assetImports,
+				legacyId,
+			}) => {
 				if (!key) {
 					throw new Error(`ID must be a non-empty string`);
 				}
@@ -244,6 +254,9 @@ export default new Map([\n${lines.join(',\n')}]);
 				if (rendered) {
 					entry.rendered = rendered;
 				}
+				if (legacyId) {
+					entry.legacyId = legacyId;
+				}
 				if (deferredRender) {
 					entry.deferredRender = deferredRender;
 					if (filePath) {
@@ -335,30 +348,7 @@ export interface DataStore {
 		key: string,
 	) => DataEntry<TData> | undefined;
 	entries: () => Array<[id: string, DataEntry]>;
-	set: <TData extends Record<string, unknown>>(opts: {
-		/** The ID of the entry. Must be unique per collection. */
-		id: string;
-		/** The data to store. */
-		data: TData;
-		/** The raw body of the content, if applicable. */
-		body?: string;
-		/** The file path of the content, if applicable. Relative to the site root. */
-		filePath?: string;
-		/** A content digest, to check if the content has changed. */
-		digest?: number | string;
-		/** The rendered content, if applicable. */
-		rendered?: RenderedContent;
-		/**
-		 * If an entry is a deferred, its rendering phase is delegated to a virtual module during the runtime phase.
-		 */
-		deferredRender?: boolean;
-		/**
-		 * Assets such as images to process during the build. These should be files on disk, with a path relative to filePath.
-		 * Any values that use image() in the schema will already be added automatically.
-		 * @internal
-		 */
-		assetImports?: Array<string>;
-	}) => boolean;
+	set: <TData extends Record<string, unknown>>(opts: DataEntry<TData>) => boolean;
 	values: () => Array<DataEntry>;
 	keys: () => Array<string>;
 	delete: (key: string) => void;

packages/astro/src/content/runtime.ts

@@ -94,7 +94,7 @@ export function createGetCollection({
 				if (hasFilter && !filter(entry)) {
 					continue;
 				}
-				result.push(entry);
+				result.push(entry.legacyId ? emulateLegacyEntry(entry) : entry);
 			}
 			return result;
 		} else {
@@ -162,23 +162,31 @@ export function createGetEntryBySlug({
 	getEntryImport,
 	getRenderEntryImport,
 	collectionNames,
+	getEntry,
 }: {
 	getEntryImport: GetEntryImport;
 	getRenderEntryImport: GetEntryImport;
 	collectionNames: Set<string>;
+	getEntry: ReturnType<typeof createGetEntry>;
 }) {
 	return async function getEntryBySlug(collection: string, slug: string) {
 		const store = await globalDataStore.get();
 
 		if (!collectionNames.has(collection)) {
 			if (store.hasCollection(collection)) {
+				const entry = await getEntry(collection, slug);
+				if (entry && 'slug' in entry) {
+					return entry;
+				}
 				throw new AstroError({
 					...AstroErrorData.GetEntryDeprecationError,
 					message: AstroErrorData.GetEntryDeprecationError.message(collection, 'getEntryBySlug'),
 				});
 			}
 			// eslint-disable-next-line no-console
-			console.warn(`The collection ${JSON.stringify(collection)} does not exist.`);
+			console.warn(
+				`The collection ${JSON.stringify(collection)} does not exist. Please ensure it is defined in your content config.`,
+			);
 			return undefined;
 		}
 
@@ -207,22 +215,23 @@ export function createGetEntryBySlug({
 export function createGetDataEntryById({
 	getEntryImport,
 	collectionNames,
+	getEntry,
 }: {
 	getEntryImport: GetEntryImport;
 	collectionNames: Set<string>;
+	getEntry: ReturnType<typeof createGetEntry>;
 }) {
 	return async function getDataEntryById(collection: string, id: string) {
 		const store = await globalDataStore.get();
 
 		if (!collectionNames.has(collection)) {
 			if (store.hasCollection(collection)) {
-				throw new AstroError({
-					...AstroErrorData.GetEntryDeprecationError,
-					message: AstroErrorData.GetEntryDeprecationError.message(collection, 'getDataEntryById'),
-				});
+				return getEntry(collection, id);
 			}
 			// eslint-disable-next-line no-console
-			console.warn(`The collection ${JSON.stringify(collection)} does not exist.`);
+			console.warn(
+				`The collection ${JSON.stringify(collection)} does not exist. Please ensure it is defined in your content config.`,
+			);
 			return undefined;
 		}
 
@@ -256,6 +265,14 @@ type DataEntryResult = {
 
 type EntryLookupObject = { collection: string; id: string } | { collection: string; slug: string };
 
+function emulateLegacyEntry({ legacyId, id, ...entry }: DataEntry) {
+	const legacyEntry = { ...entry, id: legacyId!, slug: id };
+	return {
+		...legacyEntry,
+		render: () => renderEntry(legacyEntry),
+	};
+}
+
 export function createGetEntry({
 	getEntryImport,
 	getRenderEntryImport,
@@ -303,6 +320,9 @@ export function createGetEntry({
 			// @ts-expect-error	virtual module
 			const { default: imageAssetMap } = await import('astro:asset-imports');
 			entry.data = updateImageReferencesInData(entry.data, entry.filePath, imageAssetMap);
+			if (entry.legacyId) {
+				return { ...emulateLegacyEntry(entry), collection } as ContentEntryResult;
+			}
 			return {
 				...entry,
 				collection,
@@ -311,7 +331,9 @@ export function createGetEntry({
 
 		if (!collectionNames.has(collection)) {
 			// eslint-disable-next-line no-console
-			console.warn(`The collection ${JSON.stringify(collection)} does not exist.`);
+			console.warn(
+				`The collection ${JSON.stringify(collection)} does not exist. Please ensure it is defined in your content config.`,
+			);
 			return undefined;
 		}
 
@@ -433,9 +455,12 @@ function updateImageReferencesInData<T extends Record<string, unknown>>(
 }
 
 export async function renderEntry(
-	entry: DataEntry | { render: () => Promise<{ Content: AstroComponentFactory }> },
+	entry:
+		| DataEntry
+		| { render: () => Promise<{ Content: AstroComponentFactory }> }
+		| (DataEntry & { render: () => Promise<{ Content: AstroComponentFactory }> }),
 ) {
-	if (entry && 'render' in entry) {
+	if (entry && 'render' in entry && !('legacyId' in entry)) {
 		// This is an old content collection entry, so we use its render method
 		return entry.render();
 	}
@@ -623,13 +648,13 @@ export function createReference({ lookupMap }: { lookupMap: ContentLookupMap })
 						return { id: lookup, collection };
 					}
 
-					if (!lookupMap[collection] && store.collections().size === 0) {
+					if (!lookupMap[collection] && store.collections().size < 2) {
 						// If the collection is not in the lookup map or store, it may be a content layer collection and the store may not yet be populated.
+						// The store may still have a single collection, which would be the top level metadata collection.
 						// For now, we can't validate this reference, so we'll optimistically convert it to a reference object which we'll validate
 						// later in the pipeline when we do have access to the store.
 						return { id: lookup, collection };
 					}
-
 					const { type, entries } = lookupMap[collection];
 					const entry = entries[lookup];
 

packages/astro/src/content/types-generator.ts

@@ -498,7 +498,10 @@ async function writeContentFiles({
 				contentTypesStr += `};\n`;
 				break;
 			case CONTENT_LAYER_TYPE:
-				dataTypesStr += `${collectionKey}: Record<string, {\n  id: string;\n  collection: ${collectionKey};\n  data: ${dataType};\n  rendered?: RenderedContent \n}>;\n`;
+				const legacyTypes = (collectionConfig as any)?._legacy
+					? 'render(): Render[".md"];\n  slug: string;\n  body: string;\n'
+					: 'body?: string;\n';
+				dataTypesStr += `${collectionKey}: Record<string, {\n  id: string;\n  ${legacyTypes}  collection: ${collectionKey};\n  data: ${dataType};\n  rendered?: RenderedContent;\n}>;\n`;
 				break;
 			case 'data':
 				if (collectionEntryKeys.length === 0) {