Fixed scientists importing, and added linking by name

importersFixed.md

@@ -3,4 +3,4 @@
 - [x] Import earthquakes
 - [x] Import observatories
 - [ ] Import requests
-- [ ] Import scientists
+- [x] Import scientists

importersOrder.md

@@ -0,0 +1,6 @@
+1. Import users
+2. Import scientists
+3. Import observatories
+4. Import earthquakes
+5. Import artefacts
+6. Import requests

src/app/api/import-scientists/route.ts

@@ -2,21 +2,18 @@ import { parse } from "csv-parse/sync";
 import fs from "fs/promises";
 import { NextResponse } from "next/server";
 import path from "path";
-
 import { prisma } from "@utils/prisma";
+import { getRandomNumber } from "@utils/maths";
 
-// Path to CSV file
 const csvFilePath = path.resolve(process.cwd(), "public/scientists.csv");
 
 type CsvRow = {
 	Name: string;
-	Level?: string;
+	Level: string;
-	UserId: string;
+	SuperiorName: string;
-	SuperiorId?: string;
 };
 
 function normalizeLevel(level: string | undefined): string {
-	// Only allow JUNIOR, SENIOR; default JUNIOR
 	if (!level || !level.trim()) return "JUNIOR";
 	const lv = level.trim().toUpperCase();
 	return ["JUNIOR", "SENIOR"].includes(lv) ? lv : "JUNIOR";
@@ -24,31 +21,100 @@ function normalizeLevel(level: string | undefined): string {
 
 export async function POST() {
 	try {
-		// 1. Read the CSV file
 		const fileContent = await fs.readFile(csvFilePath, "utf8");
-
-		// 2. Parse the CSV
 		const records: CsvRow[] = parse(fileContent, {
 			columns: true,
 			skip_empty_lines: true,
 		});
 
-		// 3. Transform each record for Prisma
+		const failedImports: { row: CsvRow; reason: string }[] = [];
-		// todo add senior scientists first
-		const scientists = records.map((row) => ({
-			name: row.Name,
-			level: normalizeLevel(row.Level),
-			userId: parseInt(row.UserId, 10),
-			// todo get superior id by name from db
-			superiorId: row.SuperiorId && row.SuperiorId.trim() !== "" ? parseInt(row.SuperiorId, 10) : null,
-		}));
 
-		// 4. Bulk create scientists in database
+		// Fetch all users to match names
-		await prisma.scientist.createMany({
+		const users = await prisma.user.findMany({
-			data: scientists,
+			select: { id: true, name: true },
 		});
 
-		return NextResponse.json({ success: true, count: scientists.length });
+		// Process seniors first to ensure superiors exist for juniors
+		const seniorScientists = records.filter((row) => normalizeLevel(row.Level) === "SENIOR");
+		const juniorScientists = records.filter((row) => normalizeLevel(row.Level) === "JUNIOR");
+		const seniorScientistNames = new Map<string, number>(); // Map name to ID
+
+		const seniors = await Promise.all(
+			seniorScientists.map(async (row) => {
+				const user = users.find((u) => u.name === row.Name);
+				if (!user) {
+					failedImports.push({ row, reason: `User not found: ${row.Name}` });
+					return null;
+				}
+
+				return {
+					name: row.Name,
+					level: normalizeLevel(row.Level),
+					userId: user.id,
+					superiorId: null, // Seniors have no superior
+				};
+			})
+		);
+
+		const validSeniors = seniors.filter((senior): senior is NonNullable<typeof senior> => senior !== null);
+
+		// Create senior scientists and store their IDs
+		for (const senior of validSeniors) {
+			try {
+				const scientist = await prisma.scientist.create({ data: senior });
+				seniorScientistNames.set(senior.name, scientist.id);
+			} catch (error: any) {
+				failedImports.push({
+					row: { Name: senior.name, Level: senior.level, SuperiorName: "None" },
+					reason: `Failed to create senior scientist: ${error.message}`,
+				});
+			}
+		}
+
+		// Process junior scientists
+		const juniors = await Promise.all(
+			juniorScientists.map(async (row) => {
+				const user = users.find((u) => u.name === row.Name);
+				if (!user) {
+					failedImports.push({ row, reason: `User not found: ${row.Name}` });
+					return null;
+				}
+
+				let superiorId: number | null = null;
+				if (row.SuperiorName && row.SuperiorName.trim() !== "None") {
+					superiorId = seniorScientistNames.get(row.SuperiorName.trim()) || null;
+					if (!superiorId) {
+						failedImports.push({ row, reason: `Superior not found: ${row.SuperiorName}` });
+						return null;
+					}
+				}
+
+				return {
+					name: row.Name,
+					level: normalizeLevel(row.Level),
+					userId: user.id,
+					superiorId,
+				};
+			})
+		);
+
+		const validJuniors = juniors.filter((junior): junior is NonNullable<typeof junior> => junior !== null);
+
+		// Bulk create junior scientists
+		await prisma.scientist.createMany({
+			data: validJuniors,
+		});
+
+		if (failedImports.length > 0) {
+			console.warn("Failed imports:", failedImports);
+			await fs.writeFile(path.resolve(process.cwd(), "failed_imports_scientists.json"), JSON.stringify(failedImports, null, 2));
+		}
+
+		return NextResponse.json({
+			success: true,
+			count: validSeniors.length + validJuniors.length,
+			failedCount: failedImports.length,
+		});
 	} catch (error: any) {
 		console.error(error);
 		return NextResponse.json({ success: false, error: error.message }, { status: 500 });