Fixed scientists importing, and added linking by name

importersFixed.md

@@ -1,6 +1,6 @@
 - [x] Import users
 - [x] Import artefacts
 - [x] Import earthquakes
-- [ ] Import observatoies
+- [x] Import observatories
 - [ ] Import requests
-- [ ] Import scientists
+- [x] Import scientists

importersOrder.md

@@ -0,0 +1,6 @@
+1. Import users
+2. Import scientists
+3. Import observatories
+4. Import earthquakes
+5. Import artefacts
+6. Import requests

prisma/schema.prisma

@@ -73,9 +73,9 @@ model Observatory {
     updatedAt           DateTime     @updatedAt
     name                String
     location            String
-    longitude           String
+    longitude           Float 
-    latitude            String
+    latitude            Float 
-    dateEstablished     Int?
+    dateEstablished     DateTime
     isFunctional        Boolean
     seismicSensorOnline Boolean      @default(true)
     creatorId           Int?

public/Observatories.csv

@@ -1,25 +1,26 @@
-Pacific Apex Seismic Center,"Aleutian Trench, Alaska, USA",53.0000,-168.0000,1973-06-15,Yes
+Name,Location,Latitude,Longitude,DateEstablished,Functional,SeismicSensorOnline
-Cascadia Quake Research Institute,"Oregon Coast, USA",44.5000,-124.0000,1985-03-22,Yes
+Pacific Apex Seismic Center,"Aleutian Trench, Alaska, USA",53.0000,-168.0000,1973-06-15,Yes,Yes
-Andes Fault Survey Observatory,"Nazca-South American Plate, Santiago, Chile",-33.4500,-70.6667,1992-10-10,Yes
+Cascadia Quake Research Institute,"Oregon Coast, USA",44.5000,-124.0000,1985-03-22,Yes,Yes
-Kermadec Deep Motion Lab,"Kermadec Trench, northeast of New Zealand",-29.5000,-177.0000,2001-05-14,Yes
+Andes Fault Survey Observatory,"Nazca-South American Plate, Santiago, Chile",-33.4500,-70.6667,1992-10-10,Yes,Yes
-Japan Trench Monitoring Station,"Off the coast of Sendai, Japan",38.5000,143.0000,1968-09-01,Yes
+Kermadec Deep Motion Lab,"Kermadec Trench, northeast of New Zealand",-29.5000,-177.0000,2001-05-14,Yes,Yes
-Himalayan Rift Observatory,"Nepal-Tibet border, near Mount Everest",28.5000,86.5000,1998-12-08,Yes
+Japan Trench Monitoring Station,"Off the coast of Sendai, Japan",38.5000,143.0000,1968-09-01,Yes,Yes
-East African Rift Monitoring Center,"Rift Valley, near Nairobi, Kenya",-1.2921,36.8219,2010-03-30,Yes
+Himalayan Rift Observatory,"Nepal-Tibet border, near Mount Everest",28.5000,86.5000,1998-12-08,Yes,Yes
-Reykjavik Seismic Monitoring Hub,"Mid-Atlantic Ridge, Reykjavik, Iceland",64.1355,-21.8954,1957-11-17,Yes
+East African Rift Monitoring Center,"Rift Valley, near Nairobi, Kenya",-1.2921,36.8219,2010-03-30,Yes,Yes
-Azores Tectonic Research Base,"Azores Archipelago, Portugal",37.7412,-25.6756,1982-07-04,Yes
+Reykjavik Seismic Monitoring Hub,"Mid-Atlantic Ridge, Reykjavik, Iceland",64.1355,-21.8954,1957-11-17,Yes,Yes
-Sumatra Subduction Observatory,"West Coast of Indonesia, Aceh Province",4.6951,95.5539,2005-02-16,Yes
+Azores Tectonic Research Base,"Azores Archipelago, Portugal",37.7412,-25.6756,1982-07-04,Yes,Yes
-Tonga Trench Seismographic Unit,"Off the coast of Tonga",-20.0000,-175.0000,1994-08-21,Yes
+Sumatra Subduction Observatory,"West Coast of Indonesia, Aceh Province",4.6951,95.5539,2005-02-16,Yes,Yes
-San Andreas Fault Research Center,"San Francisco Bay Area, California, USA",37.7749,-122.4194,1929-10-07,Yes
+Tonga Trench Seismographic Unit,"Off the coast of Tonga",-20.0000,-175.0000,1994-08-21,Yes,Yes
-Kamchatka Seismic Laboratory,"Kamchatka Peninsula, Russia",56.0000,160.0000,1978-01-12,Yes
+San Andreas Fault Research Center,"San Francisco Bay Area, California, USA",37.7749,-122.4194,1929-10-07,Yes,Yes
-Hawaii Island Seismic Research Station,"Hawaii Big Island, USA",19.8968,-155.5828,1989-05-06,Yes
+Kamchatka Seismic Laboratory,"Kamchatka Peninsula, Russia",56.0000,160.0000,1978-01-12,Yes,Yes
-Cascadia Ridge Offshore Observatory,"Offshore of Vancouver Island, British Columbia, Canada",48.4284,-123.3656,2003-11-18,Yes
+Hawaii Island Seismic Research Station,"Hawaii Big Island, USA",19.8968,-155.5828,1989-05-06,Yes,Yes
-Zagros Fault Zone Research Center,"Western Iran, near Kermanshah",34.0000,46.0000,1990-06-28,Yes
+Cascadia Ridge Offshore Observatory,"Offshore of Vancouver Island, British Columbia, Canada",48.4284,-123.3656,2003-11-18,Yes,Yes
-Manila Trench Seismic Observatory,"South China Sea, west of Luzon, Philippines",15.0000,120.0000,1996-04-09,Yes
+Zagros Fault Zone Research Center,"Western Iran, near Kermanshah",34.0000,46.0000,1990-06-28,Yes,Yes
-Caribbean Subduction Monitoring Station,"Lesser Antilles Subduction Zone, Martinique",14.6000,-61.0000,2008-09-23,Yes
+Manila Trench Seismic Observatory,"South China Sea, west of Luzon, Philippines",15.0000,120.0000,1996-04-09,Yes,Yes
-Gorda Plate Analysis Hub,"Mendocino Triple Junction, California, USA",40.0000,-124.0000,1952-12-01,Yes
+Caribbean Subduction Monitoring Station,"Lesser Antilles Subduction Zone, Martinique",14.6000,-61.0000,2008-09-23,Yes,Yes
-Red Sea Rift Research Base,"Southern Red Sea, Eritrea",15.0000,42.0000,2015-07-15,Yes
+Gorda Plate Analysis Hub,"Mendocino Triple Junction, California, USA",40.0000,-124.0000,1952-12-01,Yes,Yes
-Sumatra Coastal Reserve Observatory,"West Coast of Sumatra, Indonesia",-2.0000,100.5000,1980-03-11,No
+Red Sea Rift Research Base,"Southern Red Sea, Eritrea",15.0000,42.0000,2015-07-15,Yes,Yes
-Antarctic Polar Seismology Station,"Rift vicinity near Ross Ice Shelf, Antarctica",-78.3000,-166.2500,1974-06-01,No
+Sumatra Coastal Reserve Observatory,"West Coast of Sumatra, Indonesia",-2.0000,100.5000,1980-03-11,Yes,Yes
-Yucatan Seismic Monitoring Site,"Cocos Plate near Yucatan Peninsula, Mexico",20.7000,-90.8000,1965-09-23,No
+Antarctic Polar Seismology Station,"Rift vicinity near Ross Ice Shelf, Antarctica",-78.3000,-166.2500,1974-06-01,Yes,Yes
-Makran Subduction Fault Observatory,"Coastal Pakistan and Iran junction",25.5000,62.0000,1990-08-29,No
+Yucatan Seismic Monitoring Site,"Cocos Plate near Yucatan Peninsula, Mexico",20.7000,-90.8000,1965-09-23,Yes,Yes
-Baltic Continental Drift Center,"Southeastern Sweden, Baltic Shield zone",56.0000,15.0000,1987-12-12,No
+Makran Subduction Fault Observatory,"Coastal Pakistan and Iran junction",25.5000,62.0000,1990-08-29,Yes,Yes
+Baltic Continental Drift Center,"Southeastern Sweden, Baltic Shield zone",56.0000,15.0000,1987-12-12,Yes,Yes

public/Scientists.csv

@@ -1,10 +1,10 @@
+Name,Level,SuperiorName
 Dr. Emily Neighbour Carter,Junior,Dr. Rajiv Menon
 Dr. Rajiv Menon,Senior,None
 Dr. Izzy Patterson,Senior,None
 Dr. Hiroshi Takeda,Senior,None
 Dr. Miriam Hassan,Senior,None
 Dr. Alice Johnson,Senior,None
-Tim Howitz,Admin,None
 Dr. Natalia Petrova,Junior,Dr. Izzy Patteron
 Dr. Li Cheng,Junior,Dr. Rajiv Menon
 Dr. Javier Ortega,Junior,Dr. Izzy Patterson

src/app/api/import-earthquakes/route.ts

@@ -6,6 +6,7 @@ import { prisma } from "@utils/prisma";
 import { getRandomNumber } from "@utils/maths";
 
 const csvFilePath = path.resolve(process.cwd(), "public/earthquakes.csv");
+const DISTANCE_THRESHOLD_KM = 500; // Max distance for observatory matching
 
 type CsvRow = {
 	Date: string;
@@ -18,6 +19,18 @@ type CsvRow = {
 	Depth: string;
 };
 
+// Haversine formula to calculate distance between two points (in km)
+function getDistance(lat1: number, lon1: number, lat2: number, lon2: number): number {
+	const R = 6371; // Earth's radius in km
+	const dLat = (lat2 - lat1) * (Math.PI / 180);
+	const dLon = (lon2 - lon1) * (Math.PI / 180);
+	const a =
+		Math.sin(dLat / 2) * Math.sin(dLat / 2) +
+		Math.cos(lat1 * (Math.PI / 180)) * Math.cos(lat2 * (Math.PI / 180)) * Math.sin(dLon / 2) * Math.sin(dLon / 2);
+	const c = 2 * Math.atan2(Math.sqrt(a), Math.sqrt(1 - a));
+	return R * c;
+}
+
 export async function POST() {
 	try {
 		const fileContent = await fs.readFile(csvFilePath, "utf8");
@@ -28,12 +41,15 @@ export async function POST() {
 
 		const failedImports: { row: CsvRow; reason: string }[] = [];
 
+		// Fetch all observatories once to avoid repeated queries
+		const observatories = await prisma.observatory.findMany({
+			select: { id: true, latitude: true, longitude: true },
+		});
+
 		const earthquakes = await Promise.all(
 			records.map(async (row) => {
 				const creators = await prisma.user.findMany({
-					where: {
+					where: { role: { in: ["SCIENTIST", "ADMIN"] } },
-						role: { in: ["SCIENTIST", "ADMIN"] },
-					},
 				});
 				const randomCreator = creators.length > 0 ? creators[getRandomNumber(0, creators.length - 1)] : null;
 
@@ -42,16 +58,43 @@ export async function POST() {
 					return null;
 				}
 
+				const eqLat = parseFloat(row.Latitude);
+				const eqLon = parseFloat(row.Longitude);
+
+				// Find observatories within distance threshold
+				let nearbyObservatories = observatories
+					.map((obs) => ({
+						id: obs.id,
+						distance: getDistance(eqLat, eqLon, obs.latitude, obs.longitude),
+					}))
+					.filter((obs) => obs.distance <= DISTANCE_THRESHOLD_KM)
+					.map((obs) => ({ id: obs.id }));
+
+				// If no observatories within range, find the nearest one
+				if (nearbyObservatories.length === 0) {
+					const nearest = observatories.reduce(
+						(min, obs) => {
+							const distance = getDistance(eqLat, eqLon, obs.latitude, obs.longitude);
+							return distance < min.distance ? { id: obs.id, distance } : min;
+						},
+						{ id: -1, distance: Infinity }
+					);
+					if (nearest.id !== -1) {
+						nearbyObservatories = [{ id: nearest.id }];
+					}
+				}
+
 				return {
 					date: new Date(row.Date),
 					code: row.Code,
 					magnitude: parseFloat(row.Magnitude),
 					type: row.Type,
-					latitude: parseFloat(row.Latitude),
+					latitude: eqLat,
-					longitude: parseFloat(row.Longitude),
+					longitude: eqLon,
 					location: row.Location,
 					depth: row.Depth,
 					creatorId: randomCreator.id,
+					observatories: { connect: nearbyObservatories },
 				};
 			})
 		);
@@ -60,6 +103,7 @@ export async function POST() {
 			(earthquake): earthquake is NonNullable<typeof earthquake> => earthquake !== null
 		);
 
+		// Bulk insert earthquakes with observatory connections
 		await prisma.earthquake.createMany({
 			data: validEarthquakes,
 		});

src/app/api/import-observatories/route.ts

@@ -1,11 +1,11 @@
 import { parse } from "csv-parse/sync";
+import { stringToBool } from "@utils/parsingUtils";
 import fs from "fs/promises";
 import { NextResponse } from "next/server";
 import path from "path";
-
 import { prisma } from "@utils/prisma";
+import { getRandomNumber } from "@utils/maths";
 
-// CSV location (update filename as needed)
 const csvFilePath = path.resolve(process.cwd(), "public/observatories.csv");
 
 type CsvRow = {
@@ -13,49 +13,68 @@ type CsvRow = {
 	Location: string;
 	Latitude: string;
 	Longitude: string;
-	DateEstablished?: string;
+	DateEstablished: string;
 	Functional: string;
-	SeismicSensorOnline?: string;
+	SeismicSensorOnline: string;
 };
 
-function stringToBool(val: string | undefined): boolean {
-	// Accepts "TRUE", "true", "True", etc.
-	if (!val) return false;
-	return /^true$/i.test(val.trim());
-}
-
 export async function POST() {
 	try {
-		// 1. Read file
 		const fileContent = await fs.readFile(csvFilePath, "utf8");
-
-		// 2. Parse CSV
 		const records: CsvRow[] = parse(fileContent, {
 			columns: true,
 			skip_empty_lines: true,
 		});
 
-		// 3. Map records to Prisma inputs
+		const failedImports: { row: CsvRow; reason: string }[] = [];
-		const observatories = records.map((row) => ({
+
+		const observatories = await Promise.all(
+			records.map(async (row) => {
+				const creators = await prisma.user.findMany({
+					where: {
+						role: { in: ["SCIENTIST", "ADMIN"] },
+					},
+				});
+				const randomCreator = creators.length > 0 ? creators[getRandomNumber(0, creators.length - 1)] : null;
+
+				if (!randomCreator) {
+					failedImports.push({ row, reason: `RandomCreator: ${randomCreator}` });
+					return null;
+				}
+
+				return {
 					name: row.Name,
 					location: row.Location,
-			latitude: row.Latitude,
+					latitude: parseFloat(row.Latitude),
-			longitude: row.Longitude,
+					longitude: parseFloat(row.Longitude),
-			dateEstablished: row.DateEstablished ? parseInt(row.DateEstablished, 10) : null,
+					dateEstablished: new Date(row.DateEstablished),
-			functional: stringToBool(row.Functional),
+					isFunctional: stringToBool(row.Functional),
-			seismicSensorOnline: row.SeismicSensorOnline ? stringToBool(row.SeismicSensorOnline) : true, // default true per schema
+					seismicSensorOnline: stringToBool(row.SeismicSensorOnline),
-			// todo add random selection of creatorId
+					creatorId: randomCreator.id,
-			creatorId: null,
+				};
-		}));
+			})
+		);
+
+		const validObservatories = observatories.filter(
+			(observatory): observatory is NonNullable<typeof observatory> => observatory !== null
+		);
 
-		// 4. Bulk insert
 		await prisma.observatory.createMany({
-			data: observatories,
+			data: validObservatories,
 		});
 
+		if (failedImports.length > 0) {
+			console.warn("Failed imports:", failedImports);
+			await fs.writeFile(
+				path.resolve(process.cwd(), "failed_imports_observatories.json"),
+				JSON.stringify(failedImports, null, 2)
+			);
+		}
+
 		return NextResponse.json({
 			success: true,
-			count: observatories.length,
+			count: validObservatories.length,
+			failedCount: failedImports.length,
 		});
 	} catch (error: any) {
 		console.error(error);

src/app/api/import-scientists/route.ts

@@ -2,21 +2,18 @@ import { parse } from "csv-parse/sync";
 import fs from "fs/promises";
 import { NextResponse } from "next/server";
 import path from "path";
-
 import { prisma } from "@utils/prisma";
+import { getRandomNumber } from "@utils/maths";
 
-// Path to CSV file
 const csvFilePath = path.resolve(process.cwd(), "public/scientists.csv");
 
 type CsvRow = {
 	Name: string;
-	Level?: string;
+	Level: string;
-	UserId: string;
+	SuperiorName: string;
-	SuperiorId?: string;
 };
 
 function normalizeLevel(level: string | undefined): string {
-	// Only allow JUNIOR, SENIOR; default JUNIOR
 	if (!level || !level.trim()) return "JUNIOR";
 	const lv = level.trim().toUpperCase();
 	return ["JUNIOR", "SENIOR"].includes(lv) ? lv : "JUNIOR";
@@ -24,31 +21,100 @@ function normalizeLevel(level: string | undefined): string {
 
 export async function POST() {
 	try {
-		// 1. Read the CSV file
 		const fileContent = await fs.readFile(csvFilePath, "utf8");
-
-		// 2. Parse the CSV
 		const records: CsvRow[] = parse(fileContent, {
 			columns: true,
 			skip_empty_lines: true,
 		});
 
-		// 3. Transform each record for Prisma
+		const failedImports: { row: CsvRow; reason: string }[] = [];
-		// todo add senior scientists first
-		const scientists = records.map((row) => ({
-			name: row.Name,
-			level: normalizeLevel(row.Level),
-			userId: parseInt(row.UserId, 10),
-			// todo get superior id by name from db
-			superiorId: row.SuperiorId && row.SuperiorId.trim() !== "" ? parseInt(row.SuperiorId, 10) : null,
-		}));
 
-		// 4. Bulk create scientists in database
+		// Fetch all users to match names
-		await prisma.scientist.createMany({
+		const users = await prisma.user.findMany({
-			data: scientists,
+			select: { id: true, name: true },
 		});
 
-		return NextResponse.json({ success: true, count: scientists.length });
+		// Process seniors first to ensure superiors exist for juniors
+		const seniorScientists = records.filter((row) => normalizeLevel(row.Level) === "SENIOR");
+		const juniorScientists = records.filter((row) => normalizeLevel(row.Level) === "JUNIOR");
+		const seniorScientistNames = new Map<string, number>(); // Map name to ID
+
+		const seniors = await Promise.all(
+			seniorScientists.map(async (row) => {
+				const user = users.find((u) => u.name === row.Name);
+				if (!user) {
+					failedImports.push({ row, reason: `User not found: ${row.Name}` });
+					return null;
+				}
+
+				return {
+					name: row.Name,
+					level: normalizeLevel(row.Level),
+					userId: user.id,
+					superiorId: null, // Seniors have no superior
+				};
+			})
+		);
+
+		const validSeniors = seniors.filter((senior): senior is NonNullable<typeof senior> => senior !== null);
+
+		// Create senior scientists and store their IDs
+		for (const senior of validSeniors) {
+			try {
+				const scientist = await prisma.scientist.create({ data: senior });
+				seniorScientistNames.set(senior.name, scientist.id);
+			} catch (error: any) {
+				failedImports.push({
+					row: { Name: senior.name, Level: senior.level, SuperiorName: "None" },
+					reason: `Failed to create senior scientist: ${error.message}`,
+				});
+			}
+		}
+
+		// Process junior scientists
+		const juniors = await Promise.all(
+			juniorScientists.map(async (row) => {
+				const user = users.find((u) => u.name === row.Name);
+				if (!user) {
+					failedImports.push({ row, reason: `User not found: ${row.Name}` });
+					return null;
+				}
+
+				let superiorId: number | null = null;
+				if (row.SuperiorName && row.SuperiorName.trim() !== "None") {
+					superiorId = seniorScientistNames.get(row.SuperiorName.trim()) || null;
+					if (!superiorId) {
+						failedImports.push({ row, reason: `Superior not found: ${row.SuperiorName}` });
+						return null;
+					}
+				}
+
+				return {
+					name: row.Name,
+					level: normalizeLevel(row.Level),
+					userId: user.id,
+					superiorId,
+				};
+			})
+		);
+
+		const validJuniors = juniors.filter((junior): junior is NonNullable<typeof junior> => junior !== null);
+
+		// Bulk create junior scientists
+		await prisma.scientist.createMany({
+			data: validJuniors,
+		});
+
+		if (failedImports.length > 0) {
+			console.warn("Failed imports:", failedImports);
+			await fs.writeFile(path.resolve(process.cwd(), "failed_imports_scientists.json"), JSON.stringify(failedImports, null, 2));
+		}
+
+		return NextResponse.json({
+			success: true,
+			count: validSeniors.length + validJuniors.length,
+			failedCount: failedImports.length,
+		});
 	} catch (error: any) {
 		console.error(error);
 		return NextResponse.json({ success: false, error: error.message }, { status: 500 });