Food scraping updation

8 files changed, 152 insertions, 126 deletions

diff --git a/Cont/index.html b/Cont/index.html
index 95dcb32..a9d83e7 100644
--- a/Cont/index.html
+++ b/Cont/index.html
@@ -50,6 +50,8 @@
 				<div class="float-block">
 					<h2 class="shadow">\(food-header\)</h2>
 					<p>\(food\)</p>
+					<h2 class="shadow">\(vege-header\)</h2>
+					<p>\(vege\)</p>
 				</div>
 			</div>
 		</main>
diff --git a/Functions/dateFuncs.js b/Functions/dateFuncs.js
index 10c4250..9b1237a 100644
--- a/Functions/dateFuncs.js
+++ b/Functions/dateFuncs.js
@@ -17,7 +17,26 @@ function approxDate(d)
     return new Date(`${d.getFullYear()}-${(d.getMonth() + 1).toString().padStart(2, "0")}-${d.getDate().toString().padStart(2, "0")}`);
 }
 
+function weekdayToNumber(s)
+{
+	const weekdays = [
+		/ma.*/i,
+		/ti.*/i,
+		/ke.*/i,
+		/to.*/i,
+		/pe.*/i,
+		/la.*/i,
+		/su.*/i
+	];
+	for(let day = 0; day < weekdays.length; day++)
+	{
+		if (s.match(weekdays[day]))
+			return day;
+	}
+}
+
 module.exports = {
     fromString: stringToDate,
-    between: isBetweenDates
+    between: isBetweenDates,
+    weekdayToNumber: weekdayToNumber
 }
\ No newline at end of file
diff --git a/Functions/open.js b/Functions/open.js
index f04201b..f532721 100644
--- a/Functions/open.js
+++ b/Functions/open.js
@@ -1,4 +1,5 @@
 const fs = require("fs");
+const https = require("https");
 
 function openFile(path)
 {
@@ -13,4 +14,24 @@ function openFile(path)
 	});
 }
 
-exports.file = openFile;
+async function urlOpen(path)
+{
+	return new Promise((resolve, reject) =>
+	{
+		let req = https.get(path, res =>
+		{
+			res.on("data", resolve);
+		});
+	});
+	req.on("error", e =>
+	{
+		console.error(e);
+	});
+	req.end();
+}
+
+
+module.exports = {
+	file: openFile,
+	url: urlOpen
+};
\ No newline at end of file
diff --git a/README.md b/README.md
index 3286db7..53cc75e 100644
--- a/README.md
+++ b/README.md
@@ -68,6 +68,14 @@ CREATE TABLE exams (
 	message VARCHAR(256),
 	PRIMARY KEY (start, end)
 );
+CREATE TABLE foods (     
+    week INT,
+    day INT,
+    vegetarian TINYINT,
+    header VARCHAR(15),
+    dateString VARCHAR(13),
+    food VARCHAR(256)
+);
 ```
 > Note that if you had some information in a former database that you don't update manually, it will be lost.
 
diff --git a/dbparse.js b/dbparse.js
index b2204bd..dde0ff5 100644
--- a/dbparse.js
+++ b/dbparse.js
@@ -1,3 +1,5 @@
+const weekdayToNumber = require("./Functions/dateFuncs.js").weekdayToNumber;
+
 function getCharAmount(s, c)
 {
 	let n = 0;
@@ -62,24 +64,6 @@ function parseCluttered(s)
 	return s.replaceAll(".", "").replaceAll(" ", "").toUpperCase();
 }
 
-function weekdayToNumber(s)
-{
-	const weekdays = [
-		/ma.*/i,
-		/ti.*/i,
-		/ke.*/i,
-		/to.*/i,
-		/pe.*/i,
-		/la.*/i,
-		/su.*/i
-	];
-	for(let day = 0; day < weekdays.length; day++)
-	{
-		if (s.match(weekdays[day]))
-			return day;
-	}
-}
-
 async function writeShifts(data, DB)
 {
 	let deletions = await Promise.all([
diff --git a/food.js b/food.js
new file mode 100644
index 0000000..bd2ae75
--- /dev/null
+++ b/food.js
@@ -0,0 +1,61 @@
+const parse = require("./dbparse.js");
+const open = require("./Functions/open.js");
+const { weekdayToNumber } = require("./Functions/dateFuncs.js");
+
+function* scrapeFood(data)
+{
+	const foodRegex = /<title>(\w{2} (?:\d\d?\.){2}\d{4})<\/title><description><!\[CDATA\[(Lounas) ?:? ?(.*?)(Kasvislounas) ?:? ?(.*?)]]><\/description>/gm;
+	const foods = data.matchAll(foodRegex);
+	for(const food of foods)
+	{
+		yield [
+			weekdayToNumber(food[1]),	// index
+			food[1],	// header (date)
+			[food[2], food[3]],	// first header, first food
+			[food[4], food[5]]	// second header, second food
+		];
+	}
+}
+
+async function buildFoods(DB)
+{
+	await DB.query_raw("DELETE FROM foods");
+	let foodData = await Promise.all([
+		open.url(getFoodLink(1)),
+		open.url(getFoodLink(2))
+	]);
+	foodData = foodData.map((f) => f.toString("utf-8"));
+	const foodInitOperations = [];
+	const foods = foodData.map(f => scrapeFood(f));
+	for(let week = 1; week <= 2; week++)
+	{
+		for(const food of foods[week - 1])
+		{
+			foodInitOperations.push(DB.execute("INSERT INTO foods VALUES (?, ?, FALSE, ?, ?, ?)", [
+				week,
+				food[0],
+				food[2][0],
+				food[1],
+				food[2][1]
+			]));
+			foodInitOperations.push(DB.execute("INSERT INTO foods VALUES (?, ?, TRUE, ?, ?, ?)", [
+				week,
+				food[0],
+				food[3][0],
+				food[1],
+				food[3][1]
+			]));
+		}
+	}
+	await Promise.all(foodInitOperations);
+}
+
+function getFoodLink(week)
+{
+	return `https://eruokalista.lohja.fi/AromieMenus/FI/Default/Lohja/Koulut/Rss.aspx?Id=97f76449-f57c-4217-aede-b5f9dbf2b41e&DateMode=${week}`;
+}
+
+
+exports.foods = scrapeFood;
+exports.link = getFoodLink;
+exports.build = buildFoods;
\ No newline at end of file
diff --git a/scrape.js b/scrape.js
deleted file mode 100644
index c7a5743..0000000
--- a/scrape.js
+++ /dev/null
@@ -1,75 +0,0 @@
-const https = require("https");
-const parse = require("./dbparse.js");
-const fs = require("fs");
-const events = require("events");
-
-async function urlOpen(path)
-{
-	return new Promise((resolve, reject) =>
-	{
-		let req = https.get(path, res =>
-		{
-			res.on("data", resolve);
-		});
-	});
-	req.on("error", e =>
-	{
-		console.error(e);
-	});
-	req.end();
-}
-
-async function scrapeFood(url)
-{
-	let data = await urlOpen(url);
-	data = data.toString("utf-8");
-
-	let foodList = [];
-	const weekdays = ["ma", "ti", "ke", "to", "pe", "la", "su"];
-
-	let titleTags = ["<title>", "</title>"];
-	let foodTags = ["<![CDATA[", "]]>"];
-	const getSpan = (data, tags, i = 0) =>
-	{
-		return [
-			parse.find(data, tags[0], i) + tags[0].length,
-			parse.find(data, tags[1], i)
-		];
-	}
-	let mainTitle = parse.find(data, titleTags[1]) + titleTags[1].length;
-	let titleSpan = getSpan(data, titleTags, mainTitle);
-	let foodSpan = getSpan(data, foodTags);
-
-	while (
-		   (titleSpan[0] !== -1)
-		&& (titleSpan[1] !== -1)
-		&& (foodSpan[0] !== -1)
-		&& (foodSpan[1] !== -1)
-	)
-	{
-		let title = data.substring(titleSpan[0], titleSpan[1]);
-		let food = data.substring(foodSpan[0], foodSpan[1]);
-
-		let weekdayIndex = weekdays.findIndex(val => { return val === title.substring(0, 2); });
-		if (weekdayIndex !== -1)
-			foodList[weekdayIndex] = [title, neatify(food)];
-
-		titleSpan = getSpan(data, titleTags, foodSpan[1]);
-		foodSpan = getSpan(data, foodTags, titleSpan[1]);
-	}
-
-	return foodList;
-}
-
-function getFoodLink(week)
-{
-	return `https://eruokalista.lohja.fi/AromieMenus/FI/Default/Lohja/Koulut/Rss.aspx?Id=97f76449-f57c-4217-aede-b5f9dbf2b41e&DateMode=${week}`;
-}
-
-function neatify(food)
-{
-	return food.replaceAll(")", ")<br>").replaceAll(" :", ":").replaceAll(":", ":<br>");
-}
-
-exports.food = scrapeFood;
-exports.link = getFoodLink;
diff --git a/server.js b/server.js
index f4e89cf..79fc50c 100644
--- a/server.js
+++ b/server.js
@@ -1,10 +1,10 @@
 //const http	= require("http");
 const https	= require("https");
 const url	= require("url");
-const scrape	= require("./scrape.js");
+const food	= require("./food.js");
 const SQL_DBS	= require("./database.js");
 const DBPARSE	= require("./dbparse.js");
-const openFile	= require("./Functions/open.js").file;
+const open	= require("./Functions/open.js");
 const strFuncs	= require("./Functions/stringFuncs.js");
 const dateFuncs	= require("./Functions/dateFuncs.js");
 
@@ -27,14 +27,13 @@ async function init()
     let visitorCount = 0;
 
 	// await for needed things in async
-	let [foodsThisWeek, foodsNextWeek, dbcredentials, httpsKey, httpsCert] = await Promise.all([
-		scrape.food(scrape.link(1)),
-		scrape.food(scrape.link(2)),
-		openFile("../dblogin.txt"),
-		openFile("../Certificate/key.pem"),
-		openFile("../Certificate/cert.pem")
+	let [dbcredentials, httpsKey, httpsCert] = await Promise.all([
+		open.file("../dblogin.txt"),
+		open.file("../Certificate/key.pem"),
+		open.file("../Certificate/cert.pem")
 	]);
-  
+
+
   // https options, you need to get a certificate in the file ../Certificate for the server to work
 	const httpsOpts = {
 		key: httpsKey,
@@ -44,8 +43,15 @@ async function init()
 	// get the MySQL DB connection
 	const SQLDB = new SQL_DBS.Database(JSON.parse(dbcredentials));
 
-	// get the food "database"
-	const foods = [foodsThisWeek, foodsNextWeek];
+	// Add the foods to the database
+	await food.build(SQLDB);
+	setInterval(
+		() =>
+		{
+			food.build(SQLDB);
+		},
+		7 * 24 * 60 * 60 * 1000
+	);
 
 	// server code
 	async function server(req, res)
@@ -79,7 +85,6 @@ async function init()
 			"path": path,
 			"path404": errorPath,
 			"query": q.query,
-			"foods": foods,
 			"sqldb": SQLDB
 		};
 
@@ -157,10 +162,9 @@ async function buildMain(args)
 	// get the passed arguments
 	const path = args["path"];
 	const query = args["query"];
-	const foods = args["foods"];
 	const index = query.index;
 	const SQLDB = args["sqldb"];
-	const data = await openFile(path);
+	const data = await open.file(path);
 	let data_string = data.toString("utf-8");
 
 	// here are the things to replace in the html page
@@ -242,19 +246,21 @@ async function buildMain(args)
 		data_string = data_string.replace('<div id="shift-result" class="float-block">', '<div id="shift-result" class="float-block" style="display: none;">');
 	
 	// get the food
-	let food;
-	food = foods[ +(day < actualDay) ][day];
-	if (food !== undefined)
-	{
-		res["food-header"] = food[0];
-		res["food"] = food[1];
-	}
-	else
-	{
-		res["food-header"] = weekdays[day];
-		res["food"] = "Päivälle ei löytynyt ruokaa";
-	}
-	res["food-header"] = `Kouluruoka ${res["food-header"]}:`;
+	const week = +(day < actualDay) + 1; // Week = 1 if day is not past
+	const [food, vege] = await Promise.all([
+		SQLDB.execute(
+			"SELECT header, datestring, food FROM foods WHERE week=? AND day=? AND vegetarian=FALSE",
+			[week, day]
+		),
+		SQLDB.execute(
+			"SELECT header, datestring, food FROM foods WHERE week=? AND day=? AND vegetarian=TRUE",
+			[week, day]
+		)
+	]);
+	res["food-header"] = `${food[0].header} ${food[0].datestring}`;
+	res["vege-header"] = vege[0].header;
+	res["food"] = food[0].food;
+	res["vege"] = vege[0].food;
 
 	data_string = build_replace(data_string, res);
 
@@ -264,7 +270,7 @@ async function buildMain(args)
 async function buildDevs(args)
 {
 	const path = args["path"];
-	const data = await openFile(path);
+	const data = await open.file(path);
 	const DB = args["sqldb"];
 
 	let res = "";
@@ -286,7 +292,7 @@ async function buildDevs(args)
 async function build404(args)
 {
 	args["path"] = args["path"].substring("./Cont".length);
-	const data = await openFile(args["path404"]);
+	const data = await open.file(args["path404"]);
 	const data_string = data.toString("utf-8");
 	return data_string.replace("\\(path\\)", args["path"]);
 }
@@ -294,14 +300,14 @@ async function build404(args)
 async function buildDefault(args)
 {
 	const path = args["path"];
-	const data = await openFile(path);
+	const data = await open.file(path);
 	return data.toString("utf-8");
 }
 
 async function buildImage(args)
 {
 	const path = args["path"];
-	const data = await openFile(path);
+	const data = await open.file(path);
 	return data;
 }