diff options
| -rw-r--r-- | Cont/index.html | 2 | ||||
| -rw-r--r-- | Functions/dateFuncs.js | 21 | ||||
| -rw-r--r-- | Functions/open.js | 23 | ||||
| -rw-r--r-- | README.md | 8 | ||||
| -rw-r--r-- | dbparse.js | 20 | ||||
| -rw-r--r-- | food.js | 61 | ||||
| -rw-r--r-- | scrape.js | 75 | ||||
| -rw-r--r-- | server.js | 68 |
8 files changed, 152 insertions, 126 deletions
diff --git a/Cont/index.html b/Cont/index.html index 95dcb32..a9d83e7 100644 --- a/Cont/index.html +++ b/Cont/index.html @@ -50,6 +50,8 @@ <div class="float-block"> <h2 class="shadow">\(food-header\)</h2> <p>\(food\)</p> + <h2 class="shadow">\(vege-header\)</h2> + <p>\(vege\)</p> </div> </div> </main> diff --git a/Functions/dateFuncs.js b/Functions/dateFuncs.js index 10c4250..9b1237a 100644 --- a/Functions/dateFuncs.js +++ b/Functions/dateFuncs.js @@ -17,7 +17,26 @@ function approxDate(d) return new Date(`${d.getFullYear()}-${(d.getMonth() + 1).toString().padStart(2, "0")}-${d.getDate().toString().padStart(2, "0")}`); } +function weekdayToNumber(s) +{ + const weekdays = [ + /ma.*/i, + /ti.*/i, + /ke.*/i, + /to.*/i, + /pe.*/i, + /la.*/i, + /su.*/i + ]; + for(let day = 0; day < weekdays.length; day++) + { + if (s.match(weekdays[day])) + return day; + } +} + module.exports = { fromString: stringToDate, - between: isBetweenDates + between: isBetweenDates, + weekdayToNumber: weekdayToNumber }
\ No newline at end of file diff --git a/Functions/open.js b/Functions/open.js index f04201b..f532721 100644 --- a/Functions/open.js +++ b/Functions/open.js @@ -1,4 +1,5 @@ const fs = require("fs"); +const https = require("https"); function openFile(path) { @@ -13,4 +14,24 @@ function openFile(path) }); } -exports.file = openFile; +async function urlOpen(path) +{ + return new Promise((resolve, reject) => + { + let req = https.get(path, res => + { + res.on("data", resolve); + }); + }); + req.on("error", e => + { + console.error(e); + }); + req.end(); +} + + +module.exports = { + file: openFile, + url: urlOpen +};
\ No newline at end of file @@ -68,6 +68,14 @@ CREATE TABLE exams ( message VARCHAR(256), PRIMARY KEY (start, end) ); +CREATE TABLE foods ( + week INT, + day INT, + vegetarian TINYINT, + header VARCHAR(15), + dateString VARCHAR(13), + food VARCHAR(256) +); ``` > Note that if you had some information in a former database that you don't update manually, it will be lost. @@ -1,3 +1,5 @@ +const weekdayToNumber = require("./Functions/dateFuncs.js").weekdayToNumber; + function getCharAmount(s, c) { let n = 0; @@ -62,24 +64,6 @@ function parseCluttered(s) return s.replaceAll(".", "").replaceAll(" ", "").toUpperCase(); } -function weekdayToNumber(s) -{ - const weekdays = [ - /ma.*/i, - /ti.*/i, - /ke.*/i, - /to.*/i, - /pe.*/i, - /la.*/i, - /su.*/i - ]; - for(let day = 0; day < weekdays.length; day++) - { - if (s.match(weekdays[day])) - return day; - } -} - async function writeShifts(data, DB) { let deletions = await Promise.all([ @@ -0,0 +1,61 @@ +const parse = require("./dbparse.js"); +const open = require("./Functions/open.js"); +const { weekdayToNumber } = require("./Functions/dateFuncs.js"); + +function* scrapeFood(data) +{ + const foodRegex = /<title>(\w{2} (?:\d\d?\.){2}\d{4})<\/title><description><!\[CDATA\[(Lounas) ?:? ?(.*?)(Kasvislounas) ?:? ?(.*?)]]><\/description>/gm; + const foods = data.matchAll(foodRegex); + for(const food of foods) + { + yield [ + weekdayToNumber(food[1]), // index + food[1], // header (date) + [food[2], food[3]], // first header, first food + [food[4], food[5]] // second header, second food + ]; + } +} + +async function buildFoods(DB) +{ + await DB.query_raw("DELETE FROM foods"); + let foodData = await Promise.all([ + open.url(getFoodLink(1)), + open.url(getFoodLink(2)) + ]); + foodData = foodData.map((f) => f.toString("utf-8")); + const foodInitOperations = []; + const foods = foodData.map(f => scrapeFood(f)); + for(let week = 1; week <= 2; week++) + { + for(const food of foods[week - 1]) + { + foodInitOperations.push(DB.execute("INSERT INTO foods VALUES (?, ?, FALSE, ?, ?, ?)", [ + week, + food[0], + food[2][0], + food[1], + food[2][1] + ])); + foodInitOperations.push(DB.execute("INSERT INTO foods VALUES (?, ?, TRUE, ?, ?, ?)", [ + week, + food[0], + food[3][0], + food[1], + food[3][1] + ])); + } + } + await Promise.all(foodInitOperations); +} + +function getFoodLink(week) +{ + return `https://eruokalista.lohja.fi/AromieMenus/FI/Default/Lohja/Koulut/Rss.aspx?Id=97f76449-f57c-4217-aede-b5f9dbf2b41e&DateMode=${week}`; +} + + +exports.foods = scrapeFood; +exports.link = getFoodLink; +exports.build = buildFoods;
\ No newline at end of file diff --git a/scrape.js b/scrape.js deleted file mode 100644 index c7a5743..0000000 --- a/scrape.js +++ /dev/null @@ -1,75 +0,0 @@ -const https = require("https"); -const parse = require("./dbparse.js"); -const fs = require("fs"); -const events = require("events"); - -async function urlOpen(path) -{ - return new Promise((resolve, reject) => - { - let req = https.get(path, res => - { - res.on("data", resolve); - }); - }); - req.on("error", e => - { - console.error(e); - }); - req.end(); -} - -async function scrapeFood(url) -{ - let data = await urlOpen(url); - data = data.toString("utf-8"); - - let foodList = []; - const weekdays = ["ma", "ti", "ke", "to", "pe", "la", "su"]; - - let titleTags = ["<title>", "</title>"]; - let foodTags = ["<![CDATA[", "]]>"]; - const getSpan = (data, tags, i = 0) => - { - return [ - parse.find(data, tags[0], i) + tags[0].length, - parse.find(data, tags[1], i) - ]; - } - let mainTitle = parse.find(data, titleTags[1]) + titleTags[1].length; - let titleSpan = getSpan(data, titleTags, mainTitle); - let foodSpan = getSpan(data, foodTags); - - while ( - (titleSpan[0] !== -1) - && (titleSpan[1] !== -1) - && (foodSpan[0] !== -1) - && (foodSpan[1] !== -1) - ) - { - let title = data.substring(titleSpan[0], titleSpan[1]); - let food = data.substring(foodSpan[0], foodSpan[1]); - - let weekdayIndex = weekdays.findIndex(val => { return val === title.substring(0, 2); }); - if (weekdayIndex !== -1) - foodList[weekdayIndex] = [title, neatify(food)]; - - titleSpan = getSpan(data, titleTags, foodSpan[1]); - foodSpan = getSpan(data, foodTags, titleSpan[1]); - } - - return foodList; -} - -function getFoodLink(week) -{ - return `https://eruokalista.lohja.fi/AromieMenus/FI/Default/Lohja/Koulut/Rss.aspx?Id=97f76449-f57c-4217-aede-b5f9dbf2b41e&DateMode=${week}`; -} - -function neatify(food) -{ - return food.replaceAll(")", ")<br>").replaceAll(" :", ":").replaceAll(":", ":<br>"); -} - -exports.food = scrapeFood; -exports.link = getFoodLink; @@ -1,10 +1,10 @@ //const http = require("http"); const https = require("https"); const url = require("url"); -const scrape = require("./scrape.js"); +const food = require("./food.js"); const SQL_DBS = require("./database.js"); const DBPARSE = require("./dbparse.js"); -const openFile = require("./Functions/open.js").file; +const open = require("./Functions/open.js"); const strFuncs = require("./Functions/stringFuncs.js"); const dateFuncs = require("./Functions/dateFuncs.js"); @@ -27,14 +27,13 @@ async function init() let visitorCount = 0; // await for needed things in async - let [foodsThisWeek, foodsNextWeek, dbcredentials, httpsKey, httpsCert] = await Promise.all([ - scrape.food(scrape.link(1)), - scrape.food(scrape.link(2)), - openFile("../dblogin.txt"), - openFile("../Certificate/key.pem"), - openFile("../Certificate/cert.pem") + let [dbcredentials, httpsKey, httpsCert] = await Promise.all([ + open.file("../dblogin.txt"), + open.file("../Certificate/key.pem"), + open.file("../Certificate/cert.pem") ]); - + + // https options, you need to get a certificate in the file ../Certificate for the server to work const httpsOpts = { key: httpsKey, @@ -44,8 +43,15 @@ async function init() // get the MySQL DB connection const SQLDB = new SQL_DBS.Database(JSON.parse(dbcredentials)); - // get the food "database" - const foods = [foodsThisWeek, foodsNextWeek]; + // Add the foods to the database + await food.build(SQLDB); + setInterval( + () => + { + food.build(SQLDB); + }, + 7 * 24 * 60 * 60 * 1000 + ); // server code async function server(req, res) @@ -79,7 +85,6 @@ async function init() "path": path, "path404": errorPath, "query": q.query, - "foods": foods, "sqldb": SQLDB }; @@ -157,10 +162,9 @@ async function buildMain(args) // get the passed arguments const path = args["path"]; const query = args["query"]; - const foods = args["foods"]; const index = query.index; const SQLDB = args["sqldb"]; - const data = await openFile(path); + const data = await open.file(path); let data_string = data.toString("utf-8"); // here are the things to replace in the html page @@ -242,19 +246,21 @@ async function buildMain(args) data_string = data_string.replace('<div id="shift-result" class="float-block">', '<div id="shift-result" class="float-block" style="display: none;">'); // get the food - let food; - food = foods[ +(day < actualDay) ][day]; - if (food !== undefined) - { - res["food-header"] = food[0]; - res["food"] = food[1]; - } - else - { - res["food-header"] = weekdays[day]; - res["food"] = "Päivälle ei löytynyt ruokaa"; - } - res["food-header"] = `Kouluruoka ${res["food-header"]}:`; + const week = +(day < actualDay) + 1; // Week = 1 if day is not past + const [food, vege] = await Promise.all([ + SQLDB.execute( + "SELECT header, datestring, food FROM foods WHERE week=? AND day=? AND vegetarian=FALSE", + [week, day] + ), + SQLDB.execute( + "SELECT header, datestring, food FROM foods WHERE week=? AND day=? AND vegetarian=TRUE", + [week, day] + ) + ]); + res["food-header"] = `${food[0].header} ${food[0].datestring}`; + res["vege-header"] = vege[0].header; + res["food"] = food[0].food; + res["vege"] = vege[0].food; data_string = build_replace(data_string, res); @@ -264,7 +270,7 @@ async function buildMain(args) async function buildDevs(args) { const path = args["path"]; - const data = await openFile(path); + const data = await open.file(path); const DB = args["sqldb"]; let res = ""; @@ -286,7 +292,7 @@ async function buildDevs(args) async function build404(args) { args["path"] = args["path"].substring("./Cont".length); - const data = await openFile(args["path404"]); + const data = await open.file(args["path404"]); const data_string = data.toString("utf-8"); return data_string.replace("\\(path\\)", args["path"]); } @@ -294,14 +300,14 @@ async function build404(args) async function buildDefault(args) { const path = args["path"]; - const data = await openFile(path); + const data = await open.file(path); return data.toString("utf-8"); } async function buildImage(args) { const path = args["path"]; - const data = await openFile(path); + const data = await open.file(path); return data; } |