Add scripts to import data
When updating the data, we must manually insert a new catalogue version
This commit is contained in:
dario-cfpt
61
server/src/importCharGrowthRates.ts
Normal file
61
server/src/importCharGrowthRates.ts
Normal file
@ -0,0 +1,61 @@
|
||||
require("dotenv").config();
|
||||
const axios = require('axios').default;
|
||||
const cheerio = require('cheerio');
|
||||
|
||||
const {CharacterGrowthRate} = require("./mapping");
|
||||
import {FE_Charts} from "./index";
|
||||
|
||||
const url:string = "https://serenesforest.net/three-houses/characters/growth-rates/";
|
||||
const mainCharName:string = "Byleth";
|
||||
const mainCharAlternativeName:string = "Protagonist";
|
||||
const nameIndex:number = 0;
|
||||
|
||||
|
||||
async function importCharGrowthRates() {
|
||||
const characters = await FE_Charts.getAllCharacters();
|
||||
const stats = await FE_Charts.getAllStats();
|
||||
axios.get(url)
|
||||
.then(res => {
|
||||
// We need to remove all line breaks to avoid bad html format
|
||||
const html = res.data.replace(/[\n\r]/g, "");
|
||||
const $ = cheerio.load(html);
|
||||
|
||||
// parse each tables in the html
|
||||
$("tbody").each(function (i, elem) {
|
||||
// parse each rows of the table
|
||||
elem.children.forEach((row, rowIndex) => {
|
||||
// The first element is the header of the table, so we must skip it
|
||||
if (rowIndex != 0) {
|
||||
let name = $(row.children[nameIndex]).text();
|
||||
if (name == mainCharAlternativeName) {
|
||||
name = mainCharName;
|
||||
}
|
||||
const char = characters.find(x => x.firstName == name);
|
||||
|
||||
if (char !== undefined) {
|
||||
for (let j = 0; j < stats.length; j++) {
|
||||
const charGrowthRate = new CharacterGrowthRate();
|
||||
charGrowthRate.idCharacter = char.id;
|
||||
charGrowthRate.idStat = stats[j].id;
|
||||
charGrowthRate.value = parseInt($(row.children[j + 1]).text()); // the first col of the table is for the name, so we have to add 1 to the index
|
||||
|
||||
CharacterGrowthRate.upsert({
|
||||
Id_Character: charGrowthRate.idCharacter,
|
||||
Id_Stat: charGrowthRate.idStat,
|
||||
value: charGrowthRate.value,
|
||||
}).catch(err => FE_Charts.logError(err));
|
||||
}
|
||||
} else {
|
||||
console.log(`Unknown character ${name}`);
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
})
|
||||
.catch(err => {
|
||||
FE_Charts.logError(err);
|
||||
});
|
||||
}
|
||||
|
||||
importCharGrowthRates();
|
||||
60
server/src/importClassGrowthRates.ts
Normal file
60
server/src/importClassGrowthRates.ts
Normal file
@ -0,0 +1,60 @@
|
||||
require("dotenv").config();
|
||||
const axios = require('axios').default;
|
||||
const cheerio = require('cheerio');
|
||||
|
||||
const {ClassGrowthRate} = require("./mapping");
|
||||
import {FE_Charts} from "./index";
|
||||
|
||||
const url:string = "https://serenesforest.net/three-houses/classes/growth-rates/";
|
||||
const nameIndex:number = 0;
|
||||
|
||||
|
||||
async function importClassGrowthRates() {
|
||||
const classes = await FE_Charts.getAllClasses();
|
||||
const stats = await FE_Charts.getAllStats();
|
||||
axios.get(url)
|
||||
.then(res => {
|
||||
// We need to remove all line breaks to avoid bad html format
|
||||
const html = res.data.replace(/[\n\r]/g, "");
|
||||
const $ = cheerio.load(html);
|
||||
|
||||
$("#enemy").remove(); // Remove all contents for enemies
|
||||
$("tbody").first().remove(); // Remove the first table who isn't for the growth rates
|
||||
$("tbody").last().remove(); // Remove the last table who is for enemies and not playable character
|
||||
|
||||
// parse each tables in the html
|
||||
$("tbody").each(function (i, elem) {
|
||||
|
||||
// parse each rows of the table
|
||||
elem.children.forEach((row, rowIndex) => {
|
||||
// The first element is the header of the table, so we must skip it
|
||||
if (rowIndex != 0) {
|
||||
const name = $(row.children[nameIndex]).text();
|
||||
const feClass = classes.find(x => x.name == name);
|
||||
|
||||
if (feClass !== undefined) {
|
||||
for (let j = 0; j < stats.length; j++) {
|
||||
const classGrowthRate = new ClassGrowthRate();
|
||||
classGrowthRate.idClass = feClass.id;
|
||||
classGrowthRate.idStat = stats[j].id;
|
||||
classGrowthRate.value = parseInt($(row.children[j + 1]).text()); // the first col of the table is for the name, so we have to add 1 to the index
|
||||
|
||||
if (classGrowthRate.value) {
|
||||
ClassGrowthRate.upsert({
|
||||
Id_Class: classGrowthRate.idClass,
|
||||
Id_Stat: classGrowthRate.idStat,
|
||||
value: classGrowthRate.value,
|
||||
}).catch(err => FE_Charts.logError(err));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
})
|
||||
.catch(err => {
|
||||
FE_Charts.logError(err);
|
||||
});
|
||||
}
|
||||
|
||||
importClassGrowthRates();
|
||||
@ -1,12 +1,15 @@
|
||||
require("dotenv").config();
|
||||
const axios = require('axios').default;
|
||||
const cheerio = require('cheerio');
|
||||
import * as express from "express";
|
||||
import * as bodyParser from "body-parser";
|
||||
import * as status from "http-status";
|
||||
|
||||
require("./mapping");
|
||||
const {Catalogue, Character, CharacterGrowthRate, ClassGrowthRate, FE_Class, Gender, House, Stat, RestrictedCharacterClass} = require("./mapping");
|
||||
|
||||
const app = express();
|
||||
const port = process.env.NODE_PORT || 3000;
|
||||
const argImport:string = "importing";
|
||||
|
||||
app.use(bodyParser.urlencoded({extended: false}));
|
||||
app.use(bodyParser.json());
|
||||
@ -19,10 +22,126 @@ app.use(function (req, res, next) {
|
||||
next();
|
||||
});
|
||||
|
||||
abstract class FE_Charts {
|
||||
static logError(err) {
|
||||
// TODO: Log to file and manage the error
|
||||
console.log(err);
|
||||
}
|
||||
|
||||
static async getAllCharacters() {
|
||||
let characters = null;
|
||||
await Character.findAll().then(results => {
|
||||
characters = results;
|
||||
}).catch(err => FE_Charts.logError(err));
|
||||
return characters;
|
||||
}
|
||||
|
||||
static async getAllCharactersGrowthRates() {
|
||||
let charactersGrowRates = null;
|
||||
await CharacterGrowthRate.findAll().then(results => {
|
||||
charactersGrowRates = results
|
||||
}).catch(err => FE_Charts.logError(err));
|
||||
return charactersGrowRates;
|
||||
}
|
||||
|
||||
static async getAllClassesGrowthRates() {
|
||||
let classesGrowRates = null;
|
||||
await ClassGrowthRate.findAll().then(results => {
|
||||
classesGrowRates = results
|
||||
}).catch(err => FE_Charts.logError(err));
|
||||
return classesGrowRates;
|
||||
}
|
||||
|
||||
static async getAllClasses() {
|
||||
let classes = null;
|
||||
await FE_Class.findAll().then(results => {
|
||||
classes = results
|
||||
}).catch(err => FE_Charts.logError(err));
|
||||
return classes;
|
||||
}
|
||||
|
||||
static async getAllGenders() {
|
||||
let genders = null;
|
||||
await Gender.findAll().then(results => {
|
||||
genders = results
|
||||
}).catch(err => FE_Charts.logError(err));
|
||||
return genders;
|
||||
}
|
||||
|
||||
static async getAllHouses() {
|
||||
let houses = null;
|
||||
await House.findAll().then(results => {
|
||||
houses = results
|
||||
}).catch(err => FE_Charts.logError(err));
|
||||
return houses;
|
||||
}
|
||||
|
||||
static async getAllRestrictedClasses() {
|
||||
let restrictedClasses = null;
|
||||
await RestrictedCharacterClass.findAll().then(results => {
|
||||
restrictedClasses = results
|
||||
}).catch(err => FE_Charts.logError(err));
|
||||
return restrictedClasses;
|
||||
}
|
||||
|
||||
static async getAllStats() {
|
||||
let stats = null;
|
||||
await Stat.findAll().then(results => {
|
||||
stats = results
|
||||
}).catch(err => FE_Charts.logError(err));
|
||||
return stats;
|
||||
}
|
||||
|
||||
static async getCatalogueLastVersion() {
|
||||
let version = null;
|
||||
await Catalogue.findAll({
|
||||
attributes: ["version", ["Dttm_Last_Update", "lastUpdate"]],
|
||||
limit: 1,
|
||||
order: [["Dttm_Last_Update", "DESC"]],
|
||||
}).then(results => {
|
||||
version = results
|
||||
}).catch(err => FE_Charts.logError(err));
|
||||
return version;
|
||||
}
|
||||
}
|
||||
|
||||
app.get('/', (req, res) => {
|
||||
res.send('Hello World!');
|
||||
});
|
||||
|
||||
app.listen(port, () => {
|
||||
console.log(`Example app listening on port ${port}!`);
|
||||
app.get('/version', (req, res) => {
|
||||
FE_Charts.getCatalogueLastVersion().then(version => {
|
||||
res.status(status.OK).send(version);
|
||||
});
|
||||
});
|
||||
|
||||
app.get('/all', async (req, res) => {
|
||||
const characters = await FE_Charts.getAllCharacters();
|
||||
const characterGrowthRates = await FE_Charts.getAllCharactersGrowthRates();
|
||||
const classesGrowthRates = await FE_Charts.getAllClassesGrowthRates();
|
||||
const classes = await FE_Charts.getAllClasses();
|
||||
const genders = await FE_Charts.getAllGenders();
|
||||
const houses = await FE_Charts.getAllHouses();
|
||||
const restrictedClasses = await FE_Charts.getAllRestrictedClasses();
|
||||
const stats = await FE_Charts.getAllStats();
|
||||
|
||||
res.status(status.OK).send({
|
||||
characters: characters,
|
||||
charGrowthRates: characterGrowthRates,
|
||||
classGrowthRates: classesGrowthRates,
|
||||
classes: classes,
|
||||
genders: genders,
|
||||
houses: houses,
|
||||
restrictedClasses: restrictedClasses,
|
||||
stats: stats,
|
||||
});
|
||||
});
|
||||
|
||||
// Don't start the server if we're importing some data
|
||||
if (process.argv.find(x => x == argImport) == undefined) {
|
||||
app.listen(port, () => {
|
||||
console.log(`Example app listening on port ${port}!`);
|
||||
});
|
||||
}
|
||||
|
||||
export {FE_Charts}
|
||||
Reference in New Issue
Block a user