Rename folders, add DB parser and improve DB setup
This commit is contained in:
@@ -9,6 +9,8 @@ verify_ssl = true
|
||||
flask = "*"
|
||||
pymysql = "*"
|
||||
flask-sqlalchemy = "*"
|
||||
pandas = "*"
|
||||
iso3166 = "*"
|
||||
|
||||
[requires]
|
||||
python_version = "3.8"
|
||||
|
||||
82
Code/Pipfile.lock
generated
82
Code/Pipfile.lock
generated
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"_meta": {
|
||||
"hash": {
|
||||
"sha256": "77464f98cb5a9b5d12663323acdbbc44b448b5d9d2fee7c8e0e6302987ef8bfe"
|
||||
"sha256": "180a68bb151e2c8ab26c87081302da4bd38107869f77507d721f7502631c4684"
|
||||
},
|
||||
"pipfile-spec": 6,
|
||||
"requires": {
|
||||
@@ -39,6 +39,14 @@
|
||||
"index": "pypi",
|
||||
"version": "==2.4.1"
|
||||
},
|
||||
"iso3166": {
|
||||
"hashes": [
|
||||
"sha256:b07208703bd881a4f974e39fa013c4498dddd64913ada15f24be75d02ae68a44",
|
||||
"sha256:b1e58dbcf50fbb2c9c418ec7a6057f0cdb30b8f822ac852f72e71ba769dae8c5"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==1.0.1"
|
||||
},
|
||||
"itsdangerous": {
|
||||
"hashes": [
|
||||
"sha256:321b033d07f2a4136d3ec762eac9f16a10ccd60f53c0c91af90217ace7ba1f19",
|
||||
@@ -86,6 +94,57 @@
|
||||
],
|
||||
"version": "==1.1.1"
|
||||
},
|
||||
"numpy": {
|
||||
"hashes": [
|
||||
"sha256:03bbde29ac8fba860bb2c53a1525b3604a9b60417855ac3119d89868ec6041c3",
|
||||
"sha256:1baefd1fb4695e7f2e305467dbd876d765e6edd30c522894df76f8301efaee36",
|
||||
"sha256:1c35fb1131362e6090d30286cfda52ddd42e69d3e2bf1fea190a0fad83ea3a18",
|
||||
"sha256:3c68c827689ca0ca713dba598335073ce0966850ec0b30715527dce4ecd84055",
|
||||
"sha256:443ab93fc35b31f01db8704681eb2fd82f3a1b2fa08eed2dd0e71f1f57423d4a",
|
||||
"sha256:56710a756c5009af9f35b91a22790701420406d9ac24cf6b652b0e22cfbbb7ff",
|
||||
"sha256:62506e9e4d2a39c87984f081a2651d4282a1d706b1a82fe9d50a559bb58e705a",
|
||||
"sha256:6f8113c8dbfc192b58996ee77333696469ea121d1c44ea429d8fd266e4c6be51",
|
||||
"sha256:712f0c32555132f4b641b918bdb1fd3c692909ae916a233ce7f50eac2de87e37",
|
||||
"sha256:854f6ed4fa91fa6da5d764558804ba5b0f43a51e5fe9fc4fdc93270b052f188a",
|
||||
"sha256:88c5ccbc4cadf39f32193a5ef22e3f84674418a9fd877c63322917ae8f295a56",
|
||||
"sha256:905cd6fa6ac14654a6a32b21fad34670e97881d832e24a3ca32e19b455edb4a8",
|
||||
"sha256:9d6de2ad782aae68f7ed0e0e616477fbf693d6d7cc5f0f1505833ff12f84a673",
|
||||
"sha256:a30f5c3e1b1b5d16ec1f03f4df28e08b8a7529d8c920bbed657f4fde61f1fbcd",
|
||||
"sha256:a9d72d9abaf65628f0f31bbb573b7d9304e43b1e6bbae43149c17737a42764c4",
|
||||
"sha256:ac3cf835c334fcc6b74dc4e630f9b5ff7b4c43f7fb2a7813208d95d4e10b5623",
|
||||
"sha256:b091e5d4cbbe79f0e8b6b6b522346e54a282eadb06e3fd761e9b6fafc2ca91ad",
|
||||
"sha256:cc070fc43a494e42732d6ae2f6621db040611c1dde64762a40c8418023af56d7",
|
||||
"sha256:e1080e37c090534adb2dd7ae1c59ee883e5d8c3e63d2a4d43c20ee348d0459c5",
|
||||
"sha256:f084d513de729ff10cd72a1f80db468cff464fedb1ef2fea030221a0f62d7ff4",
|
||||
"sha256:f6a7421da632fc01e8a3ecd19c3f7350258d82501a646747664bae9c6a87c731"
|
||||
],
|
||||
"version": "==1.18.0"
|
||||
},
|
||||
"pandas": {
|
||||
"hashes": [
|
||||
"sha256:00dff3a8e337f5ed7ad295d98a31821d3d0fe7792da82d78d7fd79b89c03ea9d",
|
||||
"sha256:22361b1597c8c2ffd697aa9bf85423afa9e1fcfa6b1ea821054a244d5f24d75e",
|
||||
"sha256:255920e63850dc512ce356233081098554d641ba99c3767dde9e9f35630f994b",
|
||||
"sha256:26382aab9c119735908d94d2c5c08020a4a0a82969b7e5eefb92f902b3b30ad7",
|
||||
"sha256:33970f4cacdd9a0ddb8f21e151bfb9f178afb7c36eb7c25b9094c02876f385c2",
|
||||
"sha256:4545467a637e0e1393f7d05d61dace89689ad6d6f66f267f86fff737b702cce9",
|
||||
"sha256:52da74df8a9c9a103af0a72c9d5fdc8e0183a90884278db7f386b5692a2220a4",
|
||||
"sha256:61741f5aeb252f39c3031d11405305b6d10ce663c53bc3112705d7ad66c013d0",
|
||||
"sha256:6a3ac2c87e4e32a969921d1428525f09462770c349147aa8e9ab95f88c71ec71",
|
||||
"sha256:7458c48e3d15b8aaa7d575be60e1e4dd70348efcd9376656b72fecd55c59a4c3",
|
||||
"sha256:78bf638993219311377ce9836b3dc05f627a666d0dbc8cec37c0ff3c9ada673b",
|
||||
"sha256:8153705d6545fd9eb6dd2bc79301bff08825d2e2f716d5dced48daafc2d0b81f",
|
||||
"sha256:975c461accd14e89d71772e89108a050fa824c0b87a67d34cedf245f6681fc17",
|
||||
"sha256:9962957a27bfb70ab64103d0a7b42fa59c642fb4ed4cb75d0227b7bb9228535d",
|
||||
"sha256:adc3d3a3f9e59a38d923e90e20c4922fc62d1e5a03d083440468c6d8f3f1ae0a",
|
||||
"sha256:bbe3eb765a0b1e578833d243e2814b60c825b7fdbf4cdfe8e8aae8a08ed56ecf",
|
||||
"sha256:df8864824b1fe488cf778c3650ee59c3a0d8f42e53707de167ba6b4f7d35f133",
|
||||
"sha256:e45055c30a608076e31a9fcd780a956ed3b1fa20db61561b8d88b79259f526f7",
|
||||
"sha256:ee50c2142cdcf41995655d499a157d0a812fce55c97d9aad13bc1eef837ed36c"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==0.25.3"
|
||||
},
|
||||
"pymysql": {
|
||||
"hashes": [
|
||||
"sha256:3943fbbbc1e902f41daf7f9165519f140c4451c179380677e6a848587042561a",
|
||||
@@ -94,6 +153,27 @@
|
||||
"index": "pypi",
|
||||
"version": "==0.9.3"
|
||||
},
|
||||
"python-dateutil": {
|
||||
"hashes": [
|
||||
"sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c",
|
||||
"sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a"
|
||||
],
|
||||
"version": "==2.8.1"
|
||||
},
|
||||
"pytz": {
|
||||
"hashes": [
|
||||
"sha256:1c557d7d0e871de1f5ccd5833f60fb2550652da6be2693c1e02300743d21500d",
|
||||
"sha256:b02c06db6cf09c12dd25137e563b31700d3b80fcc4ad23abb7a315f2789819be"
|
||||
],
|
||||
"version": "==2019.3"
|
||||
},
|
||||
"six": {
|
||||
"hashes": [
|
||||
"sha256:1f1b7d42e254082a9db6279deae68afb421ceba6158efa6131de7b3003ee93fd",
|
||||
"sha256:30f610279e8b2578cab6db20741130331735c781b56053c59c4076da27f06b66"
|
||||
],
|
||||
"version": "==1.13.0"
|
||||
},
|
||||
"sqlalchemy": {
|
||||
"hashes": [
|
||||
"sha256:bfb8f464a5000b567ac1d350b9090cf081180ec1ab4aa87e7bca12dab25320ec"
|
||||
|
||||
@@ -1,25 +1,49 @@
|
||||
from os import environ
|
||||
|
||||
from sqlalchemy import create_engine
|
||||
from subprocess import call
|
||||
from sys import argv
|
||||
from flask_sqlalchemy import SQLAlchemy
|
||||
from flask import Flask
|
||||
from fields import Glacier, Annual_Data, Annual_Change, User
|
||||
|
||||
|
||||
def create_connection():
|
||||
db_name = get_env_variable("DB_NAME")
|
||||
db_user = get_env_variable("DB_USER")
|
||||
db_password = get_env_variable("DB_PASSWORD")
|
||||
db_url = get_env_variable("DB_URL")
|
||||
db_connection_uri = "mysql+pymysql://{user}:{pw}@{url}/{db}".format(
|
||||
user=db_user, pw=db_password, url=db_url, db=db_name
|
||||
def create_database(db_name, db_user, db_pw):
|
||||
script = "mariadb_setup.sh"
|
||||
output = call([script + db_name + db_user + db_pw])
|
||||
if output != 0:
|
||||
print("Error: couldn't create database")
|
||||
exit()
|
||||
|
||||
|
||||
def create_connection(db_name, db_user, db_pw):
|
||||
host = "localhost:3306"
|
||||
connection_uri = "mysql+pymysql://{user}:{pw}@{url}/{db}".format(
|
||||
user=db_user, pw=db_pw, url=host, db=db_name
|
||||
)
|
||||
engine = create_engine(db_connection_uri, echo=True)
|
||||
app = Flask(__name__)
|
||||
app.config["SQLALCHEMY_DATABASE_URI"] = connection_uri
|
||||
app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False
|
||||
global db
|
||||
db = SQLAlchemy(app)
|
||||
|
||||
|
||||
def create_tables():
|
||||
def create_tables(db):
|
||||
db.create_all()
|
||||
|
||||
|
||||
def get_env_variable(name):
|
||||
try:
|
||||
return environ[name]
|
||||
except KeyError:
|
||||
message = "Expected environment variable '{}' not set.".format(name)
|
||||
raise Exception(message)
|
||||
def main():
|
||||
if len(argv) != 3:
|
||||
print(
|
||||
"Usage: " + argv[0] + " <database name> <database user> <database password>"
|
||||
)
|
||||
exit()
|
||||
|
||||
db_name = str(argv[1])
|
||||
db_user = str(argv[2])
|
||||
db_password = str(argv[3])
|
||||
|
||||
create_database(db_name, db_user, db_password)
|
||||
create_connection(db_name, db_user, db_password)
|
||||
create_tables(db)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
52
Code/database/fields.py
Normal file
52
Code/database/fields.py
Normal file
@@ -0,0 +1,52 @@
|
||||
from db_setup import db
|
||||
from datetime import datetime
|
||||
|
||||
|
||||
class Glacier(db.Model):
|
||||
uid = db.Column(db.Integer, primary_key=True)
|
||||
country = db.Column(db.String(30))
|
||||
name = db.Column(db.String(30))
|
||||
|
||||
def __init__(self, uid, country, name):
|
||||
self.uid = uid
|
||||
self.country = country
|
||||
self.name = name
|
||||
|
||||
|
||||
class Annual_Data(db.Model):
|
||||
glacier = db.relationship("Glacier", backref=db.backref("posts", lazy=True))
|
||||
uid = db.Column(db.Integer, db.ForeignKey("glacier.uid"), nullable=False)
|
||||
surface = db.Column(db.Integer)
|
||||
volume = db.Column(db.Integer)
|
||||
thickness = db.Column(db.Integer)
|
||||
year = db.Column(db.Integer)
|
||||
|
||||
def __init__(self, surface, volume, thickness, year):
|
||||
self.surface = surface
|
||||
self.volume = volume
|
||||
self.thickness = thickness
|
||||
self.year = year
|
||||
|
||||
|
||||
class Annual_Change(db.Model):
|
||||
glacier = db.relationship("Glacier", backref=db.backref("posts", lazy=True))
|
||||
uid = db.Column(db.Integer, db.ForeignKey("glacier.uid"), nullable=False)
|
||||
surface_change = db.Column(db.Integer)
|
||||
volume_change = db.Column(db.Integer)
|
||||
thickness_change = db.Column(db.Integer)
|
||||
year = db.Column(db.Integer)
|
||||
|
||||
def __init__(self, surface_change, volume_change, thickness_change, year):
|
||||
self.surface_change = surface_change
|
||||
self.volume_change = volume_change
|
||||
self.thickness_change = thickness_change
|
||||
self.year = year
|
||||
|
||||
|
||||
class Users(db.Model):
|
||||
uid = db.Column(db.Integer, primary_key=True)
|
||||
registration_date = db.Column(db.DateTime, nullable=False)
|
||||
|
||||
def __init__(self, uid):
|
||||
self.uid = uid
|
||||
self.registration_date = datetime.today()
|
||||
16
Code/database/parse_csv.py
Normal file
16
Code/database/parse_csv.py
Normal file
@@ -0,0 +1,16 @@
|
||||
from pandas import read_csv, concat, DataFrame
|
||||
from iso3166 import countries
|
||||
|
||||
|
||||
def select_columns() -> [DataFrame]:
|
||||
min_year = 2010
|
||||
fields = ["POLITICAL_UNIT", "WGMS_ID", "YEAR"]
|
||||
iter_csv = read_csv(
|
||||
"../../Assets/WGMS-FoG-2019-12-D-CHANGE.csv",
|
||||
skipinitialspace=True,
|
||||
usecols=fields,
|
||||
iterator=True,
|
||||
chunksize=100,
|
||||
)
|
||||
data = concat([chunk[chunk["YEAR"] > min_year] for chunk in iter_csv])
|
||||
return data
|
||||
Reference in New Issue
Block a user