Add SQL insertion functionality

code/database/db_setup.py

@@ -29,9 +29,11 @@ db = create_connection()
 
 
 class Glacier(db.Model):
-    uid = db.Column(db.Integer, primary_key=True)
+    uid = db.Column(db.String(5), primary_key=True)
     country = db.Column(db.String(60))
     name = db.Column(db.String(60))
+    annual_data = db.relationship("Annual_Data")
+    annual_change = db.relationship("Annual_Change")
 
     def __init__(self, uid, country, name):
         self.uid = uid
@@ -40,37 +42,30 @@ class Glacier(db.Model):
 
 
 class Annual_Data(db.Model):
-    glacier = db.relationship("Glacier", backref=db.backref("posts", lazy=True))
-    uid = db.Column(
-        db.Integer, db.ForeignKey("glacier.uid"), nullable=False, primary_key=True
-    )
+    year = db.Column(db.Integer, primary_key=True)
+    uid = db.Column(db.String(5), db.ForeignKey("glacier.uid"), primary_key=True)
     surface = db.Column(db.Integer)
     length = db.Column(db.Integer)
     elevation = db.Column(db.Integer)
-    year = db.Column(db.Integer, primary_key=True)
 
-    def __init__(self, surface, length, elevation, year):
+    def __init__(self, year, surface, length, elevation):
+        self.year = year
         self.surface = surface
         self.length = length
         self.elevation = elevation
-        self.year = year
 
 
 class Annual_Change(db.Model):
-    glacier = db.relationship("Glacier", backref=db.backref("posts", lazy=True))
-    uid = db.Column(
-        db.Integer, db.ForeignKey("glacier.uid"), nullable=False, primary_key=True
-    )
+    year = db.Column(db.Integer, db.ForeignKey("annual__data.year"), primary_key=True)
+    uid = db.Column(db.String(5), db.ForeignKey("glacier.uid"), primary_key=True)
     surface_change = db.Column(db.Integer)
     length_change = db.Column(db.Integer)
     elevation_change = db.Column(db.Integer)
-    year = db.Column(db.Integer, primary_key=True)
 
-    def __init__(self, surface_change, length_change, elevation_change, year):
+    def __init__(self, surface_change, length_change, elevation_change):
         self.surface_change = surface_change
         self.length_change = length_change
         self.elevation_change = elevation_change
-        self.year = year
 
 
 class User(db.Model):

code/database/parser.py

@@ -34,8 +34,24 @@ def select_columns() -> DataFrame:
     return data
 
 
+def rename_fields(df_list):
+    new_df_list = {}
+    new_fields = {
+        "POLITICAL_UNIT": "country",
+        "NAME": "name",
+        "WGMS_ID": "uid",
+        "YEAR": "year",
+        "MEDIAN_ELEVATION": "elevation",
+        "AREA": "surface",
+        "LENGTH": "length",
+    }
+    for key, value in df_list.items():
+        new_df_list[key] = value.rename(columns=new_fields)
+    return new_df_list
+
+
 def create_databases(df):
-    users = {"UID": [7843], "USERNAME": ["admin"], "PASSWORD": [ADMIN_PW]}
+    users = {"uid": [7843], "username": ["admin"], "password": [ADMIN_PW]}
     files = {
         "glacier": "../data/glacier.csv",
         "annual__data": "../data/annual_data.csv",
@@ -48,7 +64,8 @@ def create_databases(df):
         "annual__change": df[["WGMS_ID", "YEAR"]],
         "users": DataFrame(users),
     }
-    for key, val in dataframes.items():
+    renamed_dfs = rename_fields(dataframes)
+    for key, val in renamed_dfs.items():
         val.to_csv(files[key], index=False, quoting=QUOTE_NONNUMERIC)
 
 

code/database/storage.py

@@ -1,5 +1,15 @@
-from db_setup import create_connection
-from pandas import DataFrame, read_csv, to_sql
+from constants import DB_NAME, DB_PW, DB_USER
+from pandas import DataFrame, read_csv
+from sqlalchemy import create_engine, engine
+
+
+def create_connection() -> engine:
+    host = "localhost:3306"
+    connection_uri = "mysql+pymysql://{user}:{pw}@{url}/{db}".format(
+        user=DB_USER, pw=DB_PW, url=host, db=DB_NAME
+    )
+    engine = create_engine(connection_uri)
+    return engine
 
 
 def create_dataframes() -> DataFrame:
@@ -11,13 +21,13 @@ def create_dataframes() -> DataFrame:
     }
     df_list = {}
     for csv in files.keys():
-        df_list[csv] = read_csv(files[csv], index=False)
+        df_list[csv] = read_csv(files[csv])
     return df_list
 
 
 def insert_data(df_list, conn):
     for key, value in df_list.items():
-        value.to_sql(key, con=conn, if_exists="replace", index=False)
+        value.to_sql(key, con=conn, index=False, if_exists="append", chunksize=1)
 
 
 def main():