app/constants.py

@@ -4,38 +4,42 @@ DATASETS = [
     "deconfinement-parking-relais-doublement-des-places",
 ]
 URL = "https://opendata.paris.fr/api/records/1.0/search/?dataset={}&q=&rows=-1"
-TEST_URL = "https://opendata.paris.fr/api/records/1.0/search/?dataset=deconfinement-pistes-cyclables-temporaires&rows=-1"
+FILENAMES = {
+    "coronavirus-commercants-parisiens-livraison-a-domicile": "home-delivery",
+    "deconfinement-pistes-cyclables-temporaires": "cycling-paths",
+    "deconfinement-parking-relais-doublement-des-places": "relay-parking",
+}
 FILES = {
-    "deconfinement-pistes-cyclables-temporaires": "data/cycling-paths.json",
-    "deconfinement-parking-relais-doublement-des-places": "data/relay-parking.json",
-    "coronavirus-commercants-parisiens-livraison-a-domicile": "data/home-delivery.json",
+    "cycling-paths": "data/cycling-paths.json",
+    "relay-parking": "data/relay-parking.json",
+    "home-delivery": "data/home-delivery.json",
 }
 COLUMNS = {
-    "deconfinement-pistes-cyclables-temporaires": [
-        "fields.geo_shape.coordinates",
-        "fields.statut",
+    "cycling-paths": [
+        ["fields", "geo_shape", "coordinates"],
+        "statut",
         "record_timestamp",
-        "fields.complement",
+        "complement",
     ],
-    "deconfinement-parking-relais-doublement-des-places": [
-        "fields.societe",
-        "fields.nb_places_dispositif_environ",
-        "fields.parcs",
-        "fields.geo_shape.coordinates",
-        "fields.cp",
-        "fields.ville",
-        "fields.adresse",
+    "relay-parking": [
+        "societe",
+        "nb_places_dispositif_environ",
+        "parcs",
+        "geo_shape",
+        "cp",
+        "ville",
+        "adresse",
     ],
-    "coronavirus-commercants-parisiens-livraison-a-domicile": [
-        "fields.geo_shape.coordinates",
-        "fields.adresse",
-        "fields.code_postal",
-        "fields.nom_du_commerce",
-        "fields.type_de_commerce",
-        "fields.site_internet",
+    "home-delivery": [
+        "geo_shape",
+        "adresse",
+        "code_postal",
+        "nom_du_commerce",
+        "type_du_commerce",
+        "site_internet",
         "record_timestamp",
-        "fields.precisions",
-        "fields.telephone",
-        "fields.mail",
+        "precisions",
+        "telephone",
+        "mail",
     ],
 }

app/preprocessing.py

@@ -1,6 +1,6 @@
 from json import load
-from pandas import json_normalize, DataFrame
-from .constants import FILES, COLUMNS
+from pandas import json_normalize, DataFrame, set_option
+from constants import FILES, COLUMNS
 
 
 def open_json(dataset) -> dict:
@@ -18,5 +18,4 @@ def create_dataframe(dataset) -> DataFrame:
     """
     json = open_json(dataset)
     df = json_normalize(data=json, record_path=["records"], errors="ignore",)
-    filtered_df = df.filter(items=COLUMNS[dataset])
-    return filtered_df
+    return df

app/request_datasets.py

@@ -1,6 +1,6 @@
 from json import dump
 from requests import get
-from .constants import FILES, URL
+from constants import FILENAMES, URL
 
 
 def format_url(dataset) -> str:
@@ -15,7 +15,8 @@ def save_json(data, dataset):
     """
     Dumps the data into a JSON file
     """
-    with open(FILES[dataset], "w") as f:
+    data_dir = "data/"
+    with open(data_dir + FILENAMES[dataset] + ".json", "w") as f:
         dump(data, f, ensure_ascii=False)
 
 

tests/preprocessing_test.py

@@ -1,21 +1,9 @@
-from pandas import DataFrame
-from requests import get
-from app.constants import URL, COLUMNS, FILES, DATASETS
-from app.request_datasets import request_dataset
+from app.constants import files
 from app.preprocessing import create_dataframe
-from os import remove
-
-
-def test_dataset_request():
-    for dataset in DATASETS:
-        response = get(URL.format(dataset))
-        assert response.status_code == 200
+from pandas import DataFrame
 
 
 def test_dataframe_creation():
-    for dataset in DATASETS:
-        request_dataset(dataset)
-        df = create_dataframe(dataset)
-        remove(FILES[dataset])
+    for file in files.keys():
+        df = create_dataframe(file)
         assert isinstance(df, DataFrame)
-        assert list(df) == COLUMNS[dataset]