Fix typo on readme

README.md

@@ -0,0 +1,80 @@
+graphPaname
+===========
+
+graphPaname is a system that collects real-time data, relevant to the
+COVID-19 pandemic de-escalation, from the city of Paris.
+
+It works with 4 datasets about the de-escalation:
+
+-   Retailers with home delivery
+-   Additional parking places in relay parkings (parkings connected to
+    public transportation)
+-   Temporary cycling paths
+-   Temporary pedestrian streets
+
+For each dataset, we offer a table with the data, and a map of Paris
+with markers. Additionally, there\'s a section with photos related to
+the COVID-19 pandemic.
+
+Technologies
+------------
+
+-   Flask
+-   Pandas
+-   Folium
+
+Data sources
+------------
+
+-   [Open Data](https://opendata.paris.fr/pages/home/)
+-   [OpenStreetMap](https://www.openstreetmap.org/)
+-   [Flickr](https://flickr.com)
+
+Requirements
+------------
+
+-   Nix
+
+Installation
+------------
+
+1.  Install Nix (compatible with MacOS and Linux):
+
+``` {.shell}
+curl -L https://nixos.org/nix/install | sh
+```
+
+There are alternative installation methods, if you don\'t want to pipe
+curl to sh
+
+2.  Clone the repository:
+
+``` {.shell}
+git clone https://coolneng.duckdns.org/gitea/coolneng/graphPaname
+```
+
+3.  Change the working directory to the project:
+
+``` {.shell}
+cd graphPaname
+```
+
+4.  Enter the nix-shell:
+
+``` {.shell}
+nix-shell
+```
+
+5.  Run the tests:
+
+``` {.shell}
+pytest
+```
+
+6.  Execute the Flask application:
+
+``` {.shell}
+flask run
+```
+
+The website can be accessed via **localhost:5000**

README.org

@@ -1,4 +0,0 @@
-* graphPaname
-
-This project aims to gather information about the smart city of Paris and
-organize it in different plots and tables.

app/__init__.py

@@ -5,6 +5,7 @@ from flask_bootstrap import Bootstrap
 
 app = Flask(__name__)
 app.secret_key = SECRET_KEY
+app.config['TEMPLATES_AUTO_RELOAD'] = True
 bootstrap = Bootstrap(app)
 
 from app import errors, routes

app/data_request.py

@@ -1,13 +1,17 @@
+from re import findall
+from typing import List
+
+from bs4 import BeautifulSoup
 from requests import get
 
-from constants import URL
+from constants import FLICKR_URL, DATASET_URL
 
 
 def format_url(dataset) -> str:
     """
     Constructs the API's URL for the requested dataset
     """
-    link = URL.format(dataset)
+    link = DATASET_URL.format(dataset)
     return link
 
 
@@ -21,3 +25,35 @@ def request_dataset(dataset):
     response.raise_for_status()
     data = response.json()
     return data
+
+
+def request_flickr(keywords) -> str:
+    """
+    Returns the HTML of a Flickr search
+    """
+    search_url = FLICKR_URL.format(keywords)
+    result = get(search_url)
+    html = result.text
+    return html
+
+
+def extract_urls(images):
+    """
+    Creates proper URLs from the regex matches
+    """
+    links = findall("(live.staticflickr.com/\S+.jpg)", str(images))
+    formatted_urls = ["https://" + link for link in links]
+    return formatted_urls
+
+
+def scrape_flickr(keywords) -> List[str]:
+    """
+    Creates a list of image links from a Flickr search
+    """
+    html = request_flickr(keywords)
+    soup = BeautifulSoup(html, features="html.parser")
+    images = soup.find_all(
+        "div", class_="view photo-list-photo-view requiredToShowOnServer awake",
+    )
+    links = extract_urls(images)
+    return links

app/forms.py

@@ -4,5 +4,9 @@ from wtforms import SelectField, SubmitField
 
 
 class DatasetForm(FlaskForm):
+    """
+    Web form to select a dataset
+    """
+
     dataset = SelectField(choices=CHOICES)
     submit = SubmitField("Submit")

app/preprocessing.py

@@ -1,4 +1,4 @@
-from folium import Map, Marker
+from folium import Map, Marker, PolyLine
 from pandas import DataFrame, json_normalize
 
 from app.data_request import request_dataset
@@ -15,11 +15,24 @@ def create_dataframe(dataset) -> DataFrame:
     return filtered_df
 
 
+def reverse_coordinates(row):
+    """
+    Reverses each tuples coordinates to ensure folium can parse them correctly
+    """
+    coord = [tuple(reversed(t)) for t in row]
+    return coord
+
+
 def create_map(df):
     """
-    Creates a Map with markers from the DataFrame
+    Creates a Map with markers or lines from the DataFrame
     """
-    m = Map(location=COORDINATES, zoom_start=12)
+    m = Map(location=COORDINATES, zoom_start=12, tiles="Stamen Terrain")
     for index, row in df.iterrows():
-        Marker(location=row["fields.geo_shape.coordinates"]).add_to(m)
+        if row["fields.geo_shape.type"] == "LineString":
+            coord = reverse_coordinates(row["fields.geo_shape.coordinates"])
+            PolyLine(locations=coord, color="blue", opacity=0.5).add_to(m)
+        else:
+            lng, lat = row["fields.geo_shape.coordinates"]
+            Marker(location=[lat, lng]).add_to(m)
     m.save("app/templates/map.html")

app/processing.py

@@ -2,12 +2,18 @@ from app.preprocessing import create_dataframe, create_map
 
 
 def create_table(df) -> str:
+    """
+    Renders an HTML table from a DataFrame
+    """
     df.fillna(value=0, inplace=True)
-    table = df.to_html(classes=["table-striped", "table-hover"])
+    table = df.to_html(classes=["table-striped", "table-sm", "table-responsive"])
     return table
 
 
 def process_data(dataset):
+    """
+    Creates the DataFrame, produces a map and returns a table
+    """
     df = create_dataframe(dataset)
     table = create_table(df)
     create_map(df)

app/pruebascraping.py

@@ -1,25 +0,0 @@
-from bs4 import BeautifulSoup
-from requests import get
-from constants import FLICKR_URL
-from re import findall
-from typing import List
-
-
-def request_flickr(keywords) -> str:
-    search_url = FLICKR_URL.format(keywords)
-    result = get(search_url)
-    html = result.text
-    return html
-
-
-def scrap_flickr(keywords) -> List[str]:
-    html = request_flickr(keywords)
-    soup = BeautifulSoup(html, features="html.parser")
-    images = soup.find_all(
-        "div", class_="view photo-list-photo-view requiredToShowOnServer awake",
-    )
-    image_links = findall("(live.staticflickr.com/\S+.jpg)", str(images))
-    return image_links
-
-
-scrap_flickr("paris")

app/routes.py

@@ -3,6 +3,7 @@ from flask import render_template
 from app import app
 from app.forms import DatasetForm
 from app.processing import process_data
+from app.data_request import scrape_flickr
 
 
 @app.route("/")
@@ -27,4 +28,10 @@ def visualization():
 
 @app.route("/map")
 def map():
-    return render_template("map.html", title="Map")
+    return render_template("map.html")
+
+
+@app.route("/photos")
+def photos():
+    images = scrape_flickr("paris coronavirus")
+    return render_template("photos.html", title="Photos", images=images)

app/templates/base.html

@@ -22,6 +22,7 @@
                 <a class="nav-link" href="{{ url_for('index') }}">Home <span class="sr-only">(current)</span></a>
             </li>
             <li class="nav-link"><a href="{{ url_for('data') }}">Data</a></li>
+            <li class="nav-link"><a href="{{ url_for('photos') }}">Photos</a></li>
         </ul>
     </div>
 </nav>

app/templates/index.html

@@ -3,6 +3,24 @@
 {% block content %}
 <div class="jumbotron">
     <h1 id="graphPaname">graphPaname</h1>
-    <p>graphPaname is an information system that aims to show real-time data, related to the COVID-19 outbreak, in the city of Paris</p>
+    <p>
+    graphPaname is a system that collects real-time data, relevant to the COVID-19 pandemic de-escalation,  from the city of Paris.
+    </p>
+
+
+    <p>
+    It works with 4 datasets about the de-escalation:
+    </p>
+
+    <ul class="org-ul">
+    <li>Retailers with home delivery</li>
+    <li>Additional parking places in relay parkings (parkings connected to public transportation)</li>
+    <li>Temporary cycling paths</li>
+    <li>Temporary pedestrian streets</li>
+    </ul>
+
+    <p>
+    For each dataset, we offer a table with the data, and a map of Paris with markers. Additionally, there&rsquo;s a section with photos related to the COVID-19 pandemic.
+    </p>
 </div>
 {% endblock %}

app/templates/photos.html

@@ -0,0 +1,9 @@
+{% extends "base.html" %}
+{% import 'bootstrap/wtf.html' as wtf %}
+
+{% block app_content %}
+<h1>Photos</h1>
+{% for img_path in images %}
+   <img src="{{img_path|safe}}" alt="Image placeholder" id="photo" style="width: 200px"/>
+{% endfor %}
+{% endblock %}

app/templates/visualization.html

@@ -3,9 +3,13 @@
 
 {% block app_content %}
 <h1>Dataset visualization</h1>
-<iframe class="map", src="/map" width="400" height="400"></iframe>
-<table style="margin-left: 350px;">
-{{ table|safe }}
-</table>
+<div class="row">
+    <div class="col-md-9">
+        {{ table|safe }}
+    </div>
+    <div class="col-md-1">
+        <iframe id="map", src="/map" width="350" height="350"></iframe>
+    </div>
+</div>
 <p><a href="{{ url_for('data') }}">Back</a></p>
 {% endblock %}

constants.py

@@ -4,10 +4,11 @@ DATASETS = [
     "deconfinement-parking-relais-doublement-des-places",
     "deconfinement-rues-amenagees-pour-pietons",
 ]
-URL = "https://opendata.paris.fr/api/records/1.0/search/?dataset={}&q=&rows=-1"
+DATASET_URL = "https://opendata.paris.fr/api/records/1.0/search/?dataset={}&q=&rows=-1"
 FLICKR_URL = "https://www.flickr.com/search/?text={}"
 COLUMNS = {
     "deconfinement-pistes-cyclables-temporaires": [
+        "fields.geo_shape.type",
         "fields.geo_shape.coordinates",
         "fields.statut",
         "record_timestamp",
@@ -17,12 +18,14 @@ COLUMNS = {
         "fields.societe",
         "fields.nb_places_dispositif_environ",
         "fields.parcs",
+        "fields.geo_shape.type",
         "fields.geo_shape.coordinates",
         "fields.cp",
         "fields.ville",
         "fields.adresse",
     ],
     "coronavirus-commercants-parisiens-livraison-a-domicile": [
+        "fields.geo_shape.type",
         "fields.geo_shape.coordinates",
         "fields.adresse",
         "fields.code_postal",
@@ -35,6 +38,7 @@ COLUMNS = {
         "fields.mail",
     ],
     "deconfinement-rues-amenagees-pour-pietons": [
+        "fields.geo_shape.type",
         "fields.geo_shape.coordinates",
         "fields.nom_voie",
         "fields.categorie",

shell.nix

@@ -10,17 +10,8 @@ pkgs.mkShell {
     flask
     flask-bootstrap
     flask_wtf
-    matplotlib
     folium
     pytest
     beautifulsoup4
-    # Development tools
-    black
-    isort
-    pyflakes
-    python-language-server
-    pyls-black
-    pyls-isort
-    pyls-mypy
   ];
 }

tests/preprocessing_test.py

@@ -3,7 +3,7 @@ from requests import get
 
 from app.preprocessing import create_dataframe
 from app.data_request import request_dataset
-from constants import COLUMNS, DATASETS, URL
+from constants import COLUMNS, DATASETS, DATASET_URL, FLICKR_URL
 
 
 def test_dataset_request():
@@ -11,7 +11,7 @@ def test_dataset_request():
     Checks that the datasets URLs are reachable
     """
     for dataset in DATASETS:
-        response = get(URL.format(dataset))
+        response = get(DATASET_URL.format(dataset))
         assert response.status_code == 200
 
 
@@ -23,3 +23,11 @@ def test_dataframe_creation():
         df = create_dataframe(dataset)
         assert isinstance(df, DataFrame)
         assert all(df.columns == COLUMNS[dataset])
+
+
+def test_flickr_request():
+    """
+    Checks that Flickr search is avalaible
+    """
+    response = get(FLICKR_URL.format("paris coronavirus"))
+    assert response.status_code == 200