Compare commits
No commits in common. "dd7f1bab8d83d6f91ffbe21c2b63e3cbbb06ffb3" and "bace83dc3ae3e884d4d85f964a984e82dc8005eb" have entirely different histories.
dd7f1bab8d
...
bace83dc3a
@ -1,10 +1,6 @@
|
|||||||
from re import findall
|
|
||||||
from typing import List
|
|
||||||
|
|
||||||
from bs4 import BeautifulSoup
|
|
||||||
from requests import get
|
from requests import get
|
||||||
|
|
||||||
from constants import FLICKR_URL, URL
|
from constants import URL
|
||||||
|
|
||||||
|
|
||||||
def format_url(dataset) -> str:
|
def format_url(dataset) -> str:
|
||||||
@ -25,26 +21,3 @@ def request_dataset(dataset):
|
|||||||
response.raise_for_status()
|
response.raise_for_status()
|
||||||
data = response.json()
|
data = response.json()
|
||||||
return data
|
return data
|
||||||
|
|
||||||
|
|
||||||
def request_flickr(keywords) -> str:
|
|
||||||
"""
|
|
||||||
Returns the HTML of a Flickr search
|
|
||||||
"""
|
|
||||||
search_url = FLICKR_URL.format(keywords)
|
|
||||||
result = get(search_url)
|
|
||||||
html = result.text
|
|
||||||
return html
|
|
||||||
|
|
||||||
|
|
||||||
def scrap_flickr(keywords) -> List[str]:
|
|
||||||
"""
|
|
||||||
Creates a list of image links from a Flickr search
|
|
||||||
"""
|
|
||||||
html = request_flickr(keywords)
|
|
||||||
soup = BeautifulSoup(html, features="html.parser")
|
|
||||||
images = soup.find_all(
|
|
||||||
"div", class_="view photo-list-photo-view requiredToShowOnServer awake",
|
|
||||||
)
|
|
||||||
image_links = findall("(live.staticflickr.com/\S+.jpg)", str(images))
|
|
||||||
return image_links
|
|
||||||
|
@ -5,7 +5,6 @@ DATASETS = [
|
|||||||
"deconfinement-rues-amenagees-pour-pietons",
|
"deconfinement-rues-amenagees-pour-pietons",
|
||||||
]
|
]
|
||||||
URL = "https://opendata.paris.fr/api/records/1.0/search/?dataset={}&q=&rows=-1"
|
URL = "https://opendata.paris.fr/api/records/1.0/search/?dataset={}&q=&rows=-1"
|
||||||
FLICKR_URL = "https://www.flickr.com/search/?text={}"
|
|
||||||
COLUMNS = {
|
COLUMNS = {
|
||||||
"deconfinement-pistes-cyclables-temporaires": [
|
"deconfinement-pistes-cyclables-temporaires": [
|
||||||
"fields.geo_shape.coordinates",
|
"fields.geo_shape.coordinates",
|
||||||
|
Loading…
Reference in New Issue
Block a user