moved from urllib to requests

src/recipe_graph/scrape.py

@@ -4,7 +4,7 @@ import re
 from sqlalchemy import select, desc, exists, not_, except_
 from sqlalchemy.orm import sessionmaker
 import bs4
-from urllib.request import urlopen
+import requests as req
 from urllib.parse import urljoin
 import logging
 from argparse import ArgumentParser
@@ -73,23 +73,22 @@ def reparse_ingredients(session):
 
 
 
-def load_recipe(recipe_url):
+def load_page(recipe_url):
     try:    
-        logging.info(f'Loading Recipe: {recipe_url}')
-        with urlopen(recipe_url) as f:
-            if f.getcode() == 404:
-                raise Exception(f"Recipe Does not exist: {recipe_url}")
-            return bs4.BeautifulSoup(f.read().decode(), 'html.parser')
+        logging.info(f'Loading Page: {recipe_url}')
+        with req.get(recipe_url) as resp:
+            if resp.status_code == 404:
+                raise Exception(f"Page does not exist (404): {recipe_url}")
+            return bs4.BeautifulSoup(resp.text, 'html.parser')
 
     except Exception as e:
         logging.warning(f"Could not download or parse recipe: {recipe_url}")
         logging.warning(e)
 
-    return None
 
 def parse_recipe(session, recipe, site):
     recipe_url = urljoin(site.base_url, str(recipe.identifier))
-    recipe_page = load_recipe(recipe_url)
+    recipe_page = load_page(recipe_url)
     if not recipe_page:
         return None
 
@@ -126,7 +125,7 @@ def parse_recipe(session, recipe, site):
     
     return recipe
     
-def main():
+def main(): # pragma: no cover
     parser = ArgumentParser(description="Scrape a recipe site for recipies")
     parser.add_argument('site',
                         help='Name of site')