tp/webscraping.py at master · JustinHatesFun/tp · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
# Returns the forecast, news, and calendar events by parsing through html and
# using google API

from __future__ import print_function
import requests
from bs4 import BeautifulSoup
from newspaper import fulltext
import newspaper
from apiclient.discovery import build
from httplib2 import Http
from oauth2client import file, client, tools
import datetime
import logging


# Returns a dictionary mapping periods of time to their weather
def returnForecast():

    forecast = {}

    # Sometimes the website is down and causes errors
    try:

        page =  requests.get("https://forecast.weather.gov/MapClick.php?x=212&y=" +
                "132&site=pbz&zmx=&zmy=&map_x=212&map_y=132#.WtfexYjwbD4")
        soup = BeautifulSoup(page.content, "html.parser")
        sevenDay = soup.find(id="seven-day-forecast")

        # Finds all the periods/days listed with information on the website
        periodTags = sevenDay.select(".tombstone-container .period-name")
        periods = [pt.get_text() for pt in periodTags]

        # Finds descriptions of the weather and the temp for each period on the webiste
        shortDescs = [sd.get_text() for sd in sevenDay.select(".tombstone-container .short-desc")]
        temps = [t.get_text() for t in sevenDay.select(".tombstone-container .temp")]
        descs = [d["title"] for d in sevenDay.select(".tombstone-container img")]

        # Puts each day in the dictionary mapping to its weather and temp
        for item in periods:
            for i in range(len(periods)):
                if item == periods[i]:
                    forecast[item] = [shortDescs[i], temps[i], descs[i]]

        # Adds current temperature to the dictionary as well
        current = soup.find(id="current_conditions-summary")
        currTemp = current.find(class_="myforecast-current-lrg").get_text()
        forecast["Temp"] = currTemp

    except:

        return {}

    return forecast


"""followed this documentation to use newspaper module to get headlines
   http://newspaper.readthedocs.io/en/latest/"""

# Returns a list of news headlines
def returnNews():

    news = {}

    # Parses through the google news headlines
    page = newspaper.build("https://news.google.com/news/?ned=us&gl=US&hl=en")

    for article in page.articles:
        try:
            url = article.url
            html = requests.get(url).text
            text = fulltext(html)
            # Creates a dictionary mappying titles to full text
            if article.title != None:
                news[article.title] = text
        except:
            pass
    return news

"""followed this tutorial to use the google api to extract calendar events
   https://developers.google.com/calendar/quickstart/python"""

# Returns a list of calendar events
def returnCalendar():

    calendar = []

    # Setup the Calendar API
    SCOPES = 'https://www.googleapis.com/auth/calendar.readonly'
    store = file.Storage('credentials.json')
    creds = store.get()
    if not creds or creds.invalid:
        flow = client.flow_from_clientsecrets('client_secret.json', SCOPES)
        creds = tools.run_flow(flow, store)
    service = build('calendar', 'v3', http=creds.authorize(Http()), cache_discovery=False)

    # Call the Calendar API
    now = datetime.datetime.utcnow().isoformat() + 'Z' # 'Z' indicates UTC time

    events_result = service.events().list(calendarId='primary', timeMin=now,
                                        maxResults=5, singleEvents=True,
                                        orderBy='startTime').execute()
    events = events_result.get('items', [])

    if not events:
        events = ['No upcoming events found.']
    for event in events:
        start = event['start'].get('dateTime', event['start'].get('date'))
        #print(start, event['summary'])
        calendar.append((start, event['summary']))
    return calendar