Start using new HSL API

infoscreen/hsl_fetcher.py

@@ -1,73 +1,75 @@
 """File containing Infoscreen HSL data fetcher classes."""
 
-import urllib.request
+import requests
 import json
 import logging
+import os
+import pytz
+
 from datetime import timedelta, datetime
-from django.utils import timezone
+from django.utils import timezone, dateparse
+from django.utils.dateformat import format
 
 from django.conf import settings
 
 from infoscreen.models import HSLDataModel
 
+last_fetched = timezone.now()
+INTERVAL = 1  # minutes
+# logging.info(
+#    "Set up scheduled HSL API fetch every {} minutes".format(INTERVAL))
 
-class HSLFetcher:
+with open(os.path.join(settings.BASE_DIR, 'infoscreen', 'hsl_stops.graphql')) as stops_file:
-    """Main class of Infoscreen HSL fetcher."""
+    STOPS_QUERY = stops_file.read()
 
-    last_fetched = datetime.fromtimestamp(86400)  # epoch
+with open(os.path.join(settings.BASE_DIR, 'infoscreen', 'hsl_stops_variables.json')) as vars_file:
-    INTERVAL = 1  # minutes
+    STOPS_VARS = json.loads(vars_file.read())
-    # logging.info(
-    #    "Set up scheduled HSL API fetch every {} minutes".format(INTERVAL))
 
-    def fetch_if_needed(self):
+API_URL = 'https://api.digitransit.fi/routing/v1/routers/hsl/index/graphql'
-        """Check if new fetch from HSL API is needed."""
+API_HEADERS = {'Content-Type': 'application/json'}
-        if (timezone.now() - HSLFetcher.last_fetched >
-                timedelta(minutes=HSLFetcher.INTERVAL)):
-            self.fetch()
 
-    def fetch(self):
-        """Fetch data from HSL API."""
-        location_coords = (2545565, 6675319)
-        src = urllib.request.urlopen(
-            ("https://api.reittiopas.fi/hsl/prod/?userhash={}"
-                "&request=stops_area&center_coordinate={},{}")
-            .format(settings.HSL_USERHASH, location_coords[0],
-                    location_coords[1]))\
-            .read().decode("utf-8")
 
-        data = json.loads(src)
+def fetch():
+    """Fetch data from HSL API."""
 
-        arr = []
+    query_vars = STOPS_VARS.copy()
+    query_vars['startTime_6'] = format(timezone.now(), 'U')
 
-        time = (timezone.now() +
+    post_data = json.dumps({
-                timedelta(minutes=settings.HSL_DEPARTURE_THRESHOLD))
+        'operationName': 'NearestRoutesContainer',
-        time = "{0:02d}{0:02d}".format(time.hour, time.minute)
+        'query': STOPS_QUERY,
-        for element in data:
+        'variables': query_vars,
-            src = urllib.request.urlopen(
+    })
-                ("https://api.reittiopas.fi/hsl/prod/?userhash={}"
-                    "&request=stop&code={}&dep_limit=20&time={}")
-                .format(settings.HSL_USERHASH, element['code'], time)
-            ).read().decode("utf-8")
 
-            parsed = json.loads(src)[0]
+    resp = requests.post(API_URL, data=post_data, headers=API_HEADERS)
-            arr.append({
-                "name": parsed['name_fi'],
-                "lines": parsed['lines'],
-                "dist": element['dist'],
-                "departures": parsed['departures']})
 
-        model_arr = HSLDataModel.objects.all()
+    data = resp.json()
-        count = len(model_arr)
-        json_dump = json.dumps(arr)
 
-        if count == 0:
+    items = data['data']['viewer']['_nearest']['edges']
-            HSLDataModel.objects.create(data=json_dump)
+    places = map(lambda item: item['node']['place'], items)
-        else:
-            obj = model_arr[count - 1]
-            obj.data = json_dump
-            obj.save()
-        now = timezone.now()
-        HSLFetcher.last_fetched = now
 
-        logging.info(
+    schedule = []
-            "Fetched HSL timetable data with size {} bytes.".format(len(src)))
+    for place in places:
+        route = place['pattern']['route']['shortName']
+        stop_times = place['_stoptimes']
+        for stop_time in stop_times:
+            timestamp = stop_time['serviceDay'] + stop_time['realtimeArrival']
+            headsign = stop_time['stopHeadsign']
+            stop_name = stop_time['stop']['name']
+            
+            time_diff = (timestamp - timezone.now().timestamp()) / 60  # minutes
+            if time_diff < settings.HSL_DEPARTURE_THRESHOLD:
+                continue
+            elif time_diff < settings.HSL_HURRY_THRESHOLD:
+                time = '{} min'.format(int(time_diff))
+            else:
+                time = datetime.utcfromtimestamp(timestamp).strftime('%H:%M')
+
+            schedule.append({
+                'route': route,
+                'headsign': headsign,
+                'timestamp': time,
+                'stop': stop_name,
+            })
+
+    return schedule

infoscreen/hsl_stops.graphql

@@ -0,0 +1,71 @@
+query NearestRoutesContainer($lat_0: Float!, $lon_1: Float!, $maxDistance_2: Int!, $maxResults_3: Int!, $timeRange_7: Int!, $numberOfDepartures_8: Int!, $filterByModes_4: [Mode]!, $filterByPlaceTypes_5: [FilterPlaceType]!, $startTime_6: Long!) {
+  viewer {
+    ...F5
+  }
+}
+
+fragment F0 on DepartureRow {
+  _stoptimes4caEfh: stoptimes(startTime: $startTime_6, timeRange: $timeRange_7, numberOfDepartures: $numberOfDepartures_8) {
+    pickupType
+    serviceDay
+    realtimeDeparture
+  }
+  id
+}
+
+fragment F1 on DepartureRow {
+  pattern {
+    route {
+      shortName
+    }
+  }
+  _stoptimes: stoptimes(startTime: $startTime_6, timeRange: $timeRange_7, numberOfDepartures: $numberOfDepartures_8) {
+    realtimeArrival
+    serviceDay
+    stopHeadsign
+    stop {
+      name
+    }
+  }
+}
+
+fragment F2 on BikeRentalStation {
+    id
+}
+
+fragment F3 on placeAtDistance {
+  distance
+  place {
+    id
+    __typename
+    ...F1
+    ...F2
+  }
+  id
+}
+
+fragment F4 on placeAtDistanceConnection {
+  edges {
+    node {
+      distance
+      place {
+        id
+        __typename
+        ...F0
+      }
+      id
+      ...F3
+    }
+    cursor
+  }
+  pageInfo {
+    hasNextPage
+    hasPreviousPage
+  }
+}
+
+fragment F5 on QueryType {
+  _nearest: nearest(lat: $lat_0, lon: $lon_1, maxDistance: $maxDistance_2, maxResults: $maxResults_3, first: $maxResults_3, filterByModes: $filterByModes_4, filterByPlaceTypes: $filterByPlaceTypes_5) {
+    ...F4
+  }
+}

infoscreen/hsl_stops_variables.json

@@ -0,0 +1,10 @@
+{
+    "lat_0": 60.190480099999995,
+    "lon_1": 24.8275665,
+    "maxDistance_2": 1000,
+    "maxResults_3": 50,
+    "numberOfDepartures_8": 2,
+    "timeRange_7": 7200,
+    "filterByModes_4": ["BUS"],
+    "filterByPlaceTypes_5": ["DEPARTURE_ROW"]
+}

infoscreen/static/css/hsl.css

@@ -1,5 +1,5 @@
 table {
-    font-size: 5vh;
+    font-size: 4vh;
     font-family: 'Droid Sans Mono', monospace;
 }
 .red {
@@ -58,9 +58,9 @@ thead{
     margin-left: 0;
     margin-right: 0;
 }
+
 .repeat-item.ng-leave {
-    -webkit-transition:0.5s linear all;
+
-    transition:0.5s linear all;
 }
 
 .repeat-item.ng-leave.ng-leave-active {

infoscreen/static/html/hsl.html

@@ -1,12 +1,16 @@
 <link rel="stylesheet" href="/static/css/hsl.css">
 <link href="https://fonts.googleapis.com/css?family=Droid+Sans+Mono" rel="stylesheet"> 
+<script src="https://cdnjs.cloudflare.com/ajax/libs/moment.js/2.19.1/locale/fi.js"></script>
 <div class="container" ng-app="myApp" ng-controller="timetableCtrl">
     <div class="header-row row">
         <div class="col-sm-2"></div>
         <div class="col-sm-8">HSL-Aikataulut</div>
-        <div class="col-sm-2 time"><p>{{ clock  | date:'HH:mm'}}</p></div>
+        <div class="col-sm-2 time"><p>{{clock | date:'HH:mm'}}</p></div>
     </div>
-    <table class="table table-striped row">
+    <h1 style="font-size: 10vh; text-align: center" ng-if="error">
+        {{error}}
+    </h1>
+    <table ng-if="!error" class="table table-striped row">
         <thead>
             <tr>    
                 <th>
@@ -18,25 +22,19 @@
                 <th>
                     Pys&#228;kki
                 </th>
-                <th>
-                    P&#228;&#228;tepys&#228;kki
-                </th>
             </tr>
         </thead>
         <tbody>
-            <tr class="repeat-item" ng-repeat="x in arr | orderBy: ['date','time'] | limitTo: 10">
+            <tr class="repeat-item" ng-repeat="x in stoptimes | orderBy: ['timestamp'] | limitTo: 11">
-                <td ng-class='{red : x.hurry, black: !x.hurry}'>
+                <td style="min-width: 300px">
-                    {{x.timedelta < 10 ?x.timedelta + ' min' : x.time}}
+                    {{x.timestamp}}
                 </td>
                 <td>
-                    {{x.bus}}
+                    <strong>{{x.route}}</strong>, {{x.headsign}}
                 </td>
                 <td>
                     {{x.stop}}
                 </td>
-                <td>
-                    {{x.laststop}}
-                </td>
             </tr>
         </tbody>
     </table>

infoscreen/static/js/infoscreen_controllers.js

@@ -1,4 +1,5 @@
 var app = angular.module('infoApp', ['ngAnimate', 'ngRoute']);
+
 app.controller('infoscreen_main', function($scope,$http,$timeout){
     var templates = [];
     $scope.init = function(rot){
@@ -103,13 +104,22 @@ app.controller('EventController', function($scope, $http) {
         })
 });
 
+app.filter('unixTimeToDifference', function() {
+    return function(input) {
+        var date = moment.unix(input);
+        var now = moment();
+        var res = date.diff(now, 'minutes');
+        return res;
+    }
+})
+
 app.controller('timetableCtrl',
     function($scope, $http, $interval) {
-        function load(){
+        function load() {
             $http.get('/infoscreen/hsl_data')
                 .then(function(data, status, headers, config) { //eslint-disable-line no-unused-vars
-                    $scope.arr=[];
+                    $scope.stoptimes = data.data;
-                    parse(data);
+                    $scope.error = data.data.error || null;
                 });
             $http.get('/infoscreen/hsl_data/settings')
                 .then(function(data, status, headers, config) { //eslint-disable-line no-unused-vars
@@ -117,112 +127,20 @@ app.controller('timetableCtrl',
                     $scope.hurryThreshold = data.data['hurry_threshold'];
                 });
         }
-        $scope.$on('$destroy', function() {
-            $interval.cancel(inter1);
-            $interval.cancel(inter2);
-            $interval.cancel(inter3);
-        });
-        var objects;
-        $scope.arr=[];
-        var dict=[];
-        function parse(data){
-            objects=data['data'];
-            for(var objectIndex in objects){
-                var stop = objects[objectIndex];
 
-                var lineIndex;
+        function update_clock() {
-                for (lineIndex in stop['lines']){
-                    var elem=stop['lines'][lineIndex].split(":");
-                    dict[elem[0]]=elem[1];
-                }
-                for (lineIndex in stop['departures']){
-                    var line = stop['departures'][lineIndex];
-                    var time = line['time'];
-                    var date = line['date'];
-                    var hours = Math.floor(time / 100);
-                    var minutes = time % 100;
-                    if (hours >= 24) {
-                        hours -= 24;
-                        date++;
-                    }
-                    var code = line['code'].substring(1, 5);
-                    if (code.charAt(0) == '0') {
-                        code = code.substring(1,4);
-                    }
-
-                    var departure = {
-                        "stop":       stop['name'].split(",")[0],
-                        "dist":       stop['dist'],
-                        "bus":        code,
-                        "date":       date,
-                        "time":       pad(hours, 2) + ":" + pad(minutes, 2),
-                        "laststop":   dict[line['code']].split(",")[0].split(" l.")[0],
-                        "hurry":      false
-                    };
-                    if(departure['laststop']=='Otaniemi')
-                        break;
-                    if(departure['stop']=='Alvar Aallon puisto')
-                        departure['stop']="A. A. puisto"
-                    var trigger = true;
-                    for (var arrIndex = $scope.arr.length - 1; arrIndex >= 0; arrIndex--) {
-                        if ($scope.arr[arrIndex]['bus'] == departure['bus'] &&
-                            $scope.arr[arrIndex]['laststop'] == departure['laststop']) {
-
-                            if ($scope.arr[arrIndex]['dist'] == departure['dist']){
-                                break;
-                            }
-                            else if ($scope.arr[arrIndex]['dist'] > departure['dist']){
-                                $scope.arr.splice(arrIndex, 1);
-                            }
-                            else {
-                                trigger = false;
-                            }
-                        }
-                    }
-                    if (trigger) {
-                            $scope.arr.push(departure);
-                    }
-                }
-            }
-
-            function pad(num, size) {
-                var s = num + "";
-                while (s.length < size) {
-                    s = "0" + s;
-                }
-                return s;
-            }
-
-            delOld();
-        }
-        function delOld(){
-            var tooSoon = typeof($scope.departureThreshold) != 'undefined' ? $scope.departureThreshold: 0;
-            var hurry = typeof($scope.hurryThreshold) != 'undefined' ? $scope.hurryThreshold : 0;
-            var f = new Date();
-            for (var a=$scope.arr.length-1; a>=0; a--) {
-                var time=$scope.arr[a]['time'].split(":");
-                var date=$scope.arr[a]['date'].toString();
-                var d = new Date(f);
-                d.setFullYear(date.substring(0,4), date.substring(4,6)-1, date.substring(6,8));
-                d.setHours(time[0]);
-                d.setMinutes(time[1]);
-                var diff=(d.getTime()-f.getTime());
-                $scope.arr[a]['timedelta']=Math.floor(diff/60000);
-                if(diff < tooSoon*60000) {
-                    $scope.arr.splice(a,1);
-                }
-                else if (diff < hurry*60000) {
-                    $scope.arr[a]['hurry']=true;
-                }
-            }
-        }
-        function updateTime(){
             $scope.clock = Date.now();
         }
-        $scope.clock = Date.now();
+
+        $scope.$on('$destroy', function() {
+            $interval.cancel(load_interval);
+            $interval.cancel(clock_interval);            
+        });
+
+        var load_interval = $interval(load, 5000);
+        var clock_interval = $interval(update_clock, 1000);
+
+        update_clock();
         load();
-        var inter1=$interval(delOld,2000);
-        var inter2=$interval(load,10000);
-        var inter3=$interval(updateTime, 1000);
     }
 );

infoscreen/views/public_views.py

@@ -1,12 +1,14 @@
 from django.shortcuts import render
 from django.http import HttpResponse, JsonResponse, HttpResponseBadRequest
 from django.views.decorators.http import require_http_methods
+from django.conf import settings
 
-from infoscreen.models import Rotation, InfoItem, InfoInstance
+from infoscreen.models import Rotation, InfoItem, InfoInstance, HSLDataModel
-from infoscreen.hsl_fetcher import HSLFetcher
+from infoscreen.hsl_fetcher import fetch as hsl_fetch
 
 import json
 import logging
+import threading
 
 
 @require_http_methods(["GET"])
@@ -82,22 +84,18 @@ def hsl_timetable_settings(request, *args, **kwargs):
     """Set HSL timetable settings."""
     d = {"departure_threshold": settings.HSL_DEPARTURE_THRESHOLD,
          "hurry_threshold": settings.HSL_HURRY_THRESHOLD}
-    resp = json.dumps(d)
+
-    return HttpResponse(resp, status=200)
+    return JsonResponse(d, status=200)
 
 
 @require_http_methods(["GET"])
 def CurrentHSLView(request, *args, **kwargs):
     """Get HSL data and return it."""
-    fetcher = HSLFetcher()
+    try:
-    fetcherThread = threading.Thread(target=fetcher.fetch_if_needed, args=[])
+        api_resp = hsl_fetch()
-    fetcherThread.setDaemon(False)
+    except Exception as ex:
-    fetcherThread.start()
+        logging.exception('Failed to fetch HSL timetables.')
+        error = {'error': 'Aikataulujen haku epäonnistui.'}
+        return JsonResponse(error, status=200)
 
-    data = HSLDataModel.objects.all()
+    return JsonResponse(api_resp, status=200, safe=False)
-    if len(data) < 1:
-        return HttpResponse(
-            '{"error" : "Could not find timetables from database."}',
-            status=500)
-
-    return HttpResponse(data[len(data) - 1].data, status=200)