From 1eca8867751df644a62752fbbfbc6a6de849de74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89tienne=20Simon?= Date: Mon, 11 May 2015 20:00:04 -0400 Subject: Add visualizer. Lasciate ogni speranza voi ch'entrate: I am the bone of my javascript DOM is my body and JQuery is my blood I have created over a thousand lines Unknown to death Nor known to life Have withstood pain to create many functions Yet those hands shall never type anything So, as I pray, Unlimited Openlayers Works --- visualizer/HTTPServer.py | 128 ++++ visualizer/__init__.py | 130 ++++ visualizer/extract_all.sh | 1 + visualizer/extractor/destinations.py | 19 + visualizer/extractor/stands.py | 14 + visualizer/extractor/test_positions.py | 12 + visualizer/extractor/train_poi.py | 21 + visualizer/index.html | 71 +++ visualizer/script.js | 1037 ++++++++++++++++++++++++++++++++ visualizer/style.css | 124 ++++ 10 files changed, 1557 insertions(+) create mode 100755 visualizer/HTTPServer.py create mode 100644 visualizer/__init__.py create mode 100755 visualizer/extract_all.sh create mode 100755 visualizer/extractor/destinations.py create mode 100755 visualizer/extractor/stands.py create mode 100755 visualizer/extractor/test_positions.py create mode 100755 visualizer/extractor/train_poi.py create mode 100644 visualizer/index.html create mode 100644 visualizer/script.js create mode 100644 visualizer/style.css diff --git a/visualizer/HTTPServer.py b/visualizer/HTTPServer.py new file mode 100755 index 0000000..e71bef3 --- /dev/null +++ b/visualizer/HTTPServer.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python + +import os +import sys +import urllib +import SimpleHTTPServer +import SocketServer +from cStringIO import StringIO + +import h5py + +import data +from data.hdf5 import TaxiDataset +from visualizer import Vlist, Path + + +visualizer_path = os.path.join(data.path, 'visualizer') +source_path = os.path.split(os.path.realpath(__file__))[0] + +test_data = None +train_data = None + +class VisualizerHTTPRequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler): + def send_head(self): + spath = self.path.split('?')[0] + path = spath.split('/')[1:] + if len(path) == 1: + if path[0] == '': + path[0] = 'index.html' + file_path = os.path.join(source_path, path[0]) + return self.send_file(file_path) + elif path[0] == 'ls': + return self.send_datalist() + elif path[0] == 'get': + return self.send_file(os.path.join(visualizer_path, spath[5:])) + elif path[0] == 'extract': + return self.send_extract(spath[9:]) + + def send_file(self, file_path): + file_path = urllib.unquote(file_path) + ctype = self.guess_type(file_path) + + try: + f = open(file_path, 'rb') + except IOError: + self.send_error(404, 'File not found') + return None + try: + self.send_response(200) + self.send_header('Content-type', ctype) + fs = os.fstat(f.fileno()) + self.send_header('Content-Length', str(fs[6])) + self.send_header('Last-Modified', self.date_time_string(fs.st_mtime)) + self.end_headers() + return f + except: + f.close() + raise + + def send_datalist(self): + l = [] + for path, subs, files in os.walk(visualizer_path): + for file in files: + mtime = os.stat('%s/%s' % (path, file))[8] + l.append('{"path":["%s"],"name":"%s","mtime":%d}' % ('","'.join(path[len(visualizer_path):].split('/')), file, mtime)) + l.sort() + f = StringIO() + f.write("[") + f.write(','.join(l)) + f.write("]") + length = f.tell() + f.seek(0) + self.send_response(200) + encoding = sys.getfilesystemencoding() + self.send_header("Content-type", "text/html; charset=%s" % encoding) + self.send_header("Content-Length", str(length)) + self.end_headers() + return f + + def send_extract(self, query): + f = StringIO() + query = urllib.unquote(query) + content = Vlist() + for (i,sub) in enumerate(query.split(',')): + r = sub.split('-') + if len(r)==1: + if sub.strip()[0].lower()=='t': + sub=sub.strip()[1:] + content.append(Path(test_data.extract(int(sub)), 'T%s
'%sub)) + else: + content.append(Path(train_data.extract(int(sub)), '%s
'%sub)) + elif len(r)==2: + test = False + if r[0].strip()[0].lower()=='t': + test = True + r[0]=r[0].strip()[1:] + if r[1].strip()[0].lower()=='t': + r[1]=r[1].strip()[1:] + for i in xrange(int(r[0]), int(r[1])+1): + if test: + content.append(Path(test_data.extract(i), 'T%d
'%i)) + else: + content.append(Path(train_data.extract(i), '%d
'%i)) + elif len(r)>2: + self.send_error(404, 'File not found') + return None + content.write(f) + length = f.tell() + f.seek(0) + self.send_response(200) + encoding = sys.getfilesystemencoding() + self.send_header("Content-type", "text/html; charset=%s" % encoding) + self.send_header("Content-Length", str(length)) + self.end_headers() + return f + +if __name__ == '__main__': + if len(sys.argv) != 2: + print >>sys.stderr, 'Usage: %s port' % sys.argv[0] + + print >>sys.stderr, 'Loading dataset...', + path = os.path.join(data.path, 'data.hdf5') + train_data = TaxiDataset('train') + test_data = TaxiDataset('test') + print >>sys.stderr, 'done' + + httpd = SocketServer.TCPServer(('', int(sys.argv[1])), VisualizerHTTPRequestHandler) + httpd.serve_forever() diff --git a/visualizer/__init__.py b/visualizer/__init__.py new file mode 100644 index 0000000..e1cdf73 --- /dev/null +++ b/visualizer/__init__.py @@ -0,0 +1,130 @@ +import os +import json +import getpass +from datetime import datetime +import itertools + +import numpy + +import data + + +class NumpyEncoder(json.JSONEncoder): + def default(self, o): + if type(o).__module__ == numpy.__name__: + return o.item() + super(NumpyEncoder, self).default(o) + + +class EGJ(object): + def save(self, path=getpass.getuser(), append=False): + path = os.path.join(data.path, 'visualizer', path) + if append: + if not os.path.isdir(path): + raise ValueError("Can't append to the given directory") + name = str(1+max(map(int, filter(str.isdigit, os.listdir(path)))+[-1])) + path = os.path.join(path, name) + else: + while os.path.isdir(path): + path = os.path.join(path, '0') + + with open(path, 'w') as f: + self.write(f) + + def write(self, file): + file.write(json.dumps(self.object(), cls=NumpyEncoder)) + + def type(self): + return 'raw' + + def options(self): + return [] + + def object(self): + return { + 'type': self.type(), + 'data': { + 'type': 'FeatureCollection', + 'crs': { + 'type': 'name', + 'properties': { + 'name': 'urn:ogc:def:crs:OGC:1.3:CRS84' + } + }, + 'features': self.features() + } + } + + +class Point(EGJ): + def __init__(self, latitude, longitude, info=None): + self.latitude = latitude + self.longitude = longitude + self.info = info + + def features(self): + d = { + 'type': 'Feature', + 'geometry': { + 'type': 'Point', + 'coordinates': [self.longitude, self.latitude] + } + } + if self.info is not None: + d['properties'] = { 'info': self.info } + return [d] + + +class Path(EGJ): + def __init__(self, path, info=''): + self.path = path + self.info = info + + def features(self): + info = self.info + '''trip_id: %(trip_id)s
+ call_type: %(call_type_f)s
+ origin_call: %(origin_call)d
+ origin_stand: %(origin_stand)d
+ taxi_id: %(taxi_id)d
+ timestamp: %(timestamp_f)s
+ day_type: %(day_type_f)s
+ missing_data: %(missing_data)d
''' \ + % dict(self.path, + call_type_f = ['central', 'stand', 'street'][self.path['call_type']], + timestamp_f = datetime.fromtimestamp(self.path['timestamp']).strftime('%c'), + day_type_f = ['normal', 'holiday', 'holiday eve'][self.path['day_type']]) + + return [{ + 'type': 'Feature', + 'properties': { + 'info': info, + 'display': 'path', + 'timestamp': self.path['timestamp'] + }, + 'geometry': { + 'type': 'LineString', + 'coordinates': [[lon, lat] for (lat, lon) in zip(self.path['latitude'], self.path['longitude'])] + } + }] + + +class Vlist(EGJ, list): + def __init__(self, cluster=False, heatmap=False, *args): + list.__init__(self, *args) + self.cluster = cluster + self.heatmap = heatmap + + def type(self): + if self.cluster or self.heatmap: + if all(isinstance(c, Point) for c in self): + if self.cluster: + return 'cluster' + elif self.heatmap: + return 'heatmap' + else: + raise ValueError('Building a %s with something that is not a Point' % ('cluster' if self.cluster else 'heatmap')) + else: + return 'raw' + + def features(self): + return list(itertools.chain.from_iterable(p.features() for p in self)) diff --git a/visualizer/extract_all.sh b/visualizer/extract_all.sh new file mode 100755 index 0000000..67d92aa --- /dev/null +++ b/visualizer/extract_all.sh @@ -0,0 +1 @@ +find extractor -type f -print -exec {} \; diff --git a/visualizer/extractor/destinations.py b/visualizer/extractor/destinations.py new file mode 100755 index 0000000..967e766 --- /dev/null +++ b/visualizer/extractor/destinations.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python + +from data.hdf5 import taxi_it +from visualizer import Vlist, Point + + +_sample_size = 5000 + +if __name__ == '__main__': + points = Vlist(cluster=True) + for line in taxi_it('train'): + if len(line['latitude'])>0: + points.append(Point(line['latitude'][-1], line['longitude'][-1])) + if len(points) >= _sample_size: + break + points.save('destinations (cluster)') + points.cluster = False + points.heatmap = True + points.save('destinations (heatmap)') diff --git a/visualizer/extractor/stands.py b/visualizer/extractor/stands.py new file mode 100755 index 0000000..9224143 --- /dev/null +++ b/visualizer/extractor/stands.py @@ -0,0 +1,14 @@ +#!/usr/bin/env python + +from data.hdf5 import taxi_it +from visualizer import Vlist, Point + + +if __name__ == '__main__': + it = taxi_it('stands') + next(it) # Ignore the "no stand" entry + + points = Vlist() + for (i, line) in enumerate(it): + points.append(Point(line['stands_latitude'], line['stands_longitude'], 'Stand (%d): %s' % (i+1, line['stands_name']))) + points.save('stands') diff --git a/visualizer/extractor/test_positions.py b/visualizer/extractor/test_positions.py new file mode 100755 index 0000000..a84d2ba --- /dev/null +++ b/visualizer/extractor/test_positions.py @@ -0,0 +1,12 @@ +#!/usr/bin/env python + +from data.hdf5 import taxi_it +from visualizer import Vlist, Point + + +if __name__ == '__main__': + points = Vlist(heatmap=True) + for line in taxi_it('test'): + for (lat, lon) in zip(line['latitude'], line['longitude']): + points.append(Point(lat, lon)) + points.save('test positions') diff --git a/visualizer/extractor/train_poi.py b/visualizer/extractor/train_poi.py new file mode 100755 index 0000000..a4ccbca --- /dev/null +++ b/visualizer/extractor/train_poi.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python + +import os + +import data +from data.hdf5 import TaxiDataset +from visualizer import Path + + +poi = { + 'longest': 1492417 +} + +if __name__ == '__main__': + prefix = os.path.join(data.path, 'visualizer', 'Train POI') + if not os.path.isdir(prefix): + os.mkdir(prefix) + + d = TaxiDataset('train') + for (k, v) in poi.items(): + Path(d.extract(v)).save(os.path.join('Train POI', k)) diff --git a/visualizer/index.html b/visualizer/index.html new file mode 100644 index 0000000..0094af6 --- /dev/null +++ b/visualizer/index.html @@ -0,0 +1,71 @@ + + + Taxi Visualizer + + + + + + + + + + + +
+
+
+
+
+
+
+
+ +
+
+
+
    +
    +
    + + + +
    +
    + + diff --git a/visualizer/script.js b/visualizer/script.js new file mode 100644 index 0000000..98c9593 --- /dev/null +++ b/visualizer/script.js @@ -0,0 +1,1037 @@ +/***************/ +/*** General ***/ +/***************/ + +window.app = {}; +var app = window.app; + +app.mainLayer = new ol.layer.Tile({ source: new ol.source.OSM() }); + + +/****************/ +/*** Geometry ***/ +/****************/ + +app.geometry = {} +app.geometry.REarth = 6371000; +app.geometry.toRadians = function(x){ return x * Math.PI / 180; }; + +app.geometry.haversine = function(lat1, lon1, lat2, lon2){ + var lat1 = app.geometry.toRadians(lat1); + var lon1 = app.geometry.toRadians(lon1); + var lat2 = app.geometry.toRadians(lat2); + var lon2 = app.geometry.toRadians(lon2); + + var dlat = Math.abs(lat1-lat2); + var dlon = Math.abs(lon1-lon2); + + var alpha = Math.pow(Math.sin(dlat/2), 2) + Math.cos(lat1) * Math.cos(lat2) * Math.pow(Math.sin(dlon/2), 2); + var d = Math.atan2(Math.sqrt(alpha), Math.sqrt(1-alpha)); + + return 2 * app.geometry.REarth * d; +}; + +app.geometry.equirectangular = function(lat1, lon1, lat2, lon2){ + var lat1 = app.geometry.toRadians(lat1); + var lon1 = app.geometry.toRadians(lon1); + var lat2 = app.geometry.toRadians(lat2); + var lon2 = app.geometry.toRadians(lon2); + var x = (lon2-lon1) * Math.cos((lat1+lat2)/2); + var y = (lat2-lat1); + return Math.sqrt(x*x + y*y) * app.geometry.REarth; +}; + + +/***************/ +/*** Measure ***/ +/***************/ + +app.measure = {}; +app.measure.tooltip_list = []; + +app.measure.source = new ol.source.Vector(); + +app.measure.layer = new ol.layer.Vector({ + source: app.measure.source, + style: new ol.style.Style({ + fill: new ol.style.Fill({ + color: 'rgba(255, 255, 255, 0.2)' + }), + stroke: new ol.style.Stroke({ + color: '#FC3', + width: 2 + }), + image: new ol.style.Circle({ + radius: 7, + fill: new ol.style.Fill({ + color: '#FC3' + }) + }) + }) +}); + +app.measure.pointerMoveHandler = function(evt){ + if(evt.dragging){ return; } + var tooltipCoord = evt.coordinate; + + if(app.measure.sketch){ + var output; + var geom = (app.measure.sketch.getGeometry()); + if(geom instanceof ol.geom.LineString){ + output = app.measure.formatLength((geom)); + tooltipCoord = geom.getLastCoordinate(); + } + app.measure.tooltipElement.innerHTML = output; + app.measure.tooltip.setPosition(tooltipCoord); + } +}; + +app.measure.addInteraction = function(){ + app.measure.draw = new ol.interaction.Draw({ + source: app.measure.source, + type: ('LineString'), + style: new ol.style.Style({ + fill: new ol.style.Fill({ + color: 'rgba(255, 255, 255, 0.2)' + }), + stroke: new ol.style.Stroke({ + color: 'rgba(0, 0, 0, 0.5)', + lineDash: [10, 10], + width: 2 + }), + image: new ol.style.Circle({ + radius: 5, + stroke: new ol.style.Stroke({ + color: 'rgba(0, 0, 0, 0.7)' + }), + fill: new ol.style.Fill({ + color: 'rgba(255, 255, 255, 0.2)' + }) + }) + }) + }); + app.map.addInteraction(app.measure.draw); + + app.measure.createTooltip(); + + app.measure.draw.on('drawstart', + function(evt){ + app.measure.sketch = evt.feature; + }, this); + + app.measure.draw.on('drawend', + function(evt){ + app.measure.tooltipElement.className = 'measure-tooltip measure-tooltip-static'; + app.measure.tooltip.setOffset([0, -7]); + app.measure.sketch = null; + app.measure.tooltipElement = null; + app.measure.createTooltip(); + }, this); +}; + +app.measure.createTooltip = function(){ + if(app.measure.tooltipElement){ + app.measure.tooltipElement.parentNode.removeChild(app.measure.tooltipElement); + } + app.measure.tooltipElement = document.createElement('div'); + app.measure.tooltipElement.className = 'measure-tooltip measure-tooltip-value'; + app.measure.tooltip = new ol.Overlay({ + element: app.measure.tooltipElement, + offset: [0, -15], + positioning: 'bottom-center' + }); + app.measure.tooltip_list.push(app.measure.tooltip); + app.map.addOverlay(app.measure.tooltip); +}; + +app.measure.formatLength = function(line){ + var length_euclidean = line.getLength(); + var length_equirectangular = 0; + var length_haversine = 0; + var coordinates = line.getCoordinates(); + var sourceProj = app.map.getView().getProjection(); + for(var i = 0, ii = coordinates.length - 1; i < ii; ++i){ + var c1 = ol.proj.transform(coordinates[i], sourceProj, 'EPSG:4326'); + var c2 = ol.proj.transform(coordinates[i + 1], sourceProj, 'EPSG:4326'); + length_equirectangular += app.geometry.equirectangular(c1[1], c1[0], c2[1], c2[0]); + length_haversine += app.geometry.haversine(c1[1], c1[0], c2[1], c2[0]); + } + + var disp = function(x){ + if(x > 100){ + return Math.round(x / 1000 * 1000) / 1000 + 'km'; + } else { + return Math.round(x * 1000) / 1000 + 'm'; + } + } + + var length_euclidean = disp(length_euclidean); + var length_equirectangular = disp(length_equirectangular); + var length_haversine = disp(length_haversine); + + var display_euclidean = $('input#measure-euclidean').prop('checked'); + var display_equirectangular = $('input#measure-equirectangular').prop('checked'); + var display_haversine = $('input#measure-haversine').prop('checked'); + + var header = true; + if(display_euclidean + display_equirectangular + display_haversine == 1){ + header = false; + } + + var str = ''; + if(display_euclidean){ + if(header){ str += 'euclidean: '; } + str += length_euclidean; + } + if(display_equirectangular){ + if(header){ if(display_euclidean){ str += '
    '; } str += 'equirectangular: '; } + str += length_equirectangular; + } + if(display_haversine){ + if(header){ str += '
    haversine: '; } + str += length_haversine; + } + return str; +}; + + +/*******************/ +/*** DataDisplay ***/ +/*******************/ + +app.dataDisplay = {}; +app.dataDisplay.layers = {}; +app.dataDisplay.heatmapRadius = 5; +app.dataDisplay.heatmapBlur = 5; +app.dataDisplay.pathPointMode = 1; // endpoints +app.dataDisplay.pathPointResolution = 50; + +app.dataDisplay.loadLayer = function(path){ + $.ajax({url: path, cache: false, dataType: 'json', + success: function(result){ + app.dataDisplay.layers[path] = app.dataDisplay.preprocess(result); + app.map.addLayer(app.dataDisplay.layers[path]); + } + }); +}; + +app.dataDisplay.unloadLayer = function(path){ + app.map.removeLayer(app.dataDisplay.layers[path]); + delete app.dataDisplay.layers[path]; +}; + +app.dataDisplay.rawStyle = function(feature, resolution){ + var style = [ new ol.style.Style({ + stroke: new ol.style.Stroke({ + color: '#00F', + width: 5 + }), + image: new ol.style.Circle({ + radius: 5, + fill: new ol.style.Fill({ + color: '#00F' + }) + }) + }), + new ol.style.Style({ + stroke: new ol.style.Stroke({ + color: '#000', + width: 2 + }), + image: new ol.style.Circle({ + radius: 2, + fill: new ol.style.Fill({ + color: '#FFF' + }) + }) + }) + ]; + + if(feature.get('display') == 'path' && resolution < app.dataDisplay.pathPointResolution){ + if(app.dataDisplay.pathPointMode == 2){ + var polyline = feature.getGeometry(); + var points = polyline.getCoordinates(); + for(var i=1; i= 1){ + var polyline = feature.getGeometry(); + var first = polyline.getFirstCoordinate(); + var last = polyline.getLastCoordinate(); + style.push(new ol.style.Style({ + geometry: new ol.geom.Point(first), + image: new ol.style.Circle({ + radius: 5, + fill: new ol.style.Fill({ + color: '#0F0' + }) + }) + })); + style.push(new ol.style.Style({ + geometry: new ol.geom.Point(last), + image: new ol.style.Circle({ + radius: 5, + fill: new ol.style.Fill({ + color: '#F00' + }) + }) + })); + } + } + + return style; +}; + +app.dataDisplay.clusterStyleCache = {}; +app.dataDisplay.clusterStyle = function(feature, resolution){ + var size = feature.get('features').length; + var style = app.dataDisplay.clusterStyleCache[size]; + if(!style){ + style = [new ol.style.Style({ + image: new ol.style.Circle({ + radius: 10, + stroke: new ol.style.Stroke({ + color: '#FFF' + }), + fill: new ol.style.Fill({ + color: '#39C' + }) + }), + text: new ol.style.Text({ + text: size.toString(), + fill: new ol.style.Fill({ + color: '#FFF' + }) + }) + })]; + app.dataDisplay.clusterStyleCache[size] = style; + } + return style; +}; + +app.dataDisplay.preprocess = function(egj){ + var source = new ol.source.GeoJSON({ + projection: 'EPSG:3857', + object: egj.data + }); + + if(egj.type == 'raw'){ + return new ol.layer.Vector({ + source: source, + style: app.dataDisplay.rawStyle + }); + + } else if(egj.type == 'cluster'){ + return new ol.layer.Vector({ + source: new ol.source.Cluster({ + distance: 40, + source: source + }), + style: app.dataDisplay.clusterStyle + }); + + } else if(egj.type == 'heatmap'){ + return new ol.layer.Heatmap({ + source: source, + blur: app.dataDisplay.heatmapBlur, + radius: app.dataDisplay.heatmapRadius + }); + } +}; + +app.dataDisplay.reloadPathes = function(){ + for(var layer in app.dataDisplay.layers){ + if(app.dataDisplay.layers[layer].getSource().getFeatures()[0].get('display') == 'path'){ + app.dataDisplay.layers[layer].changed(); + } + } +}; + +app.dataDisplay.reloadHeatmaps = function(){ + for(var key in app.dataDisplay.layers){ + var layer = app.dataDisplay.layers[key]; + if(layer instanceof ol.layer.Heatmap){ + layer.setBlur(app.dataDisplay.heatmapBlur); + layer.setRadius(app.dataDisplay.heatmapRadius); + } + } +}; + + +/****************/ +/*** DataList ***/ +/****************/ + +app.dataList = {}; +app.dataList.current = {}; +app.dataList.idgen = 0; + +app.dataList.init = function(){ + app.dataList.elementTree = {}; + app.dataList.elementTree.parent = null; + app.dataList.elementTree.children = {}; + app.dataList.elementTree.checkbox = null; + app.dataList.elementTree.ul = $('#datalist-tree ul'); + + app.dataList.updateList(); + setInterval(app.dataList.updateList, 1000); +}; + +app.dataList.updateList = function(){ + $.ajax({url: '/ls/', cache: false, dataType: 'json', + success: function(result){ + result.forEach(function(file){ + file.uri = file.path.join('/') + '/' + file.name + if(file.uri in app.dataList.current){ + if(file.mtime > app.dataList.current[file.uri].mtime){ + var act = app.dataList.current[file.uri]; + if(act.checkbox.prop('checked')){ + app.dataList.unloadLayer(file.uri); + app.dataList.loadLayer(file.uri); + } + act.mtime = file.mtime; + } + } else { + app.dataList.insert(file); + } + }); + } + }); +}; + +app.dataList.insert = function(file){ + var cur = app.dataList.elementTree; + var prev = null; + for(var i = 1; i') + .prop('id', 'folder-'+app.dataList.idgen) + .hide(); + + var hidelink = $('') + .prop('href', '') + .append('hide') + .hide(); + var showlink = $('') + .prop('href', '') + .append('show'); + + var playlink = $('') + .prop('href', '') + .append('play'); + var stoplink = $('') + .prop('href', '') + .append('stop') + .hide(); + + n.checkbox = $('') + .prop('type', 'checkbox') + .prop('id', 'data-'+app.dataList.idgen) + .prop('name', n.uri); + n.checkbox.change(app.dataList.selectData); + var item = $('
  • ') + .append(n.checkbox) + .append($('
  • ') + .append(file.checkbox) + .append($('