Mew data analysis tool: clustering of arrival points.

4 files changed, 40 insertions, 13 deletions

diff --git a/config/model_0.py b/config/model_0.py
index b157faa..c4985b2 100644
--- a/config/model_0.py
+++ b/config/model_0.py
@@ -12,6 +12,6 @@ dim_input = n_begin_end_pts * 2 * 2 + dim_embed + dim_embed
 dim_hidden = [200, 100]
 dim_output = 2
 
-learning_rate = 0.002
-momentum = 0.9
+learning_rate = 0.0001
+momentum = 0.99
 batch_size = 32
diff --git a/data_analysis/cluster_arrival.py b/data_analysis/cluster_arrival.py
new file mode 100644
index 0000000..fd4ea04
--- /dev/null
+++ b/data_analysis/cluster_arrival.py
@@ -0,0 +1,27 @@
+import matplotlib.pyplot as plt
+import numpy
+import cPickle
+import scipy.misc
+
+from sklearn.cluster import MeanShift, estimate_bandwidth
+from sklearn.datasets.samples_generator import make_blobs
+from itertools import cycle
+
+print "Reading arrival point list"
+with open("arrivals.pkl") as f:
+    pts = cPickle.load(f)
+
+print "Doing clustering"
+bw = estimate_bandwidth(pts, quantile=.1, n_samples=1000)
+print bw
+bw = 0.001
+
+ms = MeanShift(bandwidth=bw, bin_seeding=True, min_bin_freq=5)
+ms.fit(pts)
+cluster_centers = ms.cluster_centers_
+
+print "Clusters shape: ", cluster_centers.shape
+
+with open("arrival-cluters.pkl", "w") as f:
+    cPickle.dump(cluster_centers, f, protocol=cPickle.HIGHEST_PROTOCOL)
+
diff --git a/data_analysis/destmaps.py b/data_analysis/destmaps.py
index 0725ca7..6d08096 100644
--- a/data_analysis/destmaps.py
+++ b/data_analysis/destmaps.py
@@ -4,7 +4,7 @@ import cPickle
 import scipy.misc
 
 print "Loading data..."
-with open("train_normal.pkl") as f: normal = cPickle.load(f)
+with open("train.pkl") as f: normal = cPickle.load(f)
 
 print "Extracting x and y"
 # xes = [c[0] for l in normal for c in l[-1]]
@@ -12,21 +12,21 @@ print "Extracting x and y"
 xes = [l[-1][-1][0] for l in normal if len(l[-1]) > 0]
 yes = [l[-1][-1][1] for l in normal if len(l[-1]) > 0]
 
-xrg = [-8.75, -8.55]
-yrg = [41.05, 41.25]
+xrg = [-8.80, -8.50]
+yrg = [41.00, 41.30]
 
-print "Doing 1d x histogram"
-plt.clf(); plt.hist(xes, bins=1000, range=xrg); plt.savefig("xhist_dest.pdf")
-print "Doing 1d y histogram"
-plt.clf(); plt.hist(yes, bins=1000, range=yrg); plt.savefig("yhist_dest.pdf")
+#print "Doing 1d x histogram"
+#plt.clf(); plt.hist(xes, bins=2000, range=xrg); plt.savefig("xhist_dest.pdf")
+#print "Doing 1d y histogram"
+#plt.clf(); plt.hist(yes, bins=2000, range=yrg); plt.savefig("yhist_dest.pdf")
 
 print "Doing 2d histogram"
-hist, xx, yy = numpy.histogram2d(xes, yes, bins=2000, range=[xrg, yrg])
+hist, xx, yy = numpy.histogram2d(xes, yes, bins=4000, range=[xrg, yrg])
 
 # import ipdb; ipdb.set_trace()
 
 print "Imshow"
-plt.clf(); plt.imshow(numpy.log(hist)); plt.savefig("xyhmap_dest.png", dpi=600)
+plt.clf(); plt.imshow(numpy.log(hist)); plt.savefig("xyhmap_dest_x.png", dpi=600)
 
 print "Imsave"
-scipy.misc.imsave("xymap_dest_2.png", numpy.log(hist + 1))
+scipy.misc.imsave("xymap_dest_2_x.png", numpy.log(hist + 1))
diff --git a/model.py b/model.py
index 753fb01..744d877 100644
--- a/model.py
+++ b/model.py
@@ -53,7 +53,7 @@ def setup_train_stream():
                         load_in_memory=True)
     train = DataStream(train, iteration_scheme=SequentialExampleScheme(data.dataset_size - config.n_valid))
     train = transformers.filter_out_trips(data.valid_trips, train)
-    train = transformers.TaxiGenerateSplits(train)
+    train = transformers.TaxiGenerateSplits(train, max_splits=100)
     train = transformers.add_first_k(config.n_begin_end_pts, train)
     train = transformers.add_last_k(config.n_begin_end_pts, train)
     train = transformers.Select(train, ('origin_stand', 'origin_call', 'first_k_latitude',