diff options
| -rw-r--r-- | data.py | 14 | ||||
| -rw-r--r-- | make_valid.py | 7 | ||||
| -rw-r--r-- | model.py | 5 | ||||
| -rw-r--r-- | transformers.py | 30 |
4 files changed, 38 insertions, 18 deletions
@@ -15,6 +15,12 @@ else: client_ids = {int(x): y+1 for y, x in enumerate(open(DATA_PATH+"/client_ids.txt"))} +def get_client_id(n): + if n in client_ids: + return client_ids[n] + else: + return 0 + porto_center = numpy.array([[ -8.61612, 41.1573]], dtype=theano.config.floatX) data_std = numpy.sqrt(numpy.array([[ 0.00333233, 0.00549598]], dtype=theano.config.floatX)) @@ -127,7 +133,7 @@ class TaxiData(Dataset): taxi_columns = [ ("trip_id", lambda x: x), ("call_type", CallType.from_data), - ("origin_call", lambda x: 0 if x == '' or x == 'NA' else client_ids[int(x)]), + ("origin_call", lambda x: 0 if x == '' or x == 'NA' else get_client_id(int(x))), ("origin_stand", lambda x: 0 if x == '' or x == 'NA' else int(x)), ("taxi_id", int), ("timestamp", int), @@ -144,13 +150,11 @@ taxi_columns_valid = taxi_columns + [ train_files=["%s/split/train-%02d.csv" % (DATA_PATH, i) for i in range(100)] valid_files=["%s/split/valid.csv" % (DATA_PATH,)] +test_file="%s/test.csv" % (DATA_PATH,) train_data=TaxiData(train_files, taxi_columns) - valid_data = TaxiData(valid_files, taxi_columns_valid) - -# for the moment - will be changed later -test_data = valid_data +test_data = TaxiData(test_file, taxi_columns, has_header=True) def train_it(): return DataIterator(DataStream(train_data)) diff --git a/make_valid.py b/make_valid.py index ffaa352..d5e147d 100644 --- a/make_valid.py +++ b/make_valid.py @@ -14,8 +14,8 @@ with open("valid-full.csv") as f: def make_valid_item(l): polyline = ast.literal_eval(l[-1]) last = polyline[-1] - cut_idx = random.randrange(len(polyline)-5) - cut = polyline[:cut_idx+6] + cut_idx = random.randrange(len(polyline)+1) + cut = polyline[:cut_idx] return l[:-1] + [ cut.__str__(), last[0], @@ -23,7 +23,7 @@ def make_valid_item(l): 15 * (len(polyline)-1), ] -vlines = map(make_valid_item, filter(lambda l: (len(ast.literal_eval(l[-1])) > 5), vlines)) +vlines = map(make_valid_item, filter(lambda l: (len(ast.literal_eval(l[-1])) > 0), vlines)) with open("valid.csv", "w") as f: wr = csv.writer(f) @@ -32,5 +32,6 @@ with open("valid.csv", "w") as f: with open("valid-solution.csv", "w") as f: wr = csv.writer(f) + wr.writerow(["TRIP_ID", "LATITUDE", "LONGITUDE"]) for r in vlines: wr.writerow([r[0], r[-2], r[-3]]) @@ -140,8 +140,8 @@ def main(): extensions=[DataStreamMonitoring([cost, hcost], valid_stream, prefix='valid', - every_n_batches=1), - Printing(every_n_batches=1), + every_n_batches=1000), + Printing(every_n_batches=1000), # Dump('taxi_model', every_n_batches=100), # LoadFromDump('taxi_model'), ] @@ -163,6 +163,7 @@ def main(): outfile = open("test-output.csv", "w") outcsv = csv.writer(outfile) + outcsv.writerow(["TRIP_ID", "LATITUDE", "LONGITUDE"]) for out in apply_model.Apply(outputs=outputs, stream=test_stream, return_vars=['trip_id', 'outputs']): dest = out['outputs'] for i, trip in enumerate(out['trip_id']): diff --git a/transformers.py b/transformers.py index 57d7f14..c60d362 100644 --- a/transformers.py +++ b/transformers.py @@ -3,6 +3,17 @@ import numpy import theano import random +def at_least_k(k, pl, pad_at_begin): + if len(pl) == 0: + pl = [[ -8.61612, 41.1573]] + if len(pl) < k: + if pad_at_begin: + pl = [pl[0]] * (k - len(pl)) + pl + else: + pl = pl + [pl[-1]] * (k - len(pl)) + return pl + + class Select(Transformer): def __init__(self, data_stream, sources): super(Select, self).__init__(data_stream) @@ -18,31 +29,34 @@ class Select(Transformer): def add_first_k(k, stream): id_polyline=stream.sources.index('polyline') def first_k(x): - return (numpy.array(x[id_polyline][:k], dtype=theano.config.floatX).flatten(),) - stream = Filter(stream, lambda x: len(x[id_polyline])>=k) + pl = at_least_k(k, x[id_polyline], False) + return (numpy.array(pl[:k], dtype=theano.config.floatX).flatten(),) stream = Mapping(stream, first_k, ('first_k',)) return stream def add_random_k(k, stream): id_polyline=stream.sources.index('polyline') def random_k(x): - loc = random.randrange(len(x[id_polyline])-k+1) - return (numpy.array(x[id_polyline][loc:loc+k], dtype=theano.config.floatX).flatten(),) - stream = Filter(stream, lambda x: len(x[id_polyline])>=k) + pl = at_least_k(k, x[id_polyline], True) + loc = random.randrange(len(pl)-k+1) + return (numpy.array(pl[loc:loc+k], dtype=theano.config.floatX).flatten(),) stream = Mapping(stream, random_k, ('last_k',)) return stream def add_last_k(k, stream): id_polyline=stream.sources.index('polyline') def last_k(x): - return (numpy.array(x[id_polyline][-k:], dtype=theano.config.floatX).flatten(),) - stream = Filter(stream, lambda x: len(x[id_polyline])>=k) + pl = at_least_k(k, x[id_polyline], True) + return (numpy.array(pl[-k:], dtype=theano.config.floatX).flatten(),) stream = Mapping(stream, last_k, ('last_k',)) return stream def add_destination(stream): id_polyline=stream.sources.index('polyline') - return Mapping(stream, lambda x: (numpy.array(x[id_polyline][-1], dtype=theano.config.floatX),), ('destination',)) + return Mapping(stream, + lambda x: + (numpy.array(at_least_k(1, x[id_polyline], True)[-1], dtype=theano.config.floatX),), + ('destination',)) def concat_destination_xy(stream): id_dx=stream.sources.index('destination_x') |