diff --git a/12-946-1650.events.pbf b/12-946-1650.events.pbf new file mode 100644 index 0000000..0d22a69 Binary files /dev/null and b/12-946-1650.events.pbf differ diff --git a/12-946-1650.speeds.pbf b/12-946-1650.speeds.pbf new file mode 100644 index 0000000..6d9860d --- /dev/null +++ b/12-946-1650.speeds.pbf @@ -0,0 +1,187 @@ +3 + 0a7951ce0993bdbb409e59cbbfa06bce +: +[ +)1 + 8f82c6def614e618f5eff8f525106983 +: +[ +)1 + f3ff8aedb92bdf47a493eb141f2b97ce +: +[ +/5 + e22df1548cc1ca7c1aa3d0625965d567 +: +[ + +HJK1 + bb924e1392528396daed920b5f99db68 +: +[ +#7 + e6a403c508d20f41930dc5bc5ff2ae1c +: +[ +`ab_7 + a56dafd99b2e7938e8747e400f672d33 +: +[ +aceg3 + 6139e2ba2383a76d45db8cfd92eabcd2 +: +[ ++7 + c2706e2ed69ed1b69e57976885fe1124 +: +[ +`YZ[; + bbb1e73a73012115a4aeaba5f2a2f9d4 +: +[ +TE9ZLO5 + ba73efef9b9825864547afecb7ae6b8f +: +[ + +5 + eb819dc9e0cf88d659eb03eabfa8a186 +: +_ + +$*-9 + c47f19794ecbf6731af710ad45a11f85 +: +[ +@9*K5 + 5111b18c3eab2835ea846b2970597e98 +: +[ + +`e^5 + 181832f2d27e3b99d36f3c4fb3620a05 +: +[ + +eghA + 511d38e248522f1c8988aa98b246a5f1 +: +[ + bUVIJKL^_ 7 + 21647905da729d1c76c16389bbf72798 +: +[ +%1 + b31510a15cefba316ff5889352c3f9dc +: +[ +`1 + 1d3611d8fd592778d6a670c04c6b82bd +: +[ ++1 + 8e16557879a9eefec5c6870942273dec +: +[ +3 + d28025dea7e0af975e89f9ed4afdd9c6 +: +[ +#/7 + 8416624ea5366043391e09446bfd6cb3 +: +[ +aXZ\J + 3c1c3c09da7176a9ba7274958cfb40b6 +: +[ +9:: +_ + + 3 + c73634dd8c593fb5f3aa0217f5dd5a94 +: +[ ++/3 + 7f49df272b058404314b1e41cda67dcf +: +[ +7:3 + 65306866ca37e589dc8197c3682b211b +: +[ +a[5 + daafd9cd95b1d57042aac841ef3c2a98 +: +[ + +#61 + 860e838f5b0295c65d74b4c1208c107d +: +[ +:; + 0402dba02130ce0500727af13b5a5d89 +: +[ +`bcdZ^3 + cfbaf3affac2b939e6eb0ff6d2cf740a +: +[ +UZ7 + 939a9d060113f8c61028fad8e5e05e2c +: +[ +`a^_3 + 3a55debe4e336a2798d26296e23749d7 +: +_ +$3 + 59d26934d91cea4432b53eeeae52b7c0 +: +_ +5 + b71f61681eb3818388311494ced83432 +: +[ + +CH:3 + 1094c0157df66b6a8a1040bafcd263e0 +: +[ +):1 + 3d23cab6a81780a528a2d1a4db009a41 +: +_ +$5 + b6b5a4f9b26bf288165584655fd57209 +: +[ + +67:1 + f879485a971f816d9c6bd69653dbe6a6 +: +[ +)1 + a943228e7ecbcce979e7081dda7a760a +: +[ +:> + 747faaf9c26693ca62d33c6bbccfd089 +: +[ +9: +_ + +9 + 54a639ea11773343700535bca8f73fd7 +: +[ +`bgh_D + 32e3129d06de166229c305108be5ad94 +: +[ +89: +_ + + + \ No newline at end of file diff --git a/README.md b/README.md index 0641091..bcc2c52 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,5 @@ # SharedStreets (Python) -🚧WIP -- commits/comments welcome! 🚧 - Python implementation of [SharedStreets Reference System](https://github.com/sharedstreets/sharedstreets-ref-system). ## Install @@ -16,6 +14,8 @@ Python implementation of [SharedStreets Reference System](https://github.com/sha ## Use +#### GeoJSON Functions + - Retrieve a tile and convert to GeoJSON in Python. import sharedstreets.tile @@ -30,6 +30,39 @@ Python implementation of [SharedStreets Reference System](https://github.com/sha gunicorn sharedstreets.webapp:app +### Analysis Functions + +#### Speed Data +Read speed data from PBF encoded speed tiles. `flatten_weekly_speed_histogram` returns an array of arrays representing individual histogram bins with the structure `[sharedstreets_referenceId, period_hour_of_week, bin_kmh, observation_count]`: + + import sharedstreets.speeds + with open('12-946-1650.speeds.pbf', 'rb') as file: + fileContent = file.read() + observations = speeds.flatten_weekly_speed_histogram(fileContent) + print(observations) + +#### Binned Linear Data +Read linear references from PBF encoded event tiles. `flatten_binned_events` returns an array of arrays representing individual event counts for linear bins `[sharedstreets_referenceId, reference_length, number_of_bins, bin_position, data_type, observation_count, observation_value]`: + + import sharedstreets.linear_references + with open('12-946-1650.events.pbf', 'rb') as file: + fileContent = file.read() + observations = linear_references.load_binned_events(fileContent) + + # shows all items as flattened arrays + for observation in observations: + for flattened_observation in observation.flatten(): + print(flattened_observation) + + # shows only items passing the filter criteria (e.g. min of 2 counts per bin/time period) + for observation in observations: + observation.filter(2) # filter by min count per bin/time period + for flattened_observation in observation.flatten(): + print(flattened_observation) + + + + ## Develop Install for local development. diff --git a/sharedstreets/linear_references.py b/sharedstreets/linear_references.py new file mode 100644 index 0000000..1a416e4 --- /dev/null +++ b/sharedstreets/linear_references.py @@ -0,0 +1,248 @@ +from __future__ import print_function +import argparse +import math +import copy +from . import tile +from . import linear_references_pb2 +from google.protobuf.internal.encoder import _VarintEncoder + +from collections import defaultdict + + +protobuf_classes = [ + linear_references_pb2.SharedStreetsWeeklyBinnedLinearReferences + ] + +parser = argparse.ArgumentParser(description='Read SharedStreets binned linear reference protobuf and convert to CSV') +parser.add_argument('filename', help='Protobuf SharedStreets binned linear reference data') + +MAX_SCALING_LCM_FACTOR = 10 # caps LCM at n * greater(x, y) + +def least_common_multiple(x, y): + # choose the greater number + if x > y: + greater = x + else: + greater = y + + max_factor = greater * MAX_SCALING_LCM_FACTOR + + while True: + + if greater >= max_factor: + lcm = greater + break + + if((greater % x == 0) and (greater % y == 0)): + lcm = greater + break + + greater += 1 + + return lcm + +def get_bin_count(ref_length, bin_size): + num_bins = math.floor(ref_length / bin_size) + 1 + +class BinnedLinearReference: + + def __init__(self, item): + self.reference_id = item.referenceId + + if item.scaledCounts: + self.scaled_counts = item.scaledCounts + else: + self.scaled_counts = False + + self.reference_length = float(item.referenceLength) / 100 + self.number_of_bins = item.numberOfBins + + if type(item) is linear_references_pb2.SharedStreetsWeeklyBinnedLinearReferences: + self.weeklyCylce = True + + # data indexed as a multi-dimentional array {dataType}[binPosition][periodOffset] + self.data = {} + + for bin_index in range(0, len(item.binPosition)): + + bin_position = item.binPosition[bin_index] + linear_bin = item.binnedPeriodicData[bin_index] + + for period_index in range(0, len(linear_bin.periodOffset)): + + period_offset = linear_bin.periodOffset[period_index] + bin_data = linear_bin.bins[period_index] + + for data_index in range(0, len(bin_data.dataType)): + + data_type = bin_data.dataType[data_index] + data_count = bin_data.count[data_index] + data_value = bin_data.value[data_index] + + + self.data.setdefault(data_type, {}).setdefault(bin_position, {})[period_offset] = {"count": data_count, "value": data_value} + + def __str__(self): + return self.reference_id + ': ' + str(self.reference_length) + 'm / ' + str(self.number_of_bins) + ' bins -> ' + '{0:.3g}'.format(self.get_bin_length()) + ' m/bin' + + def get_bin_length(self): + return self.reference_length / self.number_of_bins + + def get_max_count(self): + max_count = 0 + + for data_type in self.data: + for bin_pos in self.data[data_type]: + for bin_period in self.data[data_type][bin_pos]: + if 'count' in self.data[data_type][bin_pos][bin_period] and self.data[data_type][bin_pos][bin_period]['count'] > max_count: + max_count = self.data[data_type][bin_pos]['count'] + + return max_count + + def scale_counts(self, max_count): + + if max_count == None: + max_count = self.get_max_count() + + self.scaled_counts = True + + for data_type in self.data: + for bin_pos in self.data[data_type]: + for bin_period in self.data[data_type][bin_pos]: + if 'count' in self.data[data_type][bin_pos][bin_period] and self.data[data_type][bin_pos][bin_period]['count'] > 0: + self.data[data_type][bin_pos]['count'] = float(self.data[data_type][bin_pos]['count']) / float(max_count) + + + # resize bins works only with scaled counts + def resize_bins(self, new_bin_length, max_count): + + data_copy = copy.deepcopy(self.data) + + return + + + def merge(self, new_data): + + if self.reference_id != new_data.reference_id: + raise ValueError('References IDs do not match') + + if self.nubmer_of_bins != new_data.nubmer_of_bins: + raise ValueError('Bin count does not match') + + for data_type in self.data: + for bin_pos in self.data[data_type]: + for bin_period in self.data[data_type][bin_pos]: + + existing_count = self.data[data_type][bin_pos][bin_period]['count'] + existing_value = self.data[data_type][bin_pos][bin_period]['value'] + if data_type in new_data and bin_pos in new_data[data_type] and bin_period in new_data[data_type][bin_pos]: + + new_count = new_data.data[data_type][bin_pos][bin_period]['count'] + new_value = new_data.data[data_type][bin_pos][bin_period]['value'] + + self.data[data_type][bin_pos][bin_period]['count'] = existing_count + new_count + self.data[data_type][bin_pos][bin_period]['value'] = existing_value + new_value + + + def filter(self, min_count): + + data_copy = copy.deepcopy(self.data) + + for data_type in self.data: + for bin_pos in self.data[data_type]: + for bin_period in self.data[data_type][bin_pos]: + if 'count' in self.data[data_type][bin_pos][bin_period] and self.data[data_type][bin_pos][bin_period]['count'] < min_count: + data_copy[data_type][bin_pos].pop(bin_period, None) + + if len(data_copy[data_type][bin_pos]) == 0: + data_copy[data_type][bin_pos].pop(bin_period, None) + if len(data_copy[data_type]) == 0: + data_copy[data_type].pop(bin_pos, None) + if len(data_copy) == 0: + data_copy.pop(data_type, None) + + self.data = data_copy + + def print(self): + + for data_type in self.data: + print(self.reference_id + ': ' + data_type) + for bin_pos in self.data[data_type]: + print('\t ' + str(bin_pos)) + for bin_period in self.data[data_type][bin_pos]: + print('\t\t ' + str(bin_period)) + for attribute in self.data[data_type][bin_pos][bin_period]: + print('\t\t\t ' + str(attribute) + ":" + str(self.data[data_type][bin_pos][bin_period][attribute])) + + def toPbf(self): + + pbfBinnedLinearReference = linear_references_pb2.SharedStreetsWeeklyBinnedLinearReferences() + + pbfBinnedLinearReference.referenceId = self.reference_id + pbfBinnedLinearReference.referenceLength = int(round(self.reference_length * 100)) + pbfBinnedLinearReference.numberOfBins = self.number_of_bins + pbfBinnedLinearReference.scaledCounts = self.scaled_counts + + pbfData = {} + + # pivot data to pbf format + for data_type in self.data: + for bin_pos in self.data[data_type]: + for bin_period in self.data[data_type][bin_pos]: + + data_count = self.data[data_type][bin_pos][bin_period]['count'] + data_value = self.data[data_type][bin_pos][bin_period]['value'] + + # TODO handle multi data-type writes + pbfData.setdefault(bin_pos, {})[bin_period] = {"data_type": data_type, "count": data_count, "value": data_value} + + # write data to pbf + for bin_pos in pbfData: + bin_position = pbfBinnedLinearReference.binPosition.append(bin_pos) + bin_position_data = pbfBinnedLinearReference.binnedPeriodicData.add() + for bin_period in pbfData[bin_pos]: + bin_position_data.periodOffset.append(bin_period) + bin_period_data = bin_position_data.bins.add() + + bin_period_data.dataType.append(pbfData[bin_pos][bin_period]['data_type']) + bin_period_data.count.append(pbfData[bin_pos][bin_period]['count']) + bin_period_data.value.append(pbfData[bin_pos][bin_period]['value']) + + + return pbfBinnedLinearReference.SerializeToString() + + + def flatten(self): + flattend_observations = [] + for data_type in self.data: + for bin_pos in self.data[data_type]: + for bin_period in self.data[data_type][bin_pos]: + count = self.data[data_type][bin_pos][bin_period]['count'] + value = self.data[data_type][bin_pos][bin_period]['value'] + flattened_observation = [self.reference_id, self.reference_length, self.number_of_bins, bin_pos, bin_period, data_type, count, value] + flattend_observations.append(flattened_observation) + + return flattend_observations + + +# generates a length encoded stream of pbf +def generate_pbf(output_file, binned_observations): + + for binned_reference in binned_observations: + + pbf_data = binned_reference.toPbf() + size_of_data = len(pbf_data) + _VarintEncoder()(output_file.write, size_of_data, True) + output_file.write(pbf_data) + #output.append(pbf_data) + + + +def load_binned_events(tileContent): + binned_observations = [] + + for item in tile.read_objects(0, tileContent, linear_references_pb2.SharedStreetsWeeklyBinnedLinearReferences): + binned_reference = BinnedLinearReference(item) + binned_observations.append(binned_reference) + + return binned_observations diff --git a/sharedstreets/linear_references_pb2.py b/sharedstreets/linear_references_pb2.py new file mode 100644 index 0000000..b411f6d --- /dev/null +++ b/sharedstreets/linear_references_pb2.py @@ -0,0 +1,488 @@ +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: linear_references.proto + +import sys +_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) +from google.protobuf.internal import enum_type_wrapper +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +from google.protobuf import descriptor_pb2 +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor.FileDescriptor( + name='linear_references.proto', + package='', + syntax='proto3', + serialized_pb=_b('\n\x17linear_references.proto\"V\n\x0fLinearReference\x12\x15\n\rstartDistance\x18\x01 \x01(\x04\x12\x15\n\x0b\x65ndDistance\x18\x02 \x01(\x04H\x00\x42\x15\n\x13\x65ndDistance_present\"s\n\x1dSharedStreetsLinearReferences\x12\x13\n\x0breferenceId\x18\x01 \x01(\t\x12\x17\n\x0freferenceLength\x18\x02 \x01(\x04\x12$\n\nreferences\x18\x03 \x03(\x0b\x32\x10.LinearReference\"9\n\x07\x44\x61taBin\x12\x10\n\x08\x64\x61taType\x18\x01 \x03(\t\x12\r\n\x05\x63ount\x18\x02 \x03(\x04\x12\r\n\x05value\x18\x03 \x03(\x01\"B\n\x12\x42innedPeriodicData\x12\x14\n\x0cperiodOffset\x18\x01 \x03(\r\x12\x16\n\x04\x62ins\x18\x02 \x03(\x0b\x32\x08.DataBin\"\xac\x01\n#SharedStreetsBinnedLinearReferences\x12\x13\n\x0breferenceId\x18\x01 \x01(\t\x12\x14\n\x0cscaledCounts\x18\x02 \x01(\x08\x12\x17\n\x0freferenceLength\x18\x03 \x01(\x04\x12\x14\n\x0cnumberOfBins\x18\x04 \x01(\r\x12\x13\n\x0b\x62inPosition\x18\x05 \x03(\r\x12\x16\n\x04\x62ins\x18\x06 \x03(\x0b\x32\x08.DataBin\"\xec\x01\n)SharedStreetsWeeklyBinnedLinearReferences\x12\x13\n\x0breferenceId\x18\x01 \x01(\t\x12\x1f\n\nperiodSize\x18\x02 \x01(\x0e\x32\x0b.PeriodSize\x12\x14\n\x0cscaledCounts\x18\x03 \x01(\x08\x12\x17\n\x0freferenceLength\x18\x04 \x01(\x04\x12\x14\n\x0cnumberOfBins\x18\x05 \x01(\r\x12\x13\n\x0b\x62inPosition\x18\x06 \x03(\r\x12/\n\x12\x62innedPeriodicData\x18\x07 \x03(\x0b\x32\x13.BinnedPeriodicData*\xfb\x01\n\nPeriodSize\x12\r\n\tOneSecond\x10\x00\x12\x0f\n\x0b\x46iveSeconds\x10\x01\x12\x0e\n\nTenSeconds\x10\x02\x12\x12\n\x0e\x46ifteenSeconds\x10\x03\x12\x11\n\rThirtySeconds\x10\x04\x12\r\n\tOneMinute\x10\x05\x12\x0f\n\x0b\x46iveMinutes\x10\x06\x12\x0e\n\nTenMinutes\x10\x07\x12\x12\n\x0e\x46ifteenMinutes\x10\x08\x12\x11\n\rThirtyMinutes\x10\t\x12\x0b\n\x07OneHour\x10\n\x12\n\n\x06OneDay\x10\x0b\x12\x0b\n\x07OneWeek\x10\x0c\x12\x0c\n\x08OneMonth\x10\r\x12\x0b\n\x07OneYear\x10\x0e\x42$B\"SharedStreetsLinearReferencesProtob\x06proto3') +) + +_PERIODSIZE = _descriptor.EnumDescriptor( + name='PeriodSize', + full_name='PeriodSize', + filename=None, + file=DESCRIPTOR, + values=[ + _descriptor.EnumValueDescriptor( + name='OneSecond', index=0, number=0, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='FiveSeconds', index=1, number=1, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='TenSeconds', index=2, number=2, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='FifteenSeconds', index=3, number=3, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='ThirtySeconds', index=4, number=4, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='OneMinute', index=5, number=5, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='FiveMinutes', index=6, number=6, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='TenMinutes', index=7, number=7, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='FifteenMinutes', index=8, number=8, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='ThirtyMinutes', index=9, number=9, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='OneHour', index=10, number=10, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='OneDay', index=11, number=11, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='OneWeek', index=12, number=12, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='OneMonth', index=13, number=13, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='OneYear', index=14, number=14, + options=None, + type=None), + ], + containing_type=None, + options=None, + serialized_start=774, + serialized_end=1025, +) +_sym_db.RegisterEnumDescriptor(_PERIODSIZE) + +PeriodSize = enum_type_wrapper.EnumTypeWrapper(_PERIODSIZE) +OneSecond = 0 +FiveSeconds = 1 +TenSeconds = 2 +FifteenSeconds = 3 +ThirtySeconds = 4 +OneMinute = 5 +FiveMinutes = 6 +TenMinutes = 7 +FifteenMinutes = 8 +ThirtyMinutes = 9 +OneHour = 10 +OneDay = 11 +OneWeek = 12 +OneMonth = 13 +OneYear = 14 + + + +_LINEARREFERENCE = _descriptor.Descriptor( + name='LinearReference', + full_name='LinearReference', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='startDistance', full_name='LinearReference.startDistance', index=0, + number=1, type=4, cpp_type=4, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='endDistance', full_name='LinearReference.endDistance', index=1, + number=2, type=4, cpp_type=4, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + _descriptor.OneofDescriptor( + name='endDistance_present', full_name='LinearReference.endDistance_present', + index=0, containing_type=None, fields=[]), + ], + serialized_start=27, + serialized_end=113, +) + + +_SHAREDSTREETSLINEARREFERENCES = _descriptor.Descriptor( + name='SharedStreetsLinearReferences', + full_name='SharedStreetsLinearReferences', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='referenceId', full_name='SharedStreetsLinearReferences.referenceId', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='referenceLength', full_name='SharedStreetsLinearReferences.referenceLength', index=1, + number=2, type=4, cpp_type=4, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='references', full_name='SharedStreetsLinearReferences.references', index=2, + number=3, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=115, + serialized_end=230, +) + + +_DATABIN = _descriptor.Descriptor( + name='DataBin', + full_name='DataBin', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='dataType', full_name='DataBin.dataType', index=0, + number=1, type=9, cpp_type=9, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='count', full_name='DataBin.count', index=1, + number=2, type=4, cpp_type=4, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='value', full_name='DataBin.value', index=2, + number=3, type=1, cpp_type=5, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=232, + serialized_end=289, +) + + +_BINNEDPERIODICDATA = _descriptor.Descriptor( + name='BinnedPeriodicData', + full_name='BinnedPeriodicData', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='periodOffset', full_name='BinnedPeriodicData.periodOffset', index=0, + number=1, type=13, cpp_type=3, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='bins', full_name='BinnedPeriodicData.bins', index=1, + number=2, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=291, + serialized_end=357, +) + + +_SHAREDSTREETSBINNEDLINEARREFERENCES = _descriptor.Descriptor( + name='SharedStreetsBinnedLinearReferences', + full_name='SharedStreetsBinnedLinearReferences', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='referenceId', full_name='SharedStreetsBinnedLinearReferences.referenceId', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='scaledCounts', full_name='SharedStreetsBinnedLinearReferences.scaledCounts', index=1, + number=2, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='referenceLength', full_name='SharedStreetsBinnedLinearReferences.referenceLength', index=2, + number=3, type=4, cpp_type=4, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='numberOfBins', full_name='SharedStreetsBinnedLinearReferences.numberOfBins', index=3, + number=4, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='binPosition', full_name='SharedStreetsBinnedLinearReferences.binPosition', index=4, + number=5, type=13, cpp_type=3, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='bins', full_name='SharedStreetsBinnedLinearReferences.bins', index=5, + number=6, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=360, + serialized_end=532, +) + + +_SHAREDSTREETSWEEKLYBINNEDLINEARREFERENCES = _descriptor.Descriptor( + name='SharedStreetsWeeklyBinnedLinearReferences', + full_name='SharedStreetsWeeklyBinnedLinearReferences', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='referenceId', full_name='SharedStreetsWeeklyBinnedLinearReferences.referenceId', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='periodSize', full_name='SharedStreetsWeeklyBinnedLinearReferences.periodSize', index=1, + number=2, type=14, cpp_type=8, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='scaledCounts', full_name='SharedStreetsWeeklyBinnedLinearReferences.scaledCounts', index=2, + number=3, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='referenceLength', full_name='SharedStreetsWeeklyBinnedLinearReferences.referenceLength', index=3, + number=4, type=4, cpp_type=4, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='numberOfBins', full_name='SharedStreetsWeeklyBinnedLinearReferences.numberOfBins', index=4, + number=5, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='binPosition', full_name='SharedStreetsWeeklyBinnedLinearReferences.binPosition', index=5, + number=6, type=13, cpp_type=3, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='binnedPeriodicData', full_name='SharedStreetsWeeklyBinnedLinearReferences.binnedPeriodicData', index=6, + number=7, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=535, + serialized_end=771, +) + +_LINEARREFERENCE.oneofs_by_name['endDistance_present'].fields.append( + _LINEARREFERENCE.fields_by_name['endDistance']) +_LINEARREFERENCE.fields_by_name['endDistance'].containing_oneof = _LINEARREFERENCE.oneofs_by_name['endDistance_present'] +_SHAREDSTREETSLINEARREFERENCES.fields_by_name['references'].message_type = _LINEARREFERENCE +_BINNEDPERIODICDATA.fields_by_name['bins'].message_type = _DATABIN +_SHAREDSTREETSBINNEDLINEARREFERENCES.fields_by_name['bins'].message_type = _DATABIN +_SHAREDSTREETSWEEKLYBINNEDLINEARREFERENCES.fields_by_name['periodSize'].enum_type = _PERIODSIZE +_SHAREDSTREETSWEEKLYBINNEDLINEARREFERENCES.fields_by_name['binnedPeriodicData'].message_type = _BINNEDPERIODICDATA +DESCRIPTOR.message_types_by_name['LinearReference'] = _LINEARREFERENCE +DESCRIPTOR.message_types_by_name['SharedStreetsLinearReferences'] = _SHAREDSTREETSLINEARREFERENCES +DESCRIPTOR.message_types_by_name['DataBin'] = _DATABIN +DESCRIPTOR.message_types_by_name['BinnedPeriodicData'] = _BINNEDPERIODICDATA +DESCRIPTOR.message_types_by_name['SharedStreetsBinnedLinearReferences'] = _SHAREDSTREETSBINNEDLINEARREFERENCES +DESCRIPTOR.message_types_by_name['SharedStreetsWeeklyBinnedLinearReferences'] = _SHAREDSTREETSWEEKLYBINNEDLINEARREFERENCES +DESCRIPTOR.enum_types_by_name['PeriodSize'] = _PERIODSIZE +_sym_db.RegisterFileDescriptor(DESCRIPTOR) + +LinearReference = _reflection.GeneratedProtocolMessageType('LinearReference', (_message.Message,), dict( + DESCRIPTOR = _LINEARREFERENCE, + __module__ = 'linear_references_pb2' + # @@protoc_insertion_point(class_scope:LinearReference) + )) +_sym_db.RegisterMessage(LinearReference) + +SharedStreetsLinearReferences = _reflection.GeneratedProtocolMessageType('SharedStreetsLinearReferences', (_message.Message,), dict( + DESCRIPTOR = _SHAREDSTREETSLINEARREFERENCES, + __module__ = 'linear_references_pb2' + # @@protoc_insertion_point(class_scope:SharedStreetsLinearReferences) + )) +_sym_db.RegisterMessage(SharedStreetsLinearReferences) + +DataBin = _reflection.GeneratedProtocolMessageType('DataBin', (_message.Message,), dict( + DESCRIPTOR = _DATABIN, + __module__ = 'linear_references_pb2' + # @@protoc_insertion_point(class_scope:DataBin) + )) +_sym_db.RegisterMessage(DataBin) + +BinnedPeriodicData = _reflection.GeneratedProtocolMessageType('BinnedPeriodicData', (_message.Message,), dict( + DESCRIPTOR = _BINNEDPERIODICDATA, + __module__ = 'linear_references_pb2' + # @@protoc_insertion_point(class_scope:BinnedPeriodicData) + )) +_sym_db.RegisterMessage(BinnedPeriodicData) + +SharedStreetsBinnedLinearReferences = _reflection.GeneratedProtocolMessageType('SharedStreetsBinnedLinearReferences', (_message.Message,), dict( + DESCRIPTOR = _SHAREDSTREETSBINNEDLINEARREFERENCES, + __module__ = 'linear_references_pb2' + # @@protoc_insertion_point(class_scope:SharedStreetsBinnedLinearReferences) + )) +_sym_db.RegisterMessage(SharedStreetsBinnedLinearReferences) + +SharedStreetsWeeklyBinnedLinearReferences = _reflection.GeneratedProtocolMessageType('SharedStreetsWeeklyBinnedLinearReferences', (_message.Message,), dict( + DESCRIPTOR = _SHAREDSTREETSWEEKLYBINNEDLINEARREFERENCES, + __module__ = 'linear_references_pb2' + # @@protoc_insertion_point(class_scope:SharedStreetsWeeklyBinnedLinearReferences) + )) +_sym_db.RegisterMessage(SharedStreetsWeeklyBinnedLinearReferences) + + +DESCRIPTOR.has_options = True +DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('B\"SharedStreetsLinearReferencesProto')) +# @@protoc_insertion_point(module_scope) diff --git a/sharedstreets/speeds.py b/sharedstreets/speeds.py new file mode 100644 index 0000000..2179968 --- /dev/null +++ b/sharedstreets/speeds.py @@ -0,0 +1,25 @@ +from __future__ import print_function +import argparse +from . import tile +from . import speeds_pb2 + +protobuf_classes = [ + speeds_pb2.SharedStreetsWeeklySpeeds + ] + + +parser = argparse.ArgumentParser(description='Read SharedStreets Speed protobuf and convert to CSV') +parser.add_argument('filename', help='Protobuf SharedStreets speed data') + +def flatten_weekly_speed_histogram(tileContent): + speedObservations = [] + + count = 0; + for item in tile.read_objects(0, tileContent, speeds_pb2.SharedStreetsWeeklySpeeds): + count += 1 + for period in item.speedsByPeriod: + for frame in period.histogram: + for pos in range(0, len(frame.speedBin)): + speedObservations.append([item.referenceId, period.periodOffset[0], frame.speedBin[pos], frame.observationCount[pos]]) + + return speedObservations; diff --git a/sharedstreets/speeds_pb2.py b/sharedstreets/speeds_pb2.py new file mode 100644 index 0000000..6fabb5f --- /dev/null +++ b/sharedstreets/speeds_pb2.py @@ -0,0 +1,516 @@ +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: speeds.proto + +import sys +_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) +from google.protobuf.internal import enum_type_wrapper +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +from google.protobuf import descriptor_pb2 +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor.FileDescriptor( + name='speeds.proto', + package='', + syntax='proto3', + serialized_pb=_b('\n\x0cspeeds.proto\"G\n\x0eTemporalPeriod\x12\x1f\n\nperiodSize\x18\x01 \x01(\x0e\x32\x0b.PeriodSize\x12\x14\n\x0cperiodOffset\x18\x02 \x01(\x04\"X\n\x0bWeeklyCycle\x12\x0c\n\x04year\x18\x01 \x01(\r\x12\r\n\x05month\x18\x02 \x01(\r\x12\x0b\n\x03\x64\x61y\x18\x03 \x01(\r\x12\x1f\n\nperiodSize\x18\x04 \x01(\x0e\x32\x0b.PeriodSize\"<\n\x0eSpeedHistogram\x12\x10\n\x08speedBin\x18\x01 \x03(\r\x12\x18\n\x10observationCount\x18\x02 \x03(\r\"R\n\x16SpeedHistogramByPeriod\x12\x14\n\x0cperiodOffset\x18\x01 \x03(\r\x12\"\n\thistogram\x18\x02 \x03(\x0b\x32\x0f.SpeedHistogram\"O\n\x0cSpeedSummary\x12\x11\n\tmeanSpead\x18\x01 \x01(\r\x12\x12\n\npercentile\x18\x02 \x03(\r\x12\x18\n\x10observationCount\x18\x03 \x03(\r\"Q\n\x14SpeedSummaryByPeriod\x12\x14\n\x0cperiodOffset\x18\x01 \x03(\r\x12#\n\x0cspeedSummary\x18\x03 \x03(\x0b\x32\r.SpeedSummary\"\x91\x02\n\x19SharedStreetsWeeklySpeeds\x12\x13\n\x0breferenceId\x18\x01 \x01(\t\x12\x1f\n\nperiodSize\x18\x02 \x01(\x0e\x32\x0b.PeriodSize\x12\x14\n\x0cscaledCounts\x18\x03 \x01(\x08\x12\x17\n\x0freferenceLength\x18\x04 \x01(\x04\x12\x14\n\x0cnumberOfBins\x18\x05 \x01(\r\x12\x13\n\x0b\x62inPosition\x18\x06 \x03(\r\x12/\n\x0espeedsByPeriod\x18\x07 \x03(\x0b\x32\x17.SpeedHistogramByPeriod\x12\x33\n\x14speedSummaryByPeriod\x18\x08 \x03(\x0b\x32\x15.SpeedSummaryByPeriod*\xfb\x01\n\nPeriodSize\x12\r\n\tOneSecond\x10\x00\x12\x0f\n\x0b\x46iveSeconds\x10\x01\x12\x0e\n\nTenSeconds\x10\x02\x12\x12\n\x0e\x46ifteenSeconds\x10\x03\x12\x11\n\rThirtySeconds\x10\x04\x12\r\n\tOneMinute\x10\x05\x12\x0f\n\x0b\x46iveMinutes\x10\x06\x12\x0e\n\nTenMinutes\x10\x07\x12\x12\n\x0e\x46ifteenMinutes\x10\x08\x12\x11\n\rThirtyMinutes\x10\t\x12\x0b\n\x07OneHour\x10\n\x12\n\n\x06OneDay\x10\x0b\x12\x0b\n\x07OneWeek\x10\x0c\x12\x0c\n\x08OneMonth\x10\r\x12\x0b\n\x07OneYear\x10\x0e\x42\x1a\x42\x18SharedStreetsSpeedsProtob\x06proto3') +) + +_PERIODSIZE = _descriptor.EnumDescriptor( + name='PeriodSize', + full_name='PeriodSize', + filename=None, + file=DESCRIPTOR, + values=[ + _descriptor.EnumValueDescriptor( + name='OneSecond', index=0, number=0, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='FiveSeconds', index=1, number=1, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='TenSeconds', index=2, number=2, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='FifteenSeconds', index=3, number=3, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='ThirtySeconds', index=4, number=4, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='OneMinute', index=5, number=5, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='FiveMinutes', index=6, number=6, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='TenMinutes', index=7, number=7, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='FifteenMinutes', index=8, number=8, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='ThirtyMinutes', index=9, number=9, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='OneHour', index=10, number=10, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='OneDay', index=11, number=11, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='OneWeek', index=12, number=12, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='OneMonth', index=13, number=13, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='OneYear', index=14, number=14, + options=None, + type=None), + ], + containing_type=None, + options=None, + serialized_start=766, + serialized_end=1017, +) +_sym_db.RegisterEnumDescriptor(_PERIODSIZE) + +PeriodSize = enum_type_wrapper.EnumTypeWrapper(_PERIODSIZE) +OneSecond = 0 +FiveSeconds = 1 +TenSeconds = 2 +FifteenSeconds = 3 +ThirtySeconds = 4 +OneMinute = 5 +FiveMinutes = 6 +TenMinutes = 7 +FifteenMinutes = 8 +ThirtyMinutes = 9 +OneHour = 10 +OneDay = 11 +OneWeek = 12 +OneMonth = 13 +OneYear = 14 + + + +_TEMPORALPERIOD = _descriptor.Descriptor( + name='TemporalPeriod', + full_name='TemporalPeriod', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='periodSize', full_name='TemporalPeriod.periodSize', index=0, + number=1, type=14, cpp_type=8, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='periodOffset', full_name='TemporalPeriod.periodOffset', index=1, + number=2, type=4, cpp_type=4, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=16, + serialized_end=87, +) + + +_WEEKLYCYCLE = _descriptor.Descriptor( + name='WeeklyCycle', + full_name='WeeklyCycle', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='year', full_name='WeeklyCycle.year', index=0, + number=1, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='month', full_name='WeeklyCycle.month', index=1, + number=2, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='day', full_name='WeeklyCycle.day', index=2, + number=3, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='periodSize', full_name='WeeklyCycle.periodSize', index=3, + number=4, type=14, cpp_type=8, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=89, + serialized_end=177, +) + + +_SPEEDHISTOGRAM = _descriptor.Descriptor( + name='SpeedHistogram', + full_name='SpeedHistogram', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='speedBin', full_name='SpeedHistogram.speedBin', index=0, + number=1, type=13, cpp_type=3, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='observationCount', full_name='SpeedHistogram.observationCount', index=1, + number=2, type=13, cpp_type=3, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=179, + serialized_end=239, +) + + +_SPEEDHISTOGRAMBYPERIOD = _descriptor.Descriptor( + name='SpeedHistogramByPeriod', + full_name='SpeedHistogramByPeriod', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='periodOffset', full_name='SpeedHistogramByPeriod.periodOffset', index=0, + number=1, type=13, cpp_type=3, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='histogram', full_name='SpeedHistogramByPeriod.histogram', index=1, + number=2, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=241, + serialized_end=323, +) + + +_SPEEDSUMMARY = _descriptor.Descriptor( + name='SpeedSummary', + full_name='SpeedSummary', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='meanSpead', full_name='SpeedSummary.meanSpead', index=0, + number=1, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='percentile', full_name='SpeedSummary.percentile', index=1, + number=2, type=13, cpp_type=3, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='observationCount', full_name='SpeedSummary.observationCount', index=2, + number=3, type=13, cpp_type=3, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=325, + serialized_end=404, +) + + +_SPEEDSUMMARYBYPERIOD = _descriptor.Descriptor( + name='SpeedSummaryByPeriod', + full_name='SpeedSummaryByPeriod', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='periodOffset', full_name='SpeedSummaryByPeriod.periodOffset', index=0, + number=1, type=13, cpp_type=3, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='speedSummary', full_name='SpeedSummaryByPeriod.speedSummary', index=1, + number=3, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=406, + serialized_end=487, +) + + +_SHAREDSTREETSWEEKLYSPEEDS = _descriptor.Descriptor( + name='SharedStreetsWeeklySpeeds', + full_name='SharedStreetsWeeklySpeeds', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='referenceId', full_name='SharedStreetsWeeklySpeeds.referenceId', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='periodSize', full_name='SharedStreetsWeeklySpeeds.periodSize', index=1, + number=2, type=14, cpp_type=8, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='scaledCounts', full_name='SharedStreetsWeeklySpeeds.scaledCounts', index=2, + number=3, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='referenceLength', full_name='SharedStreetsWeeklySpeeds.referenceLength', index=3, + number=4, type=4, cpp_type=4, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='numberOfBins', full_name='SharedStreetsWeeklySpeeds.numberOfBins', index=4, + number=5, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='binPosition', full_name='SharedStreetsWeeklySpeeds.binPosition', index=5, + number=6, type=13, cpp_type=3, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='speedsByPeriod', full_name='SharedStreetsWeeklySpeeds.speedsByPeriod', index=6, + number=7, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='speedSummaryByPeriod', full_name='SharedStreetsWeeklySpeeds.speedSummaryByPeriod', index=7, + number=8, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=490, + serialized_end=763, +) + +_TEMPORALPERIOD.fields_by_name['periodSize'].enum_type = _PERIODSIZE +_WEEKLYCYCLE.fields_by_name['periodSize'].enum_type = _PERIODSIZE +_SPEEDHISTOGRAMBYPERIOD.fields_by_name['histogram'].message_type = _SPEEDHISTOGRAM +_SPEEDSUMMARYBYPERIOD.fields_by_name['speedSummary'].message_type = _SPEEDSUMMARY +_SHAREDSTREETSWEEKLYSPEEDS.fields_by_name['periodSize'].enum_type = _PERIODSIZE +_SHAREDSTREETSWEEKLYSPEEDS.fields_by_name['speedsByPeriod'].message_type = _SPEEDHISTOGRAMBYPERIOD +_SHAREDSTREETSWEEKLYSPEEDS.fields_by_name['speedSummaryByPeriod'].message_type = _SPEEDSUMMARYBYPERIOD +DESCRIPTOR.message_types_by_name['TemporalPeriod'] = _TEMPORALPERIOD +DESCRIPTOR.message_types_by_name['WeeklyCycle'] = _WEEKLYCYCLE +DESCRIPTOR.message_types_by_name['SpeedHistogram'] = _SPEEDHISTOGRAM +DESCRIPTOR.message_types_by_name['SpeedHistogramByPeriod'] = _SPEEDHISTOGRAMBYPERIOD +DESCRIPTOR.message_types_by_name['SpeedSummary'] = _SPEEDSUMMARY +DESCRIPTOR.message_types_by_name['SpeedSummaryByPeriod'] = _SPEEDSUMMARYBYPERIOD +DESCRIPTOR.message_types_by_name['SharedStreetsWeeklySpeeds'] = _SHAREDSTREETSWEEKLYSPEEDS +DESCRIPTOR.enum_types_by_name['PeriodSize'] = _PERIODSIZE +_sym_db.RegisterFileDescriptor(DESCRIPTOR) + +TemporalPeriod = _reflection.GeneratedProtocolMessageType('TemporalPeriod', (_message.Message,), dict( + DESCRIPTOR = _TEMPORALPERIOD, + __module__ = 'speeds_pb2' + # @@protoc_insertion_point(class_scope:TemporalPeriod) + )) +_sym_db.RegisterMessage(TemporalPeriod) + +WeeklyCycle = _reflection.GeneratedProtocolMessageType('WeeklyCycle', (_message.Message,), dict( + DESCRIPTOR = _WEEKLYCYCLE, + __module__ = 'speeds_pb2' + # @@protoc_insertion_point(class_scope:WeeklyCycle) + )) +_sym_db.RegisterMessage(WeeklyCycle) + +SpeedHistogram = _reflection.GeneratedProtocolMessageType('SpeedHistogram', (_message.Message,), dict( + DESCRIPTOR = _SPEEDHISTOGRAM, + __module__ = 'speeds_pb2' + # @@protoc_insertion_point(class_scope:SpeedHistogram) + )) +_sym_db.RegisterMessage(SpeedHistogram) + +SpeedHistogramByPeriod = _reflection.GeneratedProtocolMessageType('SpeedHistogramByPeriod', (_message.Message,), dict( + DESCRIPTOR = _SPEEDHISTOGRAMBYPERIOD, + __module__ = 'speeds_pb2' + # @@protoc_insertion_point(class_scope:SpeedHistogramByPeriod) + )) +_sym_db.RegisterMessage(SpeedHistogramByPeriod) + +SpeedSummary = _reflection.GeneratedProtocolMessageType('SpeedSummary', (_message.Message,), dict( + DESCRIPTOR = _SPEEDSUMMARY, + __module__ = 'speeds_pb2' + # @@protoc_insertion_point(class_scope:SpeedSummary) + )) +_sym_db.RegisterMessage(SpeedSummary) + +SpeedSummaryByPeriod = _reflection.GeneratedProtocolMessageType('SpeedSummaryByPeriod', (_message.Message,), dict( + DESCRIPTOR = _SPEEDSUMMARYBYPERIOD, + __module__ = 'speeds_pb2' + # @@protoc_insertion_point(class_scope:SpeedSummaryByPeriod) + )) +_sym_db.RegisterMessage(SpeedSummaryByPeriod) + +SharedStreetsWeeklySpeeds = _reflection.GeneratedProtocolMessageType('SharedStreetsWeeklySpeeds', (_message.Message,), dict( + DESCRIPTOR = _SHAREDSTREETSWEEKLYSPEEDS, + __module__ = 'speeds_pb2' + # @@protoc_insertion_point(class_scope:SharedStreetsWeeklySpeeds) + )) +_sym_db.RegisterMessage(SharedStreetsWeeklySpeeds) + + +DESCRIPTOR.has_options = True +DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('B\030SharedStreetsSpeedsProto')) +# @@protoc_insertion_point(module_scope) diff --git a/sharedstreets/tests/test_linear_references.py b/sharedstreets/tests/test_linear_references.py new file mode 100644 index 0000000..50a14fa --- /dev/null +++ b/sharedstreets/tests/test_linear_references.py @@ -0,0 +1,65 @@ +import unittest, mock, httmock, os, posixpath, ModestMaps.Geo +from .. import speeds_pb2 +from .. import linear_references + +def respond_locally(url, request): + path_parts = url.path.split(posixpath.sep) + local_path = os.path.join(os.path.dirname(__file__), 'data', *path_parts[1:]) + + if os.path.exists(local_path): + with open(local_path, 'rb') as file: + return httmock.response(200, file.read()) + + return httmock.response(404, 'Nope') + +class TestTile (unittest.TestCase): + + def test_linear_references(self): + filename = '12-946-1650.events.pbf' + with open(filename, 'rb') as file: + fileContent = file.read() + observations = linear_references.load_binned_events(fileContent) + self.assertEqual(len(observations), 31) + + test_output_filename = '__test__' + filename + try: + os.remove(test_output_filename) + except OSError: + pass + + newFile = open(test_output_filename, "wb") + linear_references.generate_pbf(newFile, observations) + newFile.close() + + with open(test_output_filename, 'rb') as file: + fileContent = file.read() + test_observations = linear_references.load_binned_events(fileContent) + self.assertEqual(len(test_observations), 31) + + + + flattened_count = 0 + for observation in observations: + for item in observation.flatten(): + flattened_count += 1 + print(item) + + self.assertEqual(flattened_count, 89) + + flattened_count = 0 + for observation in observations: + observation.filter(1) + for item in observation.flatten(): + flattened_count += 1 + print(item) + + self.assertEqual(flattened_count, 89) + + flattened_count = 0 + for observation in observations: + observation.filter(2) + for item in observation.flatten(): + flattened_count += 1 + print(item) + + self.assertEqual(flattened_count, 0) \ No newline at end of file diff --git a/sharedstreets/tests/test_speeds.py b/sharedstreets/tests/test_speeds.py new file mode 100644 index 0000000..cb24beb --- /dev/null +++ b/sharedstreets/tests/test_speeds.py @@ -0,0 +1,25 @@ +import unittest, mock, httmock, os, posixpath, ModestMaps.Geo +from .. import speeds_pb2 +from .. import speeds + +def respond_locally(url, request): + path_parts = url.path.split(posixpath.sep) + local_path = os.path.join(os.path.dirname(__file__), 'data', *path_parts[1:]) + + if os.path.exists(local_path): + with open(local_path, 'rb') as file: + return httmock.response(200, file.read()) + + return httmock.response(404, 'Nope') + +class TestTile (unittest.TestCase): + + def test_speeds(self): + + speedObservations = [] + + with open('12-946-1650.speeds.pbf', 'rb') as file: + fileContent = file.read() + observations = speeds.flatten_weekly_speed_histogram(fileContent) + print(observations) + self.assertEqual(len(observations), 124) diff --git a/sharedstreets/tile.py b/sharedstreets/tile.py index b0ff57c..f9c509b 100644 --- a/sharedstreets/tile.py +++ b/sharedstreets/tile.py @@ -4,14 +4,15 @@ from . import sharedstreets_pb2 logger = logging.getLogger(__name__) +logging.basicConfig(level=logging.DEBUG) # https://github.com/sharedstreets/sharedstreets-ref-system/issues/16 -DATA_URL_TEMPLATE, DATA_ZOOM = 'https://tiles.sharedstreets.io/osm/planet-180312/{z}-{x}-{y}.{layer}.6.pbf', 12 +DATA_URL_TEMPLATE, DATA_ZOOM = 'https://tiles.sharedstreets.io/osm/planet-180430/{z}-{x}-{y}.{layer}.6.pbf', 12 data_classes = { 'reference': sharedstreets_pb2.SharedStreetsReference, 'intersection': sharedstreets_pb2.SharedStreetsIntersection, 'geometry': sharedstreets_pb2.SharedStreetsGeometry, - 'metadata': sharedstreets_pb2.SharedStreetsMetadata, + 'metadata': sharedstreets_pb2.SharedStreetsMetadata } # Used for Mercator projection and tile space @@ -36,20 +37,13 @@ def round_coord(float): ''' return round(float, 7) -def iter_objects(url, DataClass): - ''' Generate a stream of objects from the protobuf URL. - ''' - response, position = requests.get(url), 0 - logger.debug('Got {} bytes: {}'.format(len(response.content), repr(response.content[:32]))) - if response.status_code not in range(200, 299): - logger.debug('Got HTTP {}'.format(response.status_code)) - return +def read_objects(position, content, DataClass): - while position < len(response.content): - message_length, new_position = _DecodeVarint32(response.content, position) + while position < len(content): + message_length, new_position = _DecodeVarint32(content, position) position = new_position - message = response.content[position:position+message_length] + message = content[position:position+message_length] position += message_length try: @@ -61,6 +55,35 @@ def iter_objects(url, DataClass): else: yield object +def iter_objects(url, DataClass): + ''' Generate a stream of objects from the protobuf URL. + ''' + response, position = requests.get(url), 0 + logger.debug('Got {} bytes: {}'.format(len(response.content), repr(response.content[:32]))) + + if response.status_code not in range(200, 299): + logger.debug('Got HTTP {}'.format(response.status_code)) + return + + for item in read_objects(position, response.content, DataClass): + yield item + + # while position < len(response.content): + # message_length, new_position = _DecodeVarint32(response.content, position) + # position = new_position + # message = response.content[position:position+message_length] + # position += message_length + + # try: + # object = DataClass() + # object.ParseFromString(message) + # except google.protobuf.message.DecodeError: + # # Empty tile? Shrug. + # continue + # else: + # yield object + + def is_inside(southwest, northeast, geometry): ''' Return True if the geometry bbox is inside a location pair bbox. ''' @@ -209,6 +232,8 @@ def reference_feature(reference, id_length): ] } + + def make_geojson(tile, id_length=32): ''' Get a GeoJSON dictionary for a geographic tile.