From 7e2412f85925093d773b6bb8eba8f283005d003e Mon Sep 17 00:00:00 2001 From: Kevin Webb Date: Mon, 30 Jul 2018 15:57:52 -0400 Subject: [PATCH 1/9] added python analysis utils --- 12-946-1650.events.pbf | Bin 0 -> 4085 bytes 12-946-1650.speeds.pbf | 187 +++++++ sharedstreets/linear_references.py | 26 + sharedstreets/linear_references_pb2.py | 488 +++++++++++++++++ sharedstreets/speeds.py | 25 + sharedstreets/speeds_pb2.py | 516 ++++++++++++++++++ sharedstreets/tests/test_linear_references.py | 25 + sharedstreets/tests/test_speeds.py | 25 + sharedstreets/tile.py | 49 +- 9 files changed, 1329 insertions(+), 12 deletions(-) create mode 100644 12-946-1650.events.pbf create mode 100644 12-946-1650.speeds.pbf create mode 100644 sharedstreets/linear_references.py create mode 100644 sharedstreets/linear_references_pb2.py create mode 100644 sharedstreets/speeds.py create mode 100644 sharedstreets/speeds_pb2.py create mode 100644 sharedstreets/tests/test_linear_references.py create mode 100644 sharedstreets/tests/test_speeds.py diff --git a/12-946-1650.events.pbf b/12-946-1650.events.pbf new file mode 100644 index 0000000000000000000000000000000000000000..0d22a69b0474c9a96451f1276e7bd2b68df44dbe GIT binary patch literal 4085 zcmZ`+X>gTS8P2^o3E>t?38jPp$w$KSQJ|cCNkX|XvJ7IGhH1gVP|kOjFoTW=ZY*Ml zF<2d1Kq+M@ML>bkR+vJ^TChl2ib|oaGL>O~AqWA%E>P*dT zM03T_>)*XR|MV)H+nMr_m?8jc3WPzx5hyV*nkJe$hm?gnx^^-XH=@#eBA6+D+Sl41 zU%V#W7>M=IzTAP9$`G1qdGyyyhKJA%N8W2(d7~ls1Ak9NnNiM2nIOw07FKY=EdjtW zjts`pwRM@8piErdtN1m)ejJRLH!T$Ehx?xz|M0dDIvM7+_ng_3o1Bdp#L}WfAuO=s z%n)i4q5Z{>Rf#qz>KqZ5qg0?)+jcxM=K2S9g^V=ffLUY>lh(k*e<_!o2|=BO5RvGO zx6<)wpHWrM;u&38IS~NDP&!pu z+{e3dz^7G<-mPpMR*2IbeRfUnC9E;d9rIn&ggYp?Bt8ef0~jjDAy9rG6@*UEX#F$k zcql4M^{whPsCZKSuDo-neFP39HSYfZy#Hit6hH@#f~Ti!8XQ31Y`EOj@hS+QEhiR# zR@*i-PjnPQm_VWP6+!}2s{~;Qk{nRUtrXG0L+N-VO82fUp2GV59Rssnp&gifblv3b zwT<~eP(Xo^Lk4k+6DbX54r}5P%n(q3Mn6B5iU*=}kE-H<-fntg!t%9EfsE>Q?dhC< zwJEnPQ%K2penLc0gdEgD3Zaw~2_nod6^u6S=^1ZIp}Q+_*~pEl`Ln*bDz&V3eDSa^ z{o{{wSKTCmu+`J|9zVM`)WFu4&zD{Vq0QOaIc?yXW1*8UnGpJSc4pSFnIhCNy>!gA-wp`n`rSd05iS8WiX&;D=1fC|h4ezr5%*Jf;~Q~rlu1_vv}E46WgFv;xscDaoKn4n+k2i$YSUl2AHvsF@&VX zfI=V=OO=r%>KGQ6p-d`Wn$Mv-ea@NqX#L!gRe^%t`ndHwZ%rLrK)e6DW$f)a%bH8` z7Qi3@5H(hD3zU$h>Pp9KiC{t3?yLeahhnruYqC&Ns^un=SB_$yX(B5;$m(R|TxeIw!-r0B)o3QBS?6{Yok4$NKMe*7eR`d0{0@d@id%I>&CV-AT z`nT2KZkawr{>;nq!8EF+x*` zT|ymJ3LwZGHHIobI2$VBG)e`u|B1Oz*Bp9dRQ@3Dh%^pylmNe}zGIMh??j+BigBY| z^!)~}?jiSAed+Jb6&0XhBPj+>vWVFA%Jl3c@U56@LrWYUAgLQWLbh-27-=yt6alh9lJneUC+ KGkf)@+ + 747faaf9c26693ca62d33c6bbccfd089 +: +[ +9: +_ + +9 + 54a639ea11773343700535bca8f73fd7 +: +[ +`bgh_D + 32e3129d06de166229c305108be5ad94 +: +[ +89: +_ + + + \ No newline at end of file diff --git a/sharedstreets/linear_references.py b/sharedstreets/linear_references.py new file mode 100644 index 0000000..47d4326 --- /dev/null +++ b/sharedstreets/linear_references.py @@ -0,0 +1,26 @@ +from __future__ import print_function +import argparse +from . import tile +from . import linear_references_pb2 + +protobuf_classes = [ + linear_references_pb2.SharedStreetsWeeklyBinnedLinearReferences + ] + + +parser = argparse.ArgumentParser(description='Read SharedStreets binned linear reference protobuf and convert to CSV') +parser.add_argument('filename', help='Protobuf SharedStreets binned linear reference data') + +def flatten_binned_events(tileContent): + binnedObservations = [] + + count = 0; + for item in tile.read_objects(0, tileContent, linear_references_pb2.SharedStreetsWeeklyBinnedLinearReferences): + count += 1 + for binPos in range(0, len(item.binPosition)): + linearBin = item.binnedPeriodicData[binPos] + for periodPos in range(0, len(linearBin.bins)): + binData = linearBin.bins[periodPos] + binnedObservations.append([item.referenceId, item.numberOfBins, binPos+1, binData.dataType[0], binData.count[0], binData.value[0]]) + + return binnedObservations; diff --git a/sharedstreets/linear_references_pb2.py b/sharedstreets/linear_references_pb2.py new file mode 100644 index 0000000..b411f6d --- /dev/null +++ b/sharedstreets/linear_references_pb2.py @@ -0,0 +1,488 @@ +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: linear_references.proto + +import sys +_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) +from google.protobuf.internal import enum_type_wrapper +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +from google.protobuf import descriptor_pb2 +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor.FileDescriptor( + name='linear_references.proto', + package='', + syntax='proto3', + serialized_pb=_b('\n\x17linear_references.proto\"V\n\x0fLinearReference\x12\x15\n\rstartDistance\x18\x01 \x01(\x04\x12\x15\n\x0b\x65ndDistance\x18\x02 \x01(\x04H\x00\x42\x15\n\x13\x65ndDistance_present\"s\n\x1dSharedStreetsLinearReferences\x12\x13\n\x0breferenceId\x18\x01 \x01(\t\x12\x17\n\x0freferenceLength\x18\x02 \x01(\x04\x12$\n\nreferences\x18\x03 \x03(\x0b\x32\x10.LinearReference\"9\n\x07\x44\x61taBin\x12\x10\n\x08\x64\x61taType\x18\x01 \x03(\t\x12\r\n\x05\x63ount\x18\x02 \x03(\x04\x12\r\n\x05value\x18\x03 \x03(\x01\"B\n\x12\x42innedPeriodicData\x12\x14\n\x0cperiodOffset\x18\x01 \x03(\r\x12\x16\n\x04\x62ins\x18\x02 \x03(\x0b\x32\x08.DataBin\"\xac\x01\n#SharedStreetsBinnedLinearReferences\x12\x13\n\x0breferenceId\x18\x01 \x01(\t\x12\x14\n\x0cscaledCounts\x18\x02 \x01(\x08\x12\x17\n\x0freferenceLength\x18\x03 \x01(\x04\x12\x14\n\x0cnumberOfBins\x18\x04 \x01(\r\x12\x13\n\x0b\x62inPosition\x18\x05 \x03(\r\x12\x16\n\x04\x62ins\x18\x06 \x03(\x0b\x32\x08.DataBin\"\xec\x01\n)SharedStreetsWeeklyBinnedLinearReferences\x12\x13\n\x0breferenceId\x18\x01 \x01(\t\x12\x1f\n\nperiodSize\x18\x02 \x01(\x0e\x32\x0b.PeriodSize\x12\x14\n\x0cscaledCounts\x18\x03 \x01(\x08\x12\x17\n\x0freferenceLength\x18\x04 \x01(\x04\x12\x14\n\x0cnumberOfBins\x18\x05 \x01(\r\x12\x13\n\x0b\x62inPosition\x18\x06 \x03(\r\x12/\n\x12\x62innedPeriodicData\x18\x07 \x03(\x0b\x32\x13.BinnedPeriodicData*\xfb\x01\n\nPeriodSize\x12\r\n\tOneSecond\x10\x00\x12\x0f\n\x0b\x46iveSeconds\x10\x01\x12\x0e\n\nTenSeconds\x10\x02\x12\x12\n\x0e\x46ifteenSeconds\x10\x03\x12\x11\n\rThirtySeconds\x10\x04\x12\r\n\tOneMinute\x10\x05\x12\x0f\n\x0b\x46iveMinutes\x10\x06\x12\x0e\n\nTenMinutes\x10\x07\x12\x12\n\x0e\x46ifteenMinutes\x10\x08\x12\x11\n\rThirtyMinutes\x10\t\x12\x0b\n\x07OneHour\x10\n\x12\n\n\x06OneDay\x10\x0b\x12\x0b\n\x07OneWeek\x10\x0c\x12\x0c\n\x08OneMonth\x10\r\x12\x0b\n\x07OneYear\x10\x0e\x42$B\"SharedStreetsLinearReferencesProtob\x06proto3') +) + +_PERIODSIZE = _descriptor.EnumDescriptor( + name='PeriodSize', + full_name='PeriodSize', + filename=None, + file=DESCRIPTOR, + values=[ + _descriptor.EnumValueDescriptor( + name='OneSecond', index=0, number=0, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='FiveSeconds', index=1, number=1, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='TenSeconds', index=2, number=2, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='FifteenSeconds', index=3, number=3, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='ThirtySeconds', index=4, number=4, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='OneMinute', index=5, number=5, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='FiveMinutes', index=6, number=6, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='TenMinutes', index=7, number=7, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='FifteenMinutes', index=8, number=8, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='ThirtyMinutes', index=9, number=9, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='OneHour', index=10, number=10, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='OneDay', index=11, number=11, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='OneWeek', index=12, number=12, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='OneMonth', index=13, number=13, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='OneYear', index=14, number=14, + options=None, + type=None), + ], + containing_type=None, + options=None, + serialized_start=774, + serialized_end=1025, +) +_sym_db.RegisterEnumDescriptor(_PERIODSIZE) + +PeriodSize = enum_type_wrapper.EnumTypeWrapper(_PERIODSIZE) +OneSecond = 0 +FiveSeconds = 1 +TenSeconds = 2 +FifteenSeconds = 3 +ThirtySeconds = 4 +OneMinute = 5 +FiveMinutes = 6 +TenMinutes = 7 +FifteenMinutes = 8 +ThirtyMinutes = 9 +OneHour = 10 +OneDay = 11 +OneWeek = 12 +OneMonth = 13 +OneYear = 14 + + + +_LINEARREFERENCE = _descriptor.Descriptor( + name='LinearReference', + full_name='LinearReference', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='startDistance', full_name='LinearReference.startDistance', index=0, + number=1, type=4, cpp_type=4, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='endDistance', full_name='LinearReference.endDistance', index=1, + number=2, type=4, cpp_type=4, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + _descriptor.OneofDescriptor( + name='endDistance_present', full_name='LinearReference.endDistance_present', + index=0, containing_type=None, fields=[]), + ], + serialized_start=27, + serialized_end=113, +) + + +_SHAREDSTREETSLINEARREFERENCES = _descriptor.Descriptor( + name='SharedStreetsLinearReferences', + full_name='SharedStreetsLinearReferences', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='referenceId', full_name='SharedStreetsLinearReferences.referenceId', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='referenceLength', full_name='SharedStreetsLinearReferences.referenceLength', index=1, + number=2, type=4, cpp_type=4, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='references', full_name='SharedStreetsLinearReferences.references', index=2, + number=3, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=115, + serialized_end=230, +) + + +_DATABIN = _descriptor.Descriptor( + name='DataBin', + full_name='DataBin', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='dataType', full_name='DataBin.dataType', index=0, + number=1, type=9, cpp_type=9, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='count', full_name='DataBin.count', index=1, + number=2, type=4, cpp_type=4, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='value', full_name='DataBin.value', index=2, + number=3, type=1, cpp_type=5, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=232, + serialized_end=289, +) + + +_BINNEDPERIODICDATA = _descriptor.Descriptor( + name='BinnedPeriodicData', + full_name='BinnedPeriodicData', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='periodOffset', full_name='BinnedPeriodicData.periodOffset', index=0, + number=1, type=13, cpp_type=3, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='bins', full_name='BinnedPeriodicData.bins', index=1, + number=2, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=291, + serialized_end=357, +) + + +_SHAREDSTREETSBINNEDLINEARREFERENCES = _descriptor.Descriptor( + name='SharedStreetsBinnedLinearReferences', + full_name='SharedStreetsBinnedLinearReferences', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='referenceId', full_name='SharedStreetsBinnedLinearReferences.referenceId', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='scaledCounts', full_name='SharedStreetsBinnedLinearReferences.scaledCounts', index=1, + number=2, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='referenceLength', full_name='SharedStreetsBinnedLinearReferences.referenceLength', index=2, + number=3, type=4, cpp_type=4, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='numberOfBins', full_name='SharedStreetsBinnedLinearReferences.numberOfBins', index=3, + number=4, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='binPosition', full_name='SharedStreetsBinnedLinearReferences.binPosition', index=4, + number=5, type=13, cpp_type=3, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='bins', full_name='SharedStreetsBinnedLinearReferences.bins', index=5, + number=6, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=360, + serialized_end=532, +) + + +_SHAREDSTREETSWEEKLYBINNEDLINEARREFERENCES = _descriptor.Descriptor( + name='SharedStreetsWeeklyBinnedLinearReferences', + full_name='SharedStreetsWeeklyBinnedLinearReferences', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='referenceId', full_name='SharedStreetsWeeklyBinnedLinearReferences.referenceId', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='periodSize', full_name='SharedStreetsWeeklyBinnedLinearReferences.periodSize', index=1, + number=2, type=14, cpp_type=8, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='scaledCounts', full_name='SharedStreetsWeeklyBinnedLinearReferences.scaledCounts', index=2, + number=3, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='referenceLength', full_name='SharedStreetsWeeklyBinnedLinearReferences.referenceLength', index=3, + number=4, type=4, cpp_type=4, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='numberOfBins', full_name='SharedStreetsWeeklyBinnedLinearReferences.numberOfBins', index=4, + number=5, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='binPosition', full_name='SharedStreetsWeeklyBinnedLinearReferences.binPosition', index=5, + number=6, type=13, cpp_type=3, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='binnedPeriodicData', full_name='SharedStreetsWeeklyBinnedLinearReferences.binnedPeriodicData', index=6, + number=7, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=535, + serialized_end=771, +) + +_LINEARREFERENCE.oneofs_by_name['endDistance_present'].fields.append( + _LINEARREFERENCE.fields_by_name['endDistance']) +_LINEARREFERENCE.fields_by_name['endDistance'].containing_oneof = _LINEARREFERENCE.oneofs_by_name['endDistance_present'] +_SHAREDSTREETSLINEARREFERENCES.fields_by_name['references'].message_type = _LINEARREFERENCE +_BINNEDPERIODICDATA.fields_by_name['bins'].message_type = _DATABIN +_SHAREDSTREETSBINNEDLINEARREFERENCES.fields_by_name['bins'].message_type = _DATABIN +_SHAREDSTREETSWEEKLYBINNEDLINEARREFERENCES.fields_by_name['periodSize'].enum_type = _PERIODSIZE +_SHAREDSTREETSWEEKLYBINNEDLINEARREFERENCES.fields_by_name['binnedPeriodicData'].message_type = _BINNEDPERIODICDATA +DESCRIPTOR.message_types_by_name['LinearReference'] = _LINEARREFERENCE +DESCRIPTOR.message_types_by_name['SharedStreetsLinearReferences'] = _SHAREDSTREETSLINEARREFERENCES +DESCRIPTOR.message_types_by_name['DataBin'] = _DATABIN +DESCRIPTOR.message_types_by_name['BinnedPeriodicData'] = _BINNEDPERIODICDATA +DESCRIPTOR.message_types_by_name['SharedStreetsBinnedLinearReferences'] = _SHAREDSTREETSBINNEDLINEARREFERENCES +DESCRIPTOR.message_types_by_name['SharedStreetsWeeklyBinnedLinearReferences'] = _SHAREDSTREETSWEEKLYBINNEDLINEARREFERENCES +DESCRIPTOR.enum_types_by_name['PeriodSize'] = _PERIODSIZE +_sym_db.RegisterFileDescriptor(DESCRIPTOR) + +LinearReference = _reflection.GeneratedProtocolMessageType('LinearReference', (_message.Message,), dict( + DESCRIPTOR = _LINEARREFERENCE, + __module__ = 'linear_references_pb2' + # @@protoc_insertion_point(class_scope:LinearReference) + )) +_sym_db.RegisterMessage(LinearReference) + +SharedStreetsLinearReferences = _reflection.GeneratedProtocolMessageType('SharedStreetsLinearReferences', (_message.Message,), dict( + DESCRIPTOR = _SHAREDSTREETSLINEARREFERENCES, + __module__ = 'linear_references_pb2' + # @@protoc_insertion_point(class_scope:SharedStreetsLinearReferences) + )) +_sym_db.RegisterMessage(SharedStreetsLinearReferences) + +DataBin = _reflection.GeneratedProtocolMessageType('DataBin', (_message.Message,), dict( + DESCRIPTOR = _DATABIN, + __module__ = 'linear_references_pb2' + # @@protoc_insertion_point(class_scope:DataBin) + )) +_sym_db.RegisterMessage(DataBin) + +BinnedPeriodicData = _reflection.GeneratedProtocolMessageType('BinnedPeriodicData', (_message.Message,), dict( + DESCRIPTOR = _BINNEDPERIODICDATA, + __module__ = 'linear_references_pb2' + # @@protoc_insertion_point(class_scope:BinnedPeriodicData) + )) +_sym_db.RegisterMessage(BinnedPeriodicData) + +SharedStreetsBinnedLinearReferences = _reflection.GeneratedProtocolMessageType('SharedStreetsBinnedLinearReferences', (_message.Message,), dict( + DESCRIPTOR = _SHAREDSTREETSBINNEDLINEARREFERENCES, + __module__ = 'linear_references_pb2' + # @@protoc_insertion_point(class_scope:SharedStreetsBinnedLinearReferences) + )) +_sym_db.RegisterMessage(SharedStreetsBinnedLinearReferences) + +SharedStreetsWeeklyBinnedLinearReferences = _reflection.GeneratedProtocolMessageType('SharedStreetsWeeklyBinnedLinearReferences', (_message.Message,), dict( + DESCRIPTOR = _SHAREDSTREETSWEEKLYBINNEDLINEARREFERENCES, + __module__ = 'linear_references_pb2' + # @@protoc_insertion_point(class_scope:SharedStreetsWeeklyBinnedLinearReferences) + )) +_sym_db.RegisterMessage(SharedStreetsWeeklyBinnedLinearReferences) + + +DESCRIPTOR.has_options = True +DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('B\"SharedStreetsLinearReferencesProto')) +# @@protoc_insertion_point(module_scope) diff --git a/sharedstreets/speeds.py b/sharedstreets/speeds.py new file mode 100644 index 0000000..2179968 --- /dev/null +++ b/sharedstreets/speeds.py @@ -0,0 +1,25 @@ +from __future__ import print_function +import argparse +from . import tile +from . import speeds_pb2 + +protobuf_classes = [ + speeds_pb2.SharedStreetsWeeklySpeeds + ] + + +parser = argparse.ArgumentParser(description='Read SharedStreets Speed protobuf and convert to CSV') +parser.add_argument('filename', help='Protobuf SharedStreets speed data') + +def flatten_weekly_speed_histogram(tileContent): + speedObservations = [] + + count = 0; + for item in tile.read_objects(0, tileContent, speeds_pb2.SharedStreetsWeeklySpeeds): + count += 1 + for period in item.speedsByPeriod: + for frame in period.histogram: + for pos in range(0, len(frame.speedBin)): + speedObservations.append([item.referenceId, period.periodOffset[0], frame.speedBin[pos], frame.observationCount[pos]]) + + return speedObservations; diff --git a/sharedstreets/speeds_pb2.py b/sharedstreets/speeds_pb2.py new file mode 100644 index 0000000..6fabb5f --- /dev/null +++ b/sharedstreets/speeds_pb2.py @@ -0,0 +1,516 @@ +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: speeds.proto + +import sys +_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) +from google.protobuf.internal import enum_type_wrapper +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +from google.protobuf import descriptor_pb2 +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor.FileDescriptor( + name='speeds.proto', + package='', + syntax='proto3', + serialized_pb=_b('\n\x0cspeeds.proto\"G\n\x0eTemporalPeriod\x12\x1f\n\nperiodSize\x18\x01 \x01(\x0e\x32\x0b.PeriodSize\x12\x14\n\x0cperiodOffset\x18\x02 \x01(\x04\"X\n\x0bWeeklyCycle\x12\x0c\n\x04year\x18\x01 \x01(\r\x12\r\n\x05month\x18\x02 \x01(\r\x12\x0b\n\x03\x64\x61y\x18\x03 \x01(\r\x12\x1f\n\nperiodSize\x18\x04 \x01(\x0e\x32\x0b.PeriodSize\"<\n\x0eSpeedHistogram\x12\x10\n\x08speedBin\x18\x01 \x03(\r\x12\x18\n\x10observationCount\x18\x02 \x03(\r\"R\n\x16SpeedHistogramByPeriod\x12\x14\n\x0cperiodOffset\x18\x01 \x03(\r\x12\"\n\thistogram\x18\x02 \x03(\x0b\x32\x0f.SpeedHistogram\"O\n\x0cSpeedSummary\x12\x11\n\tmeanSpead\x18\x01 \x01(\r\x12\x12\n\npercentile\x18\x02 \x03(\r\x12\x18\n\x10observationCount\x18\x03 \x03(\r\"Q\n\x14SpeedSummaryByPeriod\x12\x14\n\x0cperiodOffset\x18\x01 \x03(\r\x12#\n\x0cspeedSummary\x18\x03 \x03(\x0b\x32\r.SpeedSummary\"\x91\x02\n\x19SharedStreetsWeeklySpeeds\x12\x13\n\x0breferenceId\x18\x01 \x01(\t\x12\x1f\n\nperiodSize\x18\x02 \x01(\x0e\x32\x0b.PeriodSize\x12\x14\n\x0cscaledCounts\x18\x03 \x01(\x08\x12\x17\n\x0freferenceLength\x18\x04 \x01(\x04\x12\x14\n\x0cnumberOfBins\x18\x05 \x01(\r\x12\x13\n\x0b\x62inPosition\x18\x06 \x03(\r\x12/\n\x0espeedsByPeriod\x18\x07 \x03(\x0b\x32\x17.SpeedHistogramByPeriod\x12\x33\n\x14speedSummaryByPeriod\x18\x08 \x03(\x0b\x32\x15.SpeedSummaryByPeriod*\xfb\x01\n\nPeriodSize\x12\r\n\tOneSecond\x10\x00\x12\x0f\n\x0b\x46iveSeconds\x10\x01\x12\x0e\n\nTenSeconds\x10\x02\x12\x12\n\x0e\x46ifteenSeconds\x10\x03\x12\x11\n\rThirtySeconds\x10\x04\x12\r\n\tOneMinute\x10\x05\x12\x0f\n\x0b\x46iveMinutes\x10\x06\x12\x0e\n\nTenMinutes\x10\x07\x12\x12\n\x0e\x46ifteenMinutes\x10\x08\x12\x11\n\rThirtyMinutes\x10\t\x12\x0b\n\x07OneHour\x10\n\x12\n\n\x06OneDay\x10\x0b\x12\x0b\n\x07OneWeek\x10\x0c\x12\x0c\n\x08OneMonth\x10\r\x12\x0b\n\x07OneYear\x10\x0e\x42\x1a\x42\x18SharedStreetsSpeedsProtob\x06proto3') +) + +_PERIODSIZE = _descriptor.EnumDescriptor( + name='PeriodSize', + full_name='PeriodSize', + filename=None, + file=DESCRIPTOR, + values=[ + _descriptor.EnumValueDescriptor( + name='OneSecond', index=0, number=0, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='FiveSeconds', index=1, number=1, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='TenSeconds', index=2, number=2, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='FifteenSeconds', index=3, number=3, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='ThirtySeconds', index=4, number=4, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='OneMinute', index=5, number=5, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='FiveMinutes', index=6, number=6, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='TenMinutes', index=7, number=7, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='FifteenMinutes', index=8, number=8, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='ThirtyMinutes', index=9, number=9, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='OneHour', index=10, number=10, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='OneDay', index=11, number=11, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='OneWeek', index=12, number=12, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='OneMonth', index=13, number=13, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='OneYear', index=14, number=14, + options=None, + type=None), + ], + containing_type=None, + options=None, + serialized_start=766, + serialized_end=1017, +) +_sym_db.RegisterEnumDescriptor(_PERIODSIZE) + +PeriodSize = enum_type_wrapper.EnumTypeWrapper(_PERIODSIZE) +OneSecond = 0 +FiveSeconds = 1 +TenSeconds = 2 +FifteenSeconds = 3 +ThirtySeconds = 4 +OneMinute = 5 +FiveMinutes = 6 +TenMinutes = 7 +FifteenMinutes = 8 +ThirtyMinutes = 9 +OneHour = 10 +OneDay = 11 +OneWeek = 12 +OneMonth = 13 +OneYear = 14 + + + +_TEMPORALPERIOD = _descriptor.Descriptor( + name='TemporalPeriod', + full_name='TemporalPeriod', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='periodSize', full_name='TemporalPeriod.periodSize', index=0, + number=1, type=14, cpp_type=8, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='periodOffset', full_name='TemporalPeriod.periodOffset', index=1, + number=2, type=4, cpp_type=4, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=16, + serialized_end=87, +) + + +_WEEKLYCYCLE = _descriptor.Descriptor( + name='WeeklyCycle', + full_name='WeeklyCycle', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='year', full_name='WeeklyCycle.year', index=0, + number=1, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='month', full_name='WeeklyCycle.month', index=1, + number=2, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='day', full_name='WeeklyCycle.day', index=2, + number=3, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='periodSize', full_name='WeeklyCycle.periodSize', index=3, + number=4, type=14, cpp_type=8, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=89, + serialized_end=177, +) + + +_SPEEDHISTOGRAM = _descriptor.Descriptor( + name='SpeedHistogram', + full_name='SpeedHistogram', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='speedBin', full_name='SpeedHistogram.speedBin', index=0, + number=1, type=13, cpp_type=3, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='observationCount', full_name='SpeedHistogram.observationCount', index=1, + number=2, type=13, cpp_type=3, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=179, + serialized_end=239, +) + + +_SPEEDHISTOGRAMBYPERIOD = _descriptor.Descriptor( + name='SpeedHistogramByPeriod', + full_name='SpeedHistogramByPeriod', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='periodOffset', full_name='SpeedHistogramByPeriod.periodOffset', index=0, + number=1, type=13, cpp_type=3, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='histogram', full_name='SpeedHistogramByPeriod.histogram', index=1, + number=2, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=241, + serialized_end=323, +) + + +_SPEEDSUMMARY = _descriptor.Descriptor( + name='SpeedSummary', + full_name='SpeedSummary', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='meanSpead', full_name='SpeedSummary.meanSpead', index=0, + number=1, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='percentile', full_name='SpeedSummary.percentile', index=1, + number=2, type=13, cpp_type=3, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='observationCount', full_name='SpeedSummary.observationCount', index=2, + number=3, type=13, cpp_type=3, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=325, + serialized_end=404, +) + + +_SPEEDSUMMARYBYPERIOD = _descriptor.Descriptor( + name='SpeedSummaryByPeriod', + full_name='SpeedSummaryByPeriod', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='periodOffset', full_name='SpeedSummaryByPeriod.periodOffset', index=0, + number=1, type=13, cpp_type=3, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='speedSummary', full_name='SpeedSummaryByPeriod.speedSummary', index=1, + number=3, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=406, + serialized_end=487, +) + + +_SHAREDSTREETSWEEKLYSPEEDS = _descriptor.Descriptor( + name='SharedStreetsWeeklySpeeds', + full_name='SharedStreetsWeeklySpeeds', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='referenceId', full_name='SharedStreetsWeeklySpeeds.referenceId', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='periodSize', full_name='SharedStreetsWeeklySpeeds.periodSize', index=1, + number=2, type=14, cpp_type=8, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='scaledCounts', full_name='SharedStreetsWeeklySpeeds.scaledCounts', index=2, + number=3, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='referenceLength', full_name='SharedStreetsWeeklySpeeds.referenceLength', index=3, + number=4, type=4, cpp_type=4, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='numberOfBins', full_name='SharedStreetsWeeklySpeeds.numberOfBins', index=4, + number=5, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='binPosition', full_name='SharedStreetsWeeklySpeeds.binPosition', index=5, + number=6, type=13, cpp_type=3, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='speedsByPeriod', full_name='SharedStreetsWeeklySpeeds.speedsByPeriod', index=6, + number=7, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='speedSummaryByPeriod', full_name='SharedStreetsWeeklySpeeds.speedSummaryByPeriod', index=7, + number=8, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=490, + serialized_end=763, +) + +_TEMPORALPERIOD.fields_by_name['periodSize'].enum_type = _PERIODSIZE +_WEEKLYCYCLE.fields_by_name['periodSize'].enum_type = _PERIODSIZE +_SPEEDHISTOGRAMBYPERIOD.fields_by_name['histogram'].message_type = _SPEEDHISTOGRAM +_SPEEDSUMMARYBYPERIOD.fields_by_name['speedSummary'].message_type = _SPEEDSUMMARY +_SHAREDSTREETSWEEKLYSPEEDS.fields_by_name['periodSize'].enum_type = _PERIODSIZE +_SHAREDSTREETSWEEKLYSPEEDS.fields_by_name['speedsByPeriod'].message_type = _SPEEDHISTOGRAMBYPERIOD +_SHAREDSTREETSWEEKLYSPEEDS.fields_by_name['speedSummaryByPeriod'].message_type = _SPEEDSUMMARYBYPERIOD +DESCRIPTOR.message_types_by_name['TemporalPeriod'] = _TEMPORALPERIOD +DESCRIPTOR.message_types_by_name['WeeklyCycle'] = _WEEKLYCYCLE +DESCRIPTOR.message_types_by_name['SpeedHistogram'] = _SPEEDHISTOGRAM +DESCRIPTOR.message_types_by_name['SpeedHistogramByPeriod'] = _SPEEDHISTOGRAMBYPERIOD +DESCRIPTOR.message_types_by_name['SpeedSummary'] = _SPEEDSUMMARY +DESCRIPTOR.message_types_by_name['SpeedSummaryByPeriod'] = _SPEEDSUMMARYBYPERIOD +DESCRIPTOR.message_types_by_name['SharedStreetsWeeklySpeeds'] = _SHAREDSTREETSWEEKLYSPEEDS +DESCRIPTOR.enum_types_by_name['PeriodSize'] = _PERIODSIZE +_sym_db.RegisterFileDescriptor(DESCRIPTOR) + +TemporalPeriod = _reflection.GeneratedProtocolMessageType('TemporalPeriod', (_message.Message,), dict( + DESCRIPTOR = _TEMPORALPERIOD, + __module__ = 'speeds_pb2' + # @@protoc_insertion_point(class_scope:TemporalPeriod) + )) +_sym_db.RegisterMessage(TemporalPeriod) + +WeeklyCycle = _reflection.GeneratedProtocolMessageType('WeeklyCycle', (_message.Message,), dict( + DESCRIPTOR = _WEEKLYCYCLE, + __module__ = 'speeds_pb2' + # @@protoc_insertion_point(class_scope:WeeklyCycle) + )) +_sym_db.RegisterMessage(WeeklyCycle) + +SpeedHistogram = _reflection.GeneratedProtocolMessageType('SpeedHistogram', (_message.Message,), dict( + DESCRIPTOR = _SPEEDHISTOGRAM, + __module__ = 'speeds_pb2' + # @@protoc_insertion_point(class_scope:SpeedHistogram) + )) +_sym_db.RegisterMessage(SpeedHistogram) + +SpeedHistogramByPeriod = _reflection.GeneratedProtocolMessageType('SpeedHistogramByPeriod', (_message.Message,), dict( + DESCRIPTOR = _SPEEDHISTOGRAMBYPERIOD, + __module__ = 'speeds_pb2' + # @@protoc_insertion_point(class_scope:SpeedHistogramByPeriod) + )) +_sym_db.RegisterMessage(SpeedHistogramByPeriod) + +SpeedSummary = _reflection.GeneratedProtocolMessageType('SpeedSummary', (_message.Message,), dict( + DESCRIPTOR = _SPEEDSUMMARY, + __module__ = 'speeds_pb2' + # @@protoc_insertion_point(class_scope:SpeedSummary) + )) +_sym_db.RegisterMessage(SpeedSummary) + +SpeedSummaryByPeriod = _reflection.GeneratedProtocolMessageType('SpeedSummaryByPeriod', (_message.Message,), dict( + DESCRIPTOR = _SPEEDSUMMARYBYPERIOD, + __module__ = 'speeds_pb2' + # @@protoc_insertion_point(class_scope:SpeedSummaryByPeriod) + )) +_sym_db.RegisterMessage(SpeedSummaryByPeriod) + +SharedStreetsWeeklySpeeds = _reflection.GeneratedProtocolMessageType('SharedStreetsWeeklySpeeds', (_message.Message,), dict( + DESCRIPTOR = _SHAREDSTREETSWEEKLYSPEEDS, + __module__ = 'speeds_pb2' + # @@protoc_insertion_point(class_scope:SharedStreetsWeeklySpeeds) + )) +_sym_db.RegisterMessage(SharedStreetsWeeklySpeeds) + + +DESCRIPTOR.has_options = True +DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('B\030SharedStreetsSpeedsProto')) +# @@protoc_insertion_point(module_scope) diff --git a/sharedstreets/tests/test_linear_references.py b/sharedstreets/tests/test_linear_references.py new file mode 100644 index 0000000..b3cffaf --- /dev/null +++ b/sharedstreets/tests/test_linear_references.py @@ -0,0 +1,25 @@ +import unittest, mock, httmock, os, posixpath, ModestMaps.Geo +from .. import speeds_pb2 +from .. import linear_references + +def respond_locally(url, request): + path_parts = url.path.split(posixpath.sep) + local_path = os.path.join(os.path.dirname(__file__), 'data', *path_parts[1:]) + + if os.path.exists(local_path): + with open(local_path, 'rb') as file: + return httmock.response(200, file.read()) + + return httmock.response(404, 'Nope') + +class TestTile (unittest.TestCase): + + def test_linear_references(self): + + speedObservations = [] + + with open('12-946-1650.events.pbf', 'rb') as file: + fileContent = file.read() + observations = linear_references.flatten_binned_events(fileContent) + print(observations) + # self.assertEqual(len(observations), 124) diff --git a/sharedstreets/tests/test_speeds.py b/sharedstreets/tests/test_speeds.py new file mode 100644 index 0000000..cb24beb --- /dev/null +++ b/sharedstreets/tests/test_speeds.py @@ -0,0 +1,25 @@ +import unittest, mock, httmock, os, posixpath, ModestMaps.Geo +from .. import speeds_pb2 +from .. import speeds + +def respond_locally(url, request): + path_parts = url.path.split(posixpath.sep) + local_path = os.path.join(os.path.dirname(__file__), 'data', *path_parts[1:]) + + if os.path.exists(local_path): + with open(local_path, 'rb') as file: + return httmock.response(200, file.read()) + + return httmock.response(404, 'Nope') + +class TestTile (unittest.TestCase): + + def test_speeds(self): + + speedObservations = [] + + with open('12-946-1650.speeds.pbf', 'rb') as file: + fileContent = file.read() + observations = speeds.flatten_weekly_speed_histogram(fileContent) + print(observations) + self.assertEqual(len(observations), 124) diff --git a/sharedstreets/tile.py b/sharedstreets/tile.py index b0ff57c..f7f99b1 100644 --- a/sharedstreets/tile.py +++ b/sharedstreets/tile.py @@ -4,6 +4,7 @@ from . import sharedstreets_pb2 logger = logging.getLogger(__name__) +logging.basicConfig(level=logging.DEBUG) # https://github.com/sharedstreets/sharedstreets-ref-system/issues/16 DATA_URL_TEMPLATE, DATA_ZOOM = 'https://tiles.sharedstreets.io/osm/planet-180312/{z}-{x}-{y}.{layer}.6.pbf', 12 @@ -11,7 +12,7 @@ 'reference': sharedstreets_pb2.SharedStreetsReference, 'intersection': sharedstreets_pb2.SharedStreetsIntersection, 'geometry': sharedstreets_pb2.SharedStreetsGeometry, - 'metadata': sharedstreets_pb2.SharedStreetsMetadata, + 'metadata': sharedstreets_pb2.SharedStreetsMetadata } # Used for Mercator projection and tile space @@ -36,20 +37,13 @@ def round_coord(float): ''' return round(float, 7) -def iter_objects(url, DataClass): - ''' Generate a stream of objects from the protobuf URL. - ''' - response, position = requests.get(url), 0 - logger.debug('Got {} bytes: {}'.format(len(response.content), repr(response.content[:32]))) - if response.status_code not in range(200, 299): - logger.debug('Got HTTP {}'.format(response.status_code)) - return +def read_objects(position, content, DataClass): - while position < len(response.content): - message_length, new_position = _DecodeVarint32(response.content, position) + while position < len(content): + message_length, new_position = _DecodeVarint32(content, position) position = new_position - message = response.content[position:position+message_length] + message = content[position:position+message_length] position += message_length try: @@ -61,6 +55,35 @@ def iter_objects(url, DataClass): else: yield object +def iter_objects(url, DataClass): + ''' Generate a stream of objects from the protobuf URL. + ''' + response, position = requests.get(url), 0 + logger.debug('Got {} bytes: {}'.format(len(response.content), repr(response.content[:32]))) + + if response.status_code not in range(200, 299): + logger.debug('Got HTTP {}'.format(response.status_code)) + return + + for item in read_objects(position, response.content, DataClass): + yield item + + # while position < len(response.content): + # message_length, new_position = _DecodeVarint32(response.content, position) + # position = new_position + # message = response.content[position:position+message_length] + # position += message_length + + # try: + # object = DataClass() + # object.ParseFromString(message) + # except google.protobuf.message.DecodeError: + # # Empty tile? Shrug. + # continue + # else: + # yield object + + def is_inside(southwest, northeast, geometry): ''' Return True if the geometry bbox is inside a location pair bbox. ''' @@ -209,6 +232,8 @@ def reference_feature(reference, id_length): ] } + + def make_geojson(tile, id_length=32): ''' Get a GeoJSON dictionary for a geographic tile. From 14f3c05480f70e18c17e317efc7925842189b08e Mon Sep 17 00:00:00 2001 From: Kevin Webb Date: Mon, 30 Jul 2018 16:28:59 -0400 Subject: [PATCH 2/9] update README --- README.md | 17 +++++++++++++++++ sharedstreets/linear_references.py | 2 +- sharedstreets/tests/test_linear_references.py | 6 ++---- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 0641091..d4cb0ff 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,23 @@ Python implementation of [SharedStreets Reference System](https://github.com/sha gunicorn sharedstreets.webapp:app +- Read speed data from PBF encoded speed tiles. `flatten_weekly_speed_histogram` returns an array of arrays representing individual histogram bins with the structure `[sharedstreets_referenceId, period_hour_of_week, bin_kmh, observation_count`: + + import sharedstreets.speeds + with open('12-946-1650.speeds.pbf', 'rb') as file: + fileContent = file.read() + observations = speeds.flatten_weekly_speed_histogram(fileContent) + print(observations) + +- Read linear references from PBF encoded event tiles. `flatten_binned_events` returns an array of arrays representing individual event counts for linear bins `[sharedstreets_referenceId, referenceLength, numberOfBins, binPosition, dataType, observationCount, observationValue`: + + import sharedstreets.linear_references + with open('12-946-1650.events.pbf', 'rb') as file: + fileContent = file.read() + observations = linear_references.flatten_binned_events(fileContent) + print(observations) + self.assertEqual(len(observations), 89) + ## Develop Install for local development. diff --git a/sharedstreets/linear_references.py b/sharedstreets/linear_references.py index 47d4326..d95ffe7 100644 --- a/sharedstreets/linear_references.py +++ b/sharedstreets/linear_references.py @@ -21,6 +21,6 @@ def flatten_binned_events(tileContent): linearBin = item.binnedPeriodicData[binPos] for periodPos in range(0, len(linearBin.bins)): binData = linearBin.bins[periodPos] - binnedObservations.append([item.referenceId, item.numberOfBins, binPos+1, binData.dataType[0], binData.count[0], binData.value[0]]) + binnedObservations.append([item.referenceId, item.referenceLength/100, item.numberOfBins, binPos+1, binData.dataType[0], binData.count[0], binData.value[0]]) return binnedObservations; diff --git a/sharedstreets/tests/test_linear_references.py b/sharedstreets/tests/test_linear_references.py index b3cffaf..76b534d 100644 --- a/sharedstreets/tests/test_linear_references.py +++ b/sharedstreets/tests/test_linear_references.py @@ -15,11 +15,9 @@ def respond_locally(url, request): class TestTile (unittest.TestCase): def test_linear_references(self): - - speedObservations = [] - + with open('12-946-1650.events.pbf', 'rb') as file: fileContent = file.read() observations = linear_references.flatten_binned_events(fileContent) print(observations) - # self.assertEqual(len(observations), 124) + self.assertEqual(len(observations), 89) From 94f7bb073b19957d1fdc2c8d4059e0b12b7531fd Mon Sep 17 00:00:00 2001 From: Kevin Webb Date: Mon, 30 Jul 2018 17:00:08 -0400 Subject: [PATCH 3/9] update reaadme --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d4cb0ff..4b6c502 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ Python implementation of [SharedStreets Reference System](https://github.com/sha gunicorn sharedstreets.webapp:app -- Read speed data from PBF encoded speed tiles. `flatten_weekly_speed_histogram` returns an array of arrays representing individual histogram bins with the structure `[sharedstreets_referenceId, period_hour_of_week, bin_kmh, observation_count`: +- Read speed data from PBF encoded speed tiles. `flatten_weekly_speed_histogram` returns an array of arrays representing individual histogram bins with the structure `[sharedstreets_referenceId, period_hour_of_week, bin_kmh, observation_count]`: import sharedstreets.speeds with open('12-946-1650.speeds.pbf', 'rb') as file: @@ -38,7 +38,7 @@ Python implementation of [SharedStreets Reference System](https://github.com/sha observations = speeds.flatten_weekly_speed_histogram(fileContent) print(observations) -- Read linear references from PBF encoded event tiles. `flatten_binned_events` returns an array of arrays representing individual event counts for linear bins `[sharedstreets_referenceId, referenceLength, numberOfBins, binPosition, dataType, observationCount, observationValue`: +- Read linear references from PBF encoded event tiles. `flatten_binned_events` returns an array of arrays representing individual event counts for linear bins `[sharedstreets_referenceId, reference_length, number_of_bins, bin_position, data_type, observation_count, observation_value]`: import sharedstreets.linear_references with open('12-946-1650.events.pbf', 'rb') as file: From 3ede13c240256cc24021859971659ed54c4c7a18 Mon Sep 17 00:00:00 2001 From: Kevin Webb Date: Mon, 30 Jul 2018 17:09:15 -0400 Subject: [PATCH 4/9] Update README.md --- README.md | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 4b6c502..aee399b 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,5 @@ # SharedStreets (Python) -🚧WIP -- commits/comments welcome! 🚧 - Python implementation of [SharedStreets Reference System](https://github.com/sharedstreets/sharedstreets-ref-system). ## Install @@ -16,6 +14,8 @@ Python implementation of [SharedStreets Reference System](https://github.com/sha ## Use +#### GeoJSON Functions + - Retrieve a tile and convert to GeoJSON in Python. import sharedstreets.tile @@ -30,22 +30,25 @@ Python implementation of [SharedStreets Reference System](https://github.com/sha gunicorn sharedstreets.webapp:app -- Read speed data from PBF encoded speed tiles. `flatten_weekly_speed_histogram` returns an array of arrays representing individual histogram bins with the structure `[sharedstreets_referenceId, period_hour_of_week, bin_kmh, observation_count]`: +### Analysis functions + +#### Speed Data +Read speed data from PBF encoded speed tiles. `flatten_weekly_speed_histogram` returns an array of arrays representing individual histogram bins with the structure `[sharedstreets_referenceId, period_hour_of_week, bin_kmh, observation_count]`: import sharedstreets.speeds with open('12-946-1650.speeds.pbf', 'rb') as file: fileContent = file.read() observations = speeds.flatten_weekly_speed_histogram(fileContent) print(observations) - -- Read linear references from PBF encoded event tiles. `flatten_binned_events` returns an array of arrays representing individual event counts for linear bins `[sharedstreets_referenceId, reference_length, number_of_bins, bin_position, data_type, observation_count, observation_value]`: + +#### Binned Linear Data +Read linear references from PBF encoded event tiles. `flatten_binned_events` returns an array of arrays representing individual event counts for linear bins `[sharedstreets_referenceId, reference_length, number_of_bins, bin_position, data_type, observation_count, observation_value]`: import sharedstreets.linear_references with open('12-946-1650.events.pbf', 'rb') as file: fileContent = file.read() observations = linear_references.flatten_binned_events(fileContent) print(observations) - self.assertEqual(len(observations), 89) ## Develop From bc9335e9e7ea1c28051c0b2aa07d1b5e2bdfab89 Mon Sep 17 00:00:00 2001 From: Kevin Webb Date: Mon, 30 Jul 2018 17:09:43 -0400 Subject: [PATCH 5/9] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index aee399b..e7cb963 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ Python implementation of [SharedStreets Reference System](https://github.com/sha gunicorn sharedstreets.webapp:app -### Analysis functions +### Analysis Functions #### Speed Data Read speed data from PBF encoded speed tiles. `flatten_weekly_speed_histogram` returns an array of arrays representing individual histogram bins with the structure `[sharedstreets_referenceId, period_hour_of_week, bin_kmh, observation_count]`: From 736639aa377cb843ae2fa9570add7f3ab140e1af Mon Sep 17 00:00:00 2001 From: Kevin Webb Date: Mon, 13 Aug 2018 17:26:59 -0400 Subject: [PATCH 6/9] added linear referencing model --- sharedstreets/linear_references.py | 186 ++++++++++++++++-- sharedstreets/tests/test_linear_references.py | 32 ++- 2 files changed, 203 insertions(+), 15 deletions(-) diff --git a/sharedstreets/linear_references.py b/sharedstreets/linear_references.py index d95ffe7..9872ac1 100644 --- a/sharedstreets/linear_references.py +++ b/sharedstreets/linear_references.py @@ -1,26 +1,188 @@ from __future__ import print_function import argparse +import math +import copy from . import tile from . import linear_references_pb2 +from collections import defaultdict + protobuf_classes = [ linear_references_pb2.SharedStreetsWeeklyBinnedLinearReferences ] - parser = argparse.ArgumentParser(description='Read SharedStreets binned linear reference protobuf and convert to CSV') parser.add_argument('filename', help='Protobuf SharedStreets binned linear reference data') -def flatten_binned_events(tileContent): - binnedObservations = [] +MAX_SCALING_LCM_FACTOR = 10 # caps LCM at n * greater(x, y) + +def least_common_multiple(x, y): + # choose the greater number + if x > y: + greater = x + else: + greater = y + + max_factor = greater * MAX_SCALING_LCM_FACTOR + + while True: + + if greater >= max_factor: + lcm = greater + break + + if((greater % x == 0) and (greater % y == 0)): + lcm = greater + break + + greater += 1 + + return lcm + +def get_bin_count(ref_length, bin_size): + num_bins = math.floor(ref_length / bin_size) + 1 + +class BinnedLinearReference: + + def __init__(self, item): + self.reference_id = item.referenceId + + if item.scaledCounts: + self.scaled_counts = item.scaledCounts + else: + self.scaled_counts = False + + self.reference_length = float(item.referenceLength) / 100 + self.number_of_bins = item.numberOfBins + + if type(item) is linear_references_pb2.SharedStreetsWeeklyBinnedLinearReferences: + self.weeklyCylce = True + + # data indexed as a multi-dimentional array {dataType}[binPosition][periodOffset] + self.data = {} + + for bin_index in range(0, len(item.binPosition)): + bin_position = item.binPosition[bin_index] + linear_bin = item.binnedPeriodicData[bin_index] + for period_index in range(0, len(linear_bin.periodOffset)): + period_offset = linear_bin.periodOffset[period_index] + bin_data = linear_bin.bins[period_index] + for data_index in range(0, len(bin_data.dataType)): + data_type = bin_data.dataType[data_index] + data_count = bin_data.count[data_index] + data_value = bin_data.value[data_index] + + + self.data.setdefault(data_type, {}).setdefault(bin_position, {})[period_offset] = {"count": data_count, "value": data_value} + + def __str__(self): + return self.reference_id + ': ' + str(self.reference_length) + 'm / ' + str(self.number_of_bins) + ' bins -> ' + '{0:.3g}'.format(self.get_bin_length()) + ' m/bin' + + def get_bin_length(self): + return self.reference_length / self.number_of_bins + + def get_max_count(self): + max_count = 0 + + for data_type in self.data: + for bin_pos in self.data[data_type]: + for bin_period in self.data[data_type][bin_pos]: + if 'count' in self.data[data_type][bin_pos][bin_period] and self.data[data_type][bin_pos][bin_period]['count'] > max_count: + max_count = self.data[data_type][bin_pos]['count'] + + return max_count + + def scale_counts(self, max_count): + + if max_count == None: + max_count = self.get_max_count() + + self.scaled_counts = True + + for data_type in self.data: + for bin_pos in self.data[data_type]: + for bin_period in self.data[data_type][bin_pos]: + if 'count' in self.data[data_type][bin_pos][bin_period] and self.data[data_type][bin_pos][bin_period]['count'] > 0: + self.data[data_type][bin_pos]['count'] = float(self.data[data_type][bin_pos]['count']) / float(max_count) + + + # resize bins works only with scaled counts + def resize_bins(self, new_bin_length, max_count): + + # wip + return + + + def merge(self, new_data): + + if self.reference_id != new_data.reference_id: + raise ValueError('References IDs do not match') + + if self.nubmer_of_bins != new_data.nubmer_of_bins: + raise ValueError('Bin count does not match') + + for data_type in self.data: + for bin_pos in self.data[data_type]: + for bin_period in self.data[data_type][bin_pos]: + existing_count = self.data[data_type][bin_pos][bin_period]['count'] + existing_value = self.data[data_type][bin_pos][bin_period]['value'] + if data_type in new_data and bin_pos in new_data[data_type] and bin_period in new_data[data_type][bin_pos]: + new_count = new_data.data[data_type][bin_pos][bin_period]['count'] + new_value = new_data.data[data_type][bin_pos][bin_period]['value'] + + self.data[data_type][bin_pos][bin_period]['count'] = existing_count + new_count + self.data[data_type][bin_pos][bin_period]['value'] = existing_value + new_value + + + def filter(self, min_count): + + data_copy = copy.deepcopy(self.data) + + for data_type in self.data: + for bin_pos in self.data[data_type]: + for bin_period in self.data[data_type][bin_pos]: + if 'count' in self.data[data_type][bin_pos][bin_period] and self.data[data_type][bin_pos][bin_period]['count'] < min_count: + data_copy[data_type][bin_pos].pop(bin_period, None) + + if len(data_copy[data_type][bin_pos]) == 0: + data_copy[data_type][bin_pos].pop(bin_period, None) + if len(data_copy[data_type]) == 0: + data_copy[data_type].pop(bin_pos, None) + if len(data_copy) == 0: + data_copy.pop(data_type, None) + + self.data = data_copy + + def print(self): + + for data_type in self.data: + print(self.reference_id + ': ' + data_type) + for bin_pos in self.data[data_type]: + print('\t ' + str(bin_pos)) + for bin_period in self.data[data_type][bin_pos]: + print('\t\t ' + str(bin_period)) + for attribute in self.data[data_type][bin_pos][bin_period]: + print('\t\t\t ' + str(attribute) + ":" + str(self.data[data_type][bin_pos][bin_period][attribute])) + + + def flatten(self): + flattend_observations = [] + for data_type in self.data: + for bin_pos in self.data[data_type]: + for bin_period in self.data[data_type][bin_pos]: + count = self.data[data_type][bin_pos][bin_period]['count'] + value = self.data[data_type][bin_pos][bin_period]['value'] + flattened_observation = [self.reference_id, self.reference_length, self.number_of_bins, bin_pos, bin_period, data_type, count, value] + flattend_observations.append(flattened_observation) + + return flattend_observations + + +def load_binned_events(tileContent): + binned_observations = [] - count = 0; for item in tile.read_objects(0, tileContent, linear_references_pb2.SharedStreetsWeeklyBinnedLinearReferences): - count += 1 - for binPos in range(0, len(item.binPosition)): - linearBin = item.binnedPeriodicData[binPos] - for periodPos in range(0, len(linearBin.bins)): - binData = linearBin.bins[periodPos] - binnedObservations.append([item.referenceId, item.referenceLength/100, item.numberOfBins, binPos+1, binData.dataType[0], binData.count[0], binData.value[0]]) - - return binnedObservations; + binned_reference = BinnedLinearReference(item) + binned_observations.append(binned_reference) + + return binned_observations diff --git a/sharedstreets/tests/test_linear_references.py b/sharedstreets/tests/test_linear_references.py index 76b534d..c8ed7b9 100644 --- a/sharedstreets/tests/test_linear_references.py +++ b/sharedstreets/tests/test_linear_references.py @@ -18,6 +18,32 @@ def test_linear_references(self): with open('12-946-1650.events.pbf', 'rb') as file: fileContent = file.read() - observations = linear_references.flatten_binned_events(fileContent) - print(observations) - self.assertEqual(len(observations), 89) + observations = linear_references.load_binned_events(fileContent) + + self.assertEqual(len(observations), 31) + + flattened_count = 0 + for observation in observations: + for item in observation.flatten(): + flattened_count += 1 + print(item) + + self.assertEqual(flattened_count, 89) + + flattened_count = 0 + for observation in observations: + observation.filter(1) + for item in observation.flatten(): + flattened_count += 1 + print(item) + + self.assertEqual(flattened_count, 89) + + flattened_count = 0 + for observation in observations: + observation.filter(2) + for item in observation.flatten(): + flattened_count += 1 + print(item) + + self.assertEqual(flattened_count, 0) \ No newline at end of file From 21569f69df5b124cdb073d57943f698a7b4d8b74 Mon Sep 17 00:00:00 2001 From: Kevin Webb Date: Mon, 13 Aug 2018 17:32:03 -0400 Subject: [PATCH 7/9] adding readme for linear references --- README.md | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e7cb963..bcc2c52 100644 --- a/README.md +++ b/README.md @@ -47,8 +47,21 @@ Read linear references from PBF encoded event tiles. `flatten_binned_events` ret import sharedstreets.linear_references with open('12-946-1650.events.pbf', 'rb') as file: fileContent = file.read() - observations = linear_references.flatten_binned_events(fileContent) - print(observations) + observations = linear_references.load_binned_events(fileContent) + + # shows all items as flattened arrays + for observation in observations: + for flattened_observation in observation.flatten(): + print(flattened_observation) + + # shows only items passing the filter criteria (e.g. min of 2 counts per bin/time period) + for observation in observations: + observation.filter(2) # filter by min count per bin/time period + for flattened_observation in observation.flatten(): + print(flattened_observation) + + + ## Develop From b5668c23f53e231fdbb8eec3bd592808bedc889b Mon Sep 17 00:00:00 2001 From: Kevin Webb Date: Tue, 14 Aug 2018 16:50:17 -0400 Subject: [PATCH 8/9] merged pbf ouput functions --- sharedstreets/linear_references.py | 62 ++++++++++++++++++- sharedstreets/tests/test_linear_references.py | 22 ++++++- 2 files changed, 80 insertions(+), 4 deletions(-) diff --git a/sharedstreets/linear_references.py b/sharedstreets/linear_references.py index 9872ac1..1a416e4 100644 --- a/sharedstreets/linear_references.py +++ b/sharedstreets/linear_references.py @@ -4,9 +4,11 @@ import copy from . import tile from . import linear_references_pb2 +from google.protobuf.internal.encoder import _VarintEncoder from collections import defaultdict + protobuf_classes = [ linear_references_pb2.SharedStreetsWeeklyBinnedLinearReferences ] @@ -62,12 +64,17 @@ def __init__(self, item): self.data = {} for bin_index in range(0, len(item.binPosition)): + bin_position = item.binPosition[bin_index] linear_bin = item.binnedPeriodicData[bin_index] + for period_index in range(0, len(linear_bin.periodOffset)): + period_offset = linear_bin.periodOffset[period_index] bin_data = linear_bin.bins[period_index] + for data_index in range(0, len(bin_data.dataType)): + data_type = bin_data.dataType[data_index] data_count = bin_data.count[data_index] data_value = bin_data.value[data_index] @@ -109,7 +116,8 @@ def scale_counts(self, max_count): # resize bins works only with scaled counts def resize_bins(self, new_bin_length, max_count): - # wip + data_copy = copy.deepcopy(self.data) + return @@ -124,9 +132,11 @@ def merge(self, new_data): for data_type in self.data: for bin_pos in self.data[data_type]: for bin_period in self.data[data_type][bin_pos]: + existing_count = self.data[data_type][bin_pos][bin_period]['count'] existing_value = self.data[data_type][bin_pos][bin_period]['value'] if data_type in new_data and bin_pos in new_data[data_type] and bin_period in new_data[data_type][bin_pos]: + new_count = new_data.data[data_type][bin_pos][bin_period]['count'] new_value = new_data.data[data_type][bin_pos][bin_period]['value'] @@ -164,6 +174,43 @@ def print(self): for attribute in self.data[data_type][bin_pos][bin_period]: print('\t\t\t ' + str(attribute) + ":" + str(self.data[data_type][bin_pos][bin_period][attribute])) + def toPbf(self): + + pbfBinnedLinearReference = linear_references_pb2.SharedStreetsWeeklyBinnedLinearReferences() + + pbfBinnedLinearReference.referenceId = self.reference_id + pbfBinnedLinearReference.referenceLength = int(round(self.reference_length * 100)) + pbfBinnedLinearReference.numberOfBins = self.number_of_bins + pbfBinnedLinearReference.scaledCounts = self.scaled_counts + + pbfData = {} + + # pivot data to pbf format + for data_type in self.data: + for bin_pos in self.data[data_type]: + for bin_period in self.data[data_type][bin_pos]: + + data_count = self.data[data_type][bin_pos][bin_period]['count'] + data_value = self.data[data_type][bin_pos][bin_period]['value'] + + # TODO handle multi data-type writes + pbfData.setdefault(bin_pos, {})[bin_period] = {"data_type": data_type, "count": data_count, "value": data_value} + + # write data to pbf + for bin_pos in pbfData: + bin_position = pbfBinnedLinearReference.binPosition.append(bin_pos) + bin_position_data = pbfBinnedLinearReference.binnedPeriodicData.add() + for bin_period in pbfData[bin_pos]: + bin_position_data.periodOffset.append(bin_period) + bin_period_data = bin_position_data.bins.add() + + bin_period_data.dataType.append(pbfData[bin_pos][bin_period]['data_type']) + bin_period_data.count.append(pbfData[bin_pos][bin_period]['count']) + bin_period_data.value.append(pbfData[bin_pos][bin_period]['value']) + + + return pbfBinnedLinearReference.SerializeToString() + def flatten(self): flattend_observations = [] @@ -178,6 +225,19 @@ def flatten(self): return flattend_observations +# generates a length encoded stream of pbf +def generate_pbf(output_file, binned_observations): + + for binned_reference in binned_observations: + + pbf_data = binned_reference.toPbf() + size_of_data = len(pbf_data) + _VarintEncoder()(output_file.write, size_of_data, True) + output_file.write(pbf_data) + #output.append(pbf_data) + + + def load_binned_events(tileContent): binned_observations = [] diff --git a/sharedstreets/tests/test_linear_references.py b/sharedstreets/tests/test_linear_references.py index c8ed7b9..50a14fa 100644 --- a/sharedstreets/tests/test_linear_references.py +++ b/sharedstreets/tests/test_linear_references.py @@ -15,13 +15,29 @@ def respond_locally(url, request): class TestTile (unittest.TestCase): def test_linear_references(self): - - with open('12-946-1650.events.pbf', 'rb') as file: + filename = '12-946-1650.events.pbf' + with open(filename, 'rb') as file: fileContent = file.read() observations = linear_references.load_binned_events(fileContent) - self.assertEqual(len(observations), 31) + test_output_filename = '__test__' + filename + try: + os.remove(test_output_filename) + except OSError: + pass + + newFile = open(test_output_filename, "wb") + linear_references.generate_pbf(newFile, observations) + newFile.close() + + with open(test_output_filename, 'rb') as file: + fileContent = file.read() + test_observations = linear_references.load_binned_events(fileContent) + self.assertEqual(len(test_observations), 31) + + + flattened_count = 0 for observation in observations: for item in observation.flatten(): From 9801f1fcfed0dfd299a3be58ae1dfc98f095c72f Mon Sep 17 00:00:00 2001 From: Kevin Webb Date: Mon, 20 Aug 2018 18:24:45 -0400 Subject: [PATCH 9/9] bump osm tiles to latest planet build --- sharedstreets/tile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sharedstreets/tile.py b/sharedstreets/tile.py index f7f99b1..f9c509b 100644 --- a/sharedstreets/tile.py +++ b/sharedstreets/tile.py @@ -7,7 +7,7 @@ logging.basicConfig(level=logging.DEBUG) # https://github.com/sharedstreets/sharedstreets-ref-system/issues/16 -DATA_URL_TEMPLATE, DATA_ZOOM = 'https://tiles.sharedstreets.io/osm/planet-180312/{z}-{x}-{y}.{layer}.6.pbf', 12 +DATA_URL_TEMPLATE, DATA_ZOOM = 'https://tiles.sharedstreets.io/osm/planet-180430/{z}-{x}-{y}.{layer}.6.pbf', 12 data_classes = { 'reference': sharedstreets_pb2.SharedStreetsReference, 'intersection': sharedstreets_pb2.SharedStreetsIntersection,