diff --git a/metroextractor/tests.py b/metroextractor/tests.py index 5686861..d0f0599 100644 --- a/metroextractor/tests.py +++ b/metroextractor/tests.py @@ -1,6 +1,7 @@ -import unittest, unittest.mock +import unittest, unittest.mock, httmock +from urllib.parse import parse_qsl -from . import nothingburger +from . import nothingburger, wof_bundle class Nothingburger (unittest.TestCase): @@ -11,3 +12,100 @@ def test_tell_us(self): io_open.assert_called_once_with('filename.txt', 'w') write_call = io_open.return_value.__enter__.return_value.mock_calls[0] self.assertTrue(write_call[1][0].startswith('[0, 1, 2, 3]')) + +class WofBundle (unittest.TestCase): + + def test_iterate_places_type(self): + def response_content(url, request): + query = dict(parse_qsl(url.query)) + + if url.hostname == 'whosonfirst-api.dev.mapzen.com' and query['method'] == 'whosonfirst.places.getIntersects': + if query.get('cursor', '0') == '0': + content = '{"results":[{"wof:id":554753227,"wof:parent_id":85921881,"wof:name":"94610","wof:placetype":"postalcode","wof:country":"US","wof:repo":"whosonfirst-data-postalcode-us","mz:uri":"https:\/\/whosonfirst.mapzen.com\/data\/554\/753\/227\/554753227.geojson","wof:path":"554\/753\/227\/554753227.geojson"}],"cursor":1,"stat":"ok"}' + return httmock.response(200, content, headers={'Content-Type': 'text/json'}) + if query.get('cursor', '0') == '1': + content = '{"results":[{"wof:id":554753229,"wof:parent_id":85921881,"wof:name":"94611","wof:placetype":"postalcode","wof:country":"US","wof:repo":"whosonfirst-data-postalcode-us","mz:uri":"https:\/\/whosonfirst.mapzen.com\/data\/554\/753\/229\/554753229.geojson","wof:path":"554\/753\/229\/554753229.geojson"}],"cursor":0,"stat":"ok"}' + return httmock.response(200, content, headers={'Content-Type': 'text/json'}) + + raise Exception('Unknown URL') + + with httmock.HTTMock(response_content): + bbox = -180, -90, 180, 90 + places = list(wof_bundle.iterate_places_type(bbox, 'postalcode')) + + self.assertEqual(len(places), 2) + self.assertEqual(places[0]['wof:id'], 554753227) + self.assertEqual(places[1]['wof:id'], 554753229) + + def test_add_tar_place(self): + def response_content(url, request): + if url.hostname == 'whosonfirst.mapzen.com' and url.path == '/data/554/753/227/554753227.geojson': + content = '''{ + "id": 554753227, + "type": "Feature", + "properties": { + "edtf:cessation":"uuuu", + "edtf:inception":"uuuu", + "geom:area":0.00064, + "geom:bbox":"-122.264364,37.798735,-122.215174,37.823831", + "geom:latitude":37.811405, + "geom:longitude":-122.242396, + "gp:parent_id":"2463583", + "iso:country":"US", + "lbl:latitude":37.811406, + "lbl:longitude":-122.242458, + "mz:hierarchy_label":1, + "src:geom":"uszcta", + "wof:belongsto":[ + 85688637, + 102191575, + 85633793, + 85921881, + 102086959 + ], + "wof:breaches":[], + "wof:concordances":{ + "gp:id":"12797397" + }, + "wof:country":"US", + "wof:geomhash":"bc1121d80cf8c9d1346af6838b94cf06", + "wof:hierarchy":[ + { + "continent_id":102191575, + "country_id":85633793, + "county_id":102086959, + "locality_id":85921881, + "postalcode_id":554753227, + "region_id":85688637 + } + ], + "wof:id":554753227, + "wof:lastmodified":1470248660, + "wof:name":"94610", + "wof:parent_id":85921881, + "wof:placetype":"postalcode", + "wof:repo":"whosonfirst-data-postalcode-us", + "wof:superseded_by":[], + "wof:supersedes":[], + "wof:tags":[] +}, + "bbox": [ + -122.264364, + 37.798735, + -122.215174, + 37.823831 +], + "geometry": {"coordinates":[[[-122.25527,37.81732],[-122.255628,37.817603],[-122.25557000000001,37.817643],[-122.255399,37.817765],[-122.255342,37.817806],[-122.254155,37.818646],[-122.254042,37.818731],[-122.25402699999999,37.818743],[-122.254131,37.818873],[-122.254186,37.818964],[-122.254197,37.818981],[-122.254212,37.819013],[-122.254242,37.819077],[-122.25426299999999,37.819183],[-122.25433700000001,37.819515],[-122.254423,37.819581],[-122.254446,37.819599],[-122.254479,37.819625],[-122.254499,37.819771],[-122.25448900000001,37.819804],[-122.254409,37.820056],[-122.254451,37.820095],[-122.255399,37.8209],[-122.255235,37.82103],[-122.254744,37.821422],[-122.254727,37.821437],[-122.254643,37.821497],[-122.254566,37.821528],[-122.25452900000001,37.821417],[-122.254499,37.821292],[-122.25433200000001,37.820577],[-122.254277,37.820339],[-122.254132,37.819833],[-122.25403300000001,37.819729],[-122.254003,37.819701],[-122.253598,37.819326],[-122.25346399999999,37.819202],[-122.253404,37.819148],[-122.253089,37.819364],[-122.252416,37.819831],[-122.251884,37.820184],[-122.25172499999999,37.820257],[-122.25139299999999,37.820368],[-122.251257,37.82025],[-122.25116,37.820166],[-122.250665,37.819734],[-122.250468,37.819562],[-122.25014899999999,37.819835],[-122.249619,37.820291],[-122.24924300000001,37.820586],[-122.24918599999999,37.820646],[-122.249162,37.820673],[-122.249121,37.820748],[-122.24910199999999,37.82081],[-122.249098,37.820824],[-122.249095,37.820961],[-122.2491,37.821043],[-122.24840399999999,37.821259],[-122.24841000000001,37.821271],[-122.248431,37.821309],[-122.24843799999999,37.821322],[-122.248244,37.821377],[-122.24821,37.821387],[-122.24766200000001,37.821543],[-122.24760499999999,37.82156],[-122.247573,37.821569],[-122.247502,37.821526],[-122.247398,37.821457],[-122.24666999999999,37.820848],[-122.246486,37.820694],[-122.24645599999999,37.820704],[-122.246421,37.820645],[-122.24639500000001,37.820602],[-122.24632200000001,37.820499],[-122.246123,37.820215],[-122.245992,37.820079],[-122.245841,37.819942],[-122.245788,37.819901],[-122.24575299999999,37.819874],[-122.245684,37.819835],[-122.245643,37.819813],[-122.245604,37.81981],[-122.24557900000001,37.819819],[-122.245554,37.819838],[-122.245532,37.819855],[-122.245474,37.819929],[-122.245418,37.820029],[-122.245413,37.820039],[-122.245364,37.820144],[-122.24530900000001,37.820313],[-122.245272,37.820464],[-122.245244,37.82061],[-122.245238,37.820711],[-122.24523499999999,37.820779],[-122.245249,37.820948],[-122.24516199999999,37.820957],[-122.245059,37.820949],[-122.245034,37.820944],[-122.244783,37.820898],[-122.244399,37.820821],[-122.24418799999999,37.820779],[-122.24413199999999,37.820941],[-122.244079,37.821096],[-122.243976,37.821431],[-122.243927,37.821595],[-122.243843,37.821834],[-122.243593,37.822554],[-122.24354099999999,37.822708],[-122.243514,37.822796],[-122.243424,37.822782],[-122.2433,37.822818],[-122.242822,37.82296],[-122.24266799999999,37.823005],[-122.242458,37.823069],[-122.242181,37.823155],[-122.24135,37.823412],[-122.241073,37.823499],[-122.24090099999999,37.823551],[-122.240416,37.823699],[-122.240385,37.823709],[-122.24021399999999,37.823762],[-122.240109,37.823799],[-122.24002,37.823831],[-122.239755,37.823531],[-122.23898699999999,37.823463],[-122.237725,37.82241],[-122.23748999999999,37.82241],[-122.237211,37.822215],[-122.23741800000001,37.822069],[-122.237543,37.821982],[-122.237453,37.821952],[-122.237279,37.821783],[-122.237162,37.821669],[-122.237076,37.821644],[-122.236999,37.82165],[-122.23636,37.821884],[-122.23604400000001,37.822],[-122.236006,37.821945],[-122.235895,37.821782],[-122.23585799999999,37.821728],[-122.23460300000001,37.819992],[-122.23444499999999,37.820074],[-122.23436,37.820107],[-122.234297,37.820118],[-122.234137,37.820132],[-122.234025,37.820114],[-122.23392200000001,37.820077],[-122.23386600000001,37.820032],[-122.23387,37.81996],[-122.233856,37.819905],[-122.233512,37.819743],[-122.233007,37.819238],[-122.23193000000001,37.818795],[-122.231703,37.817482],[-122.23106900000001,37.817091],[-122.228887,37.816526],[-122.228061,37.816665],[-122.228094,37.817092],[-122.228033,37.817112],[-122.227755,37.817234],[-122.227671,37.817335],[-122.227557,37.817429],[-122.227538,37.817439],[-122.227469,37.81748],[-122.22734800000001,37.817531],[-122.22722,37.817548],[-122.22709999999999,37.817536],[-122.22698699999999,37.817479],[-122.22680099999999,37.81737],[-122.226719,37.817316],[-122.226674,37.817286],[-122.226561,37.817231],[-122.22644699999999,37.817194],[-122.22634100000001,37.817157],[-122.22623,37.817125],[-122.226213,37.817122],[-122.226141,37.817109],[-122.22601,37.817099],[-122.225481,37.817089],[-122.22523700000001,37.817085],[-122.225229,37.817387],[-122.225218,37.817896],[-122.22522499999999,37.818293],[-122.22523200000001,37.818596],[-122.22523,37.818655],[-122.22522600000001,37.818836],[-122.22522499999999,37.818896],[-122.22498,37.81875],[-122.224245,37.818314],[-122.224,37.818169],[-122.223795,37.818025],[-122.223635,37.817914],[-122.22339700000001,37.81778],[-122.22319,37.817731],[-122.223122,37.817721],[-122.223004,37.817705],[-122.222874,37.817702],[-122.222657,37.817706],[-122.222533,37.817709],[-122.22242900000001,37.817719],[-122.22227599999999,37.81775],[-122.222032,37.817855],[-122.221833,37.817941],[-122.22163500000001,37.818057],[-122.221086,37.818382],[-122.221041,37.818409],[-122.22098200000001,37.818447],[-122.220833,37.818504],[-122.22071,37.818264],[-122.22068400000001,37.818213],[-122.220613,37.818091],[-122.220511,37.81795],[-122.22030599999999,37.817737],[-122.22018799999999,37.81766],[-122.21996300000001,37.817514],[-122.220051,37.817388],[-122.2201,37.817278],[-122.22019899999999,37.817007],[-122.22022800000001,37.81693],[-122.220308,37.816527],[-122.22034499999999,37.816313],[-122.22039599999999,37.816075],[-122.22049,37.815967],[-122.22057700000001,37.815909],[-122.220681,37.815854],[-122.22103,37.815743],[-122.221107,37.815719],[-122.221475,37.815652],[-122.220536,37.813807],[-122.22018300000001,37.813692],[-122.218515,37.814052],[-122.217703,37.813793],[-122.216531,37.813067],[-122.216677,37.812829],[-122.216521,37.812776],[-122.21625,37.812677],[-122.21616400000001,37.812646],[-122.215945,37.812543],[-122.215445,37.81237],[-122.215174,37.812276],[-122.215204,37.812217],[-122.21529700000001,37.812042],[-122.215329,37.811984],[-122.215571,37.812051],[-122.215827,37.812113],[-122.21587599999999,37.812126],[-122.216138,37.812201],[-122.21675,37.812427],[-122.217069,37.812536],[-122.21724,37.812552],[-122.217291,37.812583],[-122.217365,37.812629],[-122.217765,37.812783],[-122.217755,37.812824],[-122.217753,37.812835],[-122.217727,37.812946],[-122.217718,37.812988],[-122.218206,37.813096],[-122.21966999999999,37.813421],[-122.21974400000001,37.813437],[-122.219885,37.813333],[-122.221068,37.812464],[-122.221135,37.811495],[-122.222627,37.810175],[-122.223919,37.809891],[-122.224493,37.809546],[-122.223106,37.80807],[-122.22327900000001,37.807903],[-122.223451,37.808059],[-122.223584,37.808016],[-122.223533,37.807975],[-122.223516,37.807927],[-122.223518,37.807912],[-122.223522,37.807891],[-122.22353699999999,37.807877],[-122.22380200000001,37.807749],[-122.22401000000001,37.807923],[-122.224326,37.808187],[-122.224397,37.808261],[-122.224538,37.808409],[-122.22456699999999,37.808457],[-122.22459000000001,37.808482],[-122.224627,37.808489],[-122.22466300000001,37.808482],[-122.224712,37.808448],[-122.224954,37.808288],[-122.22503500000001,37.808235],[-122.225674,37.807743],[-122.225723,37.807773],[-122.226108,37.807502],[-122.22600199999999,37.807352],[-122.2274,37.8064],[-122.22899,37.805697],[-122.22908700000001,37.805208],[-122.228799,37.804835],[-122.22862600000001,37.804929],[-122.22860300000001,37.804857],[-122.228571,37.804778],[-122.2285,37.804649],[-122.228892,37.804571],[-122.228842,37.804437],[-122.22904200000001,37.804288],[-122.228933,37.804256],[-122.228724,37.804213],[-122.228499,37.804167],[-122.22776500000001,37.804016],[-122.227446,37.803951],[-122.227456,37.80387],[-122.22747099999999,37.803818],[-122.227588,37.80343],[-122.227627,37.803301],[-122.227661,37.803175],[-122.227711,37.802998],[-122.22776500000001,37.80282],[-122.227773,37.802799],[-122.227805,37.80272],[-122.227829,37.802682],[-122.22802900000001,37.802718],[-122.22863,37.802828],[-122.228831,37.802865],[-122.229125,37.802928],[-122.22956499999999,37.803024],[-122.229767,37.803082],[-122.229924,37.803135],[-122.22999299999999,37.80317],[-122.23003,37.803189],[-122.230104,37.803233],[-122.230249,37.803329],[-122.23035900000001,37.803244],[-122.230692,37.80299],[-122.23080299999999,37.802906],[-122.23087599999999,37.802845],[-122.23109599999999,37.802665],[-122.23117000000001,37.802605],[-122.231233,37.802556],[-122.231257,37.802536],[-122.231331,37.802478],[-122.23152,37.802331],[-122.23160900000001,37.802264],[-122.23235200000001,37.801683],[-122.233082,37.801109],[-122.233361,37.80089],[-122.233791,37.800539],[-122.234679,37.799827],[-122.234669,37.799864],[-122.234824,37.799743],[-122.234886,37.799696],[-122.235229,37.799419],[-122.235281,37.799359],[-122.235394,37.799208],[-122.235427,37.799165],[-122.23552599999999,37.799037],[-122.23556000000001,37.798995],[-122.235665,37.79908],[-122.235983,37.799336],[-122.23608900000001,37.799422],[-122.236214,37.799537],[-122.236592,37.799883],[-122.236718,37.799999],[-122.23688300000001,37.800144],[-122.23737800000001,37.800581],[-122.237543,37.800727],[-122.237708,37.800877],[-122.23820600000001,37.801327],[-122.23831300000001,37.801424],[-122.238328,37.801469],[-122.238328,37.801501],[-122.238315,37.801512],[-122.23828,37.801545],[-122.23826800000001,37.801557],[-122.23828,37.801781],[-122.238259,37.801973],[-122.23823299999999,37.802226],[-122.23821100000001,37.802294],[-122.238174,37.802358],[-122.238123,37.802416],[-122.238061,37.802467],[-122.237989,37.802508],[-122.23759099999999,37.802664],[-122.23750699999999,37.802712],[-122.23743399999999,37.802769],[-122.237359,37.802853],[-122.237284,37.802973],[-122.23724900000001,37.803058],[-122.237607,37.803017],[-122.23769299999999,37.803013],[-122.237807,37.803],[-122.23808099999999,37.802944],[-122.238158,37.802913],[-122.238294,37.802866],[-122.23847499999999,37.802818],[-122.238821,37.802755],[-122.239746,37.802569],[-122.239791,37.802621],[-122.23993900000001,37.802761],[-122.240073,37.802887],[-122.24041,37.803154],[-122.240571,37.803282],[-122.240562,37.803465],[-122.240559,37.803543],[-122.24049100000001,37.803746],[-122.240442,37.803838],[-122.240375,37.80393],[-122.24035000000001,37.80396],[-122.240308,37.804012],[-122.240229,37.804098],[-122.24015900000001,37.804158],[-122.239996,37.804299],[-122.23994999999999,37.804341],[-122.23988300000001,37.804404],[-122.240082,37.804468],[-122.24028199999999,37.804533],[-122.24067599999999,37.804673],[-122.24087400000001,37.804744],[-122.24108,37.804813],[-122.24169999999999,37.805023],[-122.241907,37.805093],[-122.24200500000001,37.805123],[-122.242299,37.805214],[-122.242397,37.805245],[-122.24260700000001,37.805316],[-122.243239,37.80553],[-122.243258,37.805537],[-122.243335,37.805573],[-122.243424,37.80565],[-122.243679,37.805506],[-122.243714,37.805483],[-122.243966,37.805325],[-122.244265,37.805133],[-122.24437399999999,37.805048],[-122.244534,37.804904],[-122.24458300000001,37.80486],[-122.244776,37.804672],[-122.244907,37.804712],[-122.245013,37.804746],[-122.24511099999999,37.804779],[-122.24529800000001,37.804842],[-122.245429,37.804886],[-122.24555599999999,37.804928],[-122.24594,37.805054],[-122.24606799999999,37.805097],[-122.245914,37.805322],[-122.245867,37.805393],[-122.245434,37.805983],[-122.245273,37.806203],[-122.245385,37.806284],[-122.245482,37.806376],[-122.245604,37.806521],[-122.245654,37.8066],[-122.245722,37.806722],[-122.24575,37.806774],[-122.24576500000001,37.806837],[-122.245774,37.806925],[-122.24577499999999,37.807024],[-122.245752,37.807137],[-122.245717,37.807229],[-122.245661,37.807337],[-122.245645,37.807397],[-122.24565,37.807458],[-122.245671,37.80753],[-122.245701,37.807601],[-122.245769,37.807707],[-122.24602899999999,37.807386],[-122.24612999999999,37.807257],[-122.24654,37.806736],[-122.246612,37.806664],[-122.24669900000001,37.806586],[-122.246849,37.806468],[-122.247184,37.80619],[-122.24735699999999,37.806051],[-122.24780800000001,37.805692],[-122.247848,37.8057],[-122.247968,37.805724],[-122.248009,37.805732],[-122.24803,37.805685],[-122.24809399999999,37.805544],[-122.248116,37.805498],[-122.248195,37.805444],[-122.24843300000001,37.805285],[-122.248513,37.805232],[-122.24867500000001,37.805153],[-122.248869,37.805059],[-122.249156,37.804902],[-122.249229,37.804863],[-122.249318,37.804823],[-122.24937799999999,37.804844],[-122.249528,37.804896],[-122.249561,37.8049],[-122.24962499999999,37.80491],[-122.249511,37.805142],[-122.249171,37.805838],[-122.24914699999999,37.805888],[-122.24905200000001,37.806067],[-122.249246,37.806123],[-122.249543,37.806208],[-122.24982799999999,37.806291],[-122.250022,37.806348],[-122.249492,37.807532],[-122.249578,37.807558],[-122.249501,37.80773],[-122.249382,37.808],[-122.24930000000001,37.808171],[-122.249272,37.80822],[-122.24926000000001,37.80824],[-122.249253,37.808254],[-122.24914800000001,37.808391],[-122.24961399999999,37.808346],[-122.24957999999999,37.80827],[-122.24958700000001,37.80821],[-122.249601,37.808144],[-122.24960799999999,37.808034],[-122.249691,37.80771],[-122.24979399999999,37.807567],[-122.249877,37.807424],[-122.24998100000001,37.807127],[-122.250092,37.806946],[-122.25015399999999,37.806759],[-122.25027799999999,37.806479],[-122.250409,37.806133],[-122.250451,37.806017],[-122.25045799999999,37.805863],[-122.250513,37.805704],[-122.25058900000001,37.80555],[-122.25064399999999,37.805457],[-122.250755,37.805314],[-122.250873,37.805105],[-122.25097599999999,37.804946],[-122.251052,37.804874],[-122.25108,37.804835],[-122.25115599999999,37.804807],[-122.25128100000001,37.804708],[-122.251413,37.804637],[-122.251544,37.804615],[-122.251676,37.804626],[-122.251752,37.80461],[-122.25187,37.804555],[-122.25205699999999,37.804445],[-122.252182,37.80439],[-122.25232,37.804357],[-122.252583,37.804346],[-122.25286699999999,37.804281],[-122.253117,37.804209],[-122.25331799999999,37.804072],[-122.253428,37.803951],[-122.253595,37.803638],[-122.25383100000001,37.803171],[-122.253969,37.802924],[-122.254122,37.80271],[-122.25433700000001,37.802545],[-122.25454499999999,37.802298],[-122.254628,37.802128],[-122.254642,37.801913],[-122.254538,37.801414],[-122.254518,37.801199],[-122.254608,37.800776],[-122.254705,37.800617],[-122.25487800000001,37.80048],[-122.25508600000001,37.800337],[-122.255363,37.800222],[-122.255578,37.800189],[-122.255827,37.800205],[-122.256028,37.800249],[-122.256388,37.800261],[-122.25663,37.800255],[-122.25706,37.800173],[-122.25730900000001,37.80008],[-122.257676,37.799965],[-122.257946,37.799822],[-122.25827200000001,37.799613],[-122.25853499999999,37.799394],[-122.25886800000001,37.799059],[-122.259013,37.798844],[-122.259083,37.798757],[-122.259214,37.798735],[-122.259311,37.798757],[-122.25934599999999,37.798757],[-122.25946999999999,37.798828],[-122.259671,37.798922],[-122.259801,37.798979],[-122.26009999999999,37.799109],[-122.260626,37.799345],[-122.260869,37.799466],[-122.26112500000001,37.799615],[-122.261304,37.799725],[-122.261374,37.799812],[-122.261436,37.799933],[-122.26145699999999,37.800049],[-122.261456,37.800225],[-122.261436,37.800389],[-122.26142900000001,37.800543],[-122.26146300000001,37.800851],[-122.261428,37.800977],[-122.26134500000001,37.801109],[-122.261331,37.801197],[-122.261352,37.801356],[-122.261386,37.801543],[-122.261421,37.801702],[-122.26133799999999,37.801884],[-122.261303,37.801911],[-122.261213,37.802087],[-122.261337,37.802104],[-122.261275,37.80229],[-122.261157,37.802274],[-122.26108600000001,37.80241],[-122.261053,37.802444],[-122.260998,37.802598],[-122.260824,37.803186],[-122.260644,37.803642],[-122.260498,37.804048],[-122.260498,37.804229],[-122.26054600000001,37.804361],[-122.260712,37.804559],[-122.260863,37.804528],[-122.26122700000001,37.804524],[-122.261263,37.80459],[-122.261366,37.804649],[-122.261549,37.804763],[-122.26191,37.804974],[-122.261994,37.805029],[-122.262044,37.805068],[-122.262125,37.805152],[-122.262201,37.805244],[-122.262596,37.805832],[-122.262697,37.805982],[-122.262727,37.80603],[-122.26307300000001,37.806586],[-122.263132,37.806699],[-122.263166,37.806789],[-122.26317899999999,37.806839],[-122.263306,37.807065],[-122.263395,37.807325],[-122.263481,37.807474],[-122.263605,37.807585],[-122.263682,37.807656],[-122.263777,37.807716],[-122.26386100000001,37.807769],[-122.263942,37.807815],[-122.264089,37.807888],[-122.264364,37.807998],[-122.26423800000001,37.808024],[-122.264104,37.808059],[-122.26377599999999,37.808177],[-122.263678,37.808208],[-122.263564,37.80825],[-122.263458,37.808294],[-122.263344,37.808357],[-122.263043,37.808679],[-122.26316300000001,37.808899],[-122.26313399999999,37.80897],[-122.26304,37.809306],[-122.263018,37.809373],[-122.26275699999999,37.810126],[-122.26249799999999,37.810847],[-122.262463,37.810945],[-122.262404,37.810937],[-122.26231199999999,37.810926],[-122.262078,37.811447],[-122.26193600000001,37.811768],[-122.26155799999999,37.812589],[-122.261341,37.812652],[-122.261154,37.812707],[-122.261335,37.812867],[-122.26140599999999,37.812941],[-122.26074800000001,37.813608],[-122.26068100000001,37.813683],[-122.260751,37.813733],[-122.260679,37.813815],[-122.26054600000001,37.813961],[-122.260105,37.814452],[-122.260034,37.814521],[-122.259911,37.814626],[-122.259899,37.814637],[-122.259652,37.814789],[-122.259596,37.814817],[-122.259519,37.814856],[-122.259438,37.814916],[-122.259388,37.814954],[-122.25931199999999,37.815021],[-122.25922199999999,37.815102],[-122.259135,37.815191],[-122.25909900000001,37.815239],[-122.259039,37.815321],[-122.258914,37.815258],[-122.25878299999999,37.815414],[-122.258673,37.815515],[-122.25861500000001,37.815553],[-122.258537,37.815618],[-122.258493,37.815656],[-122.258414,37.815707],[-122.258059,37.815942],[-122.257857,37.816055],[-122.257789,37.816103],[-122.257778,37.816111],[-122.257507,37.816317],[-122.257363,37.816237],[-122.25714600000001,37.816117],[-122.25527,37.81732]]],"type":"Polygon"} +}''' + return httmock.response(200, content, headers={'Content-Type': 'application/json'}) + + raise Exception('Unknown URL') + + tar = unittest.mock.Mock() + + with httmock.HTTMock(response_content): + wof_bundle.add_tar_place(tar, '554/753/227/554753227.geojson', 'https://whosonfirst.mapzen.com/data/554/753/227/554753227.geojson') + + tar_info, bytes_io = tar.addfile.mock_calls[0][1] + self.assertEqual(tar_info.name, 'data/554/753/227/554753227.geojson') + self.assertIn(b'554753227', bytes_io.getvalue()) diff --git a/metroextractor/wof_bundle.py b/metroextractor/wof_bundle.py new file mode 100644 index 0000000..a043a67 --- /dev/null +++ b/metroextractor/wof_bundle.py @@ -0,0 +1,126 @@ +import requests, uritemplate + +from csv import DictWriter +from os import SEEK_SET, SEEK_END +from io import BytesIO, TextIOWrapper +from argparse import ArgumentParser +from time import sleep, time +from os.path import join +from sys import stderr +import tarfile + +PLACE_TYPES = 'region', 'locality', 'postalcode' #, 'neighbourhood' +WOF_URI = 'https://whosonfirst-api.dev.mapzen.com/{?method,extras,min_latitude,max_latitude,min_longitude,max_longitude}{&placetype,cursor}' +BASE_DIR = 'data' + +parser = ArgumentParser(description='Tell us what they told you.') + +parser.add_argument('bbox', type=float, nargs=4, metavar='deg.', + help='Bounding box degrees, given as (lon, lat, lon, lat)') + +parser.add_argument('filename', + help='Output filename') + +def tell_us(bbox, filename): + ''' + ''' + with io.open(filename, 'w') as file: + print(bbox, file=file) + +def main(): + args = parser.parse_args() + return(bundle_wof(args.bbox, args.filename)) + +def bundle_wof(bbox, filename): + ''' + ''' + # Prepare bundle file CSV. + meta_buf = BytesIO() + meta_txt = TextIOWrapper(meta_buf, encoding='utf8', write_through=True) + meta_csv = DictWriter(meta_txt, ('placetype', 'id', 'name', 'parent_id', 'path')) + meta_csv.writerow({f: f for f in meta_csv.fieldnames}) + + # Start populating bundle tarball. + with tarfile.open(filename, mode='w:bz2') as bundle_tar: + for place in iterate_places(bbox): + add_tar_place(bundle_tar, place['wof:path'], place['mz:uri']) + + row = {k: place['wof:{}'.format(k)] for k in meta_csv.fieldnames} + meta_csv.writerow(row) + + add_tar_file(bundle_tar, 'index.csv', meta_buf) + +def add_tar_place(tar_file, wof_path, mz_uri): + ''' Download WoF place GeoJSON and add it to the tarball. + ''' + got = get_url(mz_uri) + + if got.status_code != 200: + raise Exception('Got status={}: {}'.format(got.status_code, got.text)) + + geojson_data = BytesIO(got.content) + geojson_path = join(BASE_DIR, wof_path) + add_tar_file(tar_file, geojson_path, geojson_data) + +def add_tar_file(tar_file, filename, buffer): + ''' Add a file to the tarball with content from a buffer. + ''' + tar_info = tarfile.TarInfo(filename) + + buffer.seek(0, SEEK_END) + tar_info.size = buffer.tell() + tar_info.mtime = time() + + buffer.seek(0, SEEK_SET) + tar_file.addfile(tar_info, buffer) + +def iterate_places(bbox): + ''' Iterate over places. + ''' + for placetype in PLACE_TYPES: + for place in iterate_places_type(bbox, placetype): + yield place + +def iterate_places_type(bbox, placetype): + ''' Iterate over places of a single type. + ''' + lon1, lat1, lon2, lat2 = bbox + + query = { + 'method': 'whosonfirst.places.getIntersects', + 'min_latitude': min(lat1, lat2), + 'min_longitude': min(lon1, lon2), + 'max_latitude': max(lat1, lat2), + 'max_longitude': max(lon1, lon2), + 'extras': 'mz:uri,wof:path', + 'placetype': placetype, + } + + while True: + url = uritemplate.expand(WOF_URI, query) + got = get_url(url) + + if got.status_code != 200: + raise Exception('Got status={}: {}'.format(got.status_code, got.text)) + + if got.json().get('stat') == 'ok': + for result in got.json().get('results'): + yield result + + if got.json().get('cursor', 0) == 0: + break + + query.update(cursor=got.json().get('cursor', 0)) + +def get_url(url): + ''' Get URL response while respecting rate-limit responses. + ''' + for backoff in range(1, 99): + got = requests.get(url) + + if got.status_code != 429: + print('Got', url, file=stderr) + return got + + print('Zzzz...', file=stderr) + sleep(backoff) diff --git a/setup.py b/setup.py index 5455895..c4973b9 100644 --- a/setup.py +++ b/setup.py @@ -11,6 +11,7 @@ entry_points = dict( console_scripts = [ 'extracts-nothingburger=metroextractor.nothingburger:main', + 'extracts-wof-bundle=metroextractor.wof_bundle:main', ] ), package_data = {}, @@ -18,5 +19,6 @@ install_requires = [ 'requests == 2.11.1', 'uritemplate == 3.0.0', + 'httmock==1.2.5', ] )