1
2 """Loads OSM tiles from the Web using threading and urllib3"""
3
4 import os
5 import sys
6 import random
7 import threading
8
9 from Queue import Queue
10 import urllib3
11
12 import pyglet.image as pygletimage
13
14 __author__ = "B. Henne"
15 __maintainer__ = "B. Henne"
16 __contact__ = "henne@dcsec.uni-hannover.de"
17 __copyright__ = "(c) 2012, DCSec, Leibniz Universitaet Hannover, Germany"
18 __license__ = "GPLv3"
19
20
21 LAYERS = {"tah": ["cassini.toolserver.org:8080", "/http://a.tile.openstreetmap.org/+http://toolserver.org/~cmarqu/hill/"],
22 "oam": ["oam1.hypercube.telascience.org", "/tiles/1.0.0/openaerialmap-900913/"],
23 "mapnikold": ["tile.openstreetmap.org", "/mapnik/"],
24 "mapnik": [["a.tile.openstreetmap.org", "b.tile.openstreetmap.org", "c.tile.openstreetmap.org"], "/"],
25 }
26
27
29 """A tile downloader using threads for downloads"""
30
31 - def __init__(self, tile_list={}, cache_dir='.cache', NUM_WORKER=1, NUM_SOCKETS=3):
32 """Inits the TileLoader with its threads"""
33
34 self.dl_queue = Queue()
35 self.load_queue = Queue()
36 self.data_dir = os.path.join(os.path.dirname(os.path.abspath(sys.modules[self.__module__].__file__)), '..', 'data')
37 self.cache_dir = cache_dir
38 self.tile_list = tile_list
39 self.loading_image = pygletimage.load(os.path.join(self.data_dir, 'loading.png'))
40 self.NUM_SOCKETS = NUM_SOCKETS
41 self.t = []
42 for w in xrange(0, NUM_WORKER):
43 self.t.append(threading.Thread(target=self.worker))
44 for t in self.t:
45 t.setDaemon(True)
46 t.start()
47
49 """Thread worker"""
50
51 http = urllib3.PoolManager(maxsize=self.NUM_SOCKETS)
52 while True:
53 tile = self.dl_queue.get()
54 ok = False
55 tries = 3
56 while (ok == False) and (tries > 0):
57 tries -= 1
58 r = http.request('GET', tile[0])
59 ok = self.handle_response(r.data, tile[1], tile[2], tile[3], tile[4], tile[5])
60 self.load_queue.put((tile[1], tile[2], tile[3], tile[4], tile[5]))
61 self.dl_queue.task_done()
62
64 """Handles download responses, writes images to files"""
65
66 if '<html>' in r:
67
68 return False
69 elements = [self.cache_dir, layer, str(z), str(x)]
70 path = ''
71 for element in elements:
72 path = os.path.join(path, element)
73 try:
74 os.mkdir(path)
75 except OSError:
76
77 pass
78 path = os.path.join(path, str(y) + file_extension)
79 image_file = open(path, 'wb')
80 image_file.write(r)
81 image_file.flush()
82 image_file.close()
83 return True
84
86 """Loads images downloaded by workers to tile_list"""
87
88 for i in xrange(0, self.load_queue.qsize()):
89 x, y, z, layer, file_extension = self.load_queue.get()
90 path = os.path.join(self.cache_dir, layer, str(z), str(x), str(y) + file_extension)
91 image = pygletimage.load(path)
92 image.anchor_x = image.width / 2
93 image.anchor_y = image.height / 2
94 self.tile_list[(x, y, z)] = image
95 self.load_queue.task_done()
96
98 """Enqueues a tile to be downloaded"""
99
100 if type(LAYERS[layer][0]) == str:
101 host = LAYERS[layer][0]
102 elif type(LAYERS[layer][0]) == list:
103 host = random.choice(LAYERS[layer][0])
104 layer = layer
105
106 base_url_extension = LAYERS[layer][1]
107 file_extension = '.jpg' if layer == 'oam' else '.png'
108
109
110
111 self.tile_list[(x, y, z)] = self.loading_image
112
113 url = 'http://' + host + base_url_extension + str(z) + '/' + str(x) + '/' + str(y) + file_extension
114 self.dl_queue.put((url, x, y, z, layer, file_extension))
115 return True
116
117
119 myApp = wx.App()
120 t = TileLoader()
121 import time
122 x=40
123 while 1:
124 if x < 50:
125 for i in xrange(0,20):
126 t.enqueue_tile(34, x, 16)
127 if t.dl_queue.qsize() <= 0:
128 break
129 print "Queue size: %s" % t.dl_queue.qsize()
130 time.sleep(1)
131 t.load_images()
132 x += 1
133
134
135 if __name__ == '__main__':
136 test()
137