1
2
3
4
5
6
7
8
9
10
11
12
13
14
15 import sys
16 import xmlrpc.client
17 import time
18 import re
19 import os.path
20 import urllib.parse
21 import urllib.request
22
23 netRetryM = 0x7FFFFFFF
24 netRetryD = 1.0
25 netRetryDFun = lambda oldDelay: oldDelay * 1.5
26
27
28 def retryOnError(fun, retriesMax, retryDelay, retryDelayFun):
29 while True:
30 try:
31 return fun()
32 except:
33 if retriesMax < 1: raise
34 print("Error:", sys.exc_info())
35 print("Waiting %f seconds before retrying (Retries left: %i)..." % (
36 retryDelay, retriesMax
37 ))
38 time.sleep(retryDelay)
39 retriesMax -= 1
40 retryDelay = retryDelayFun(retryDelay)
41 continue
42
43
44 def writeFile(path, content):
45 fd = open(path, 'w')
46 fd.write(content)
47 fd.close()
48
49
50 wiki = xmlrpc.client.ServerProxy("http://www.larpwiki.de/?action=xmlrpc2")
51 pagenames = retryOnError(
52 lambda: sorted(wiki.getAllPages()), netRetryM, netRetryD, netRetryDFun
53 )
54
55
56 count = 0
57 slashRegExp = re.compile(r"/")
58 for pagename in pagenames:
59 count += 1
60 filename = slashRegExp.sub(' - ', pagename) + '.txt'
61 if (os.path.isfile(filename)):
62 print(r'%i/%i "%s" already exists.' % (count, len(pagenames), filename))
63 else:
64 print(r'%i/%i fetching "%s".' % (count, len(pagenames), filename))
65 url = r"https://larpwiki.de/" + urllib.parse.quote(pagename) + r"?action=raw"
66 response = retryOnError(
67 lambda: urllib.request.urlopen(url),
68 netRetryM, netRetryD, netRetryDFun
69 )
70 text = response.read().decode("utf-8", "replace")
71 writeFile(filename, text)
72 time.sleep(0.1)
, as shown below in the list of files. Do
link, since this is subject to change and can break easily.