PyWikiScripts/external_links_checker.py at master · 0x010C/PyWikiScripts · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
#!/usr/bin/python2.7
# -*- coding: utf-8 -*-
#Autor: Antoine "0x010C" Lamielle
#Date: 22 Mai 2016
#License: GNU GPL v3

import pywiki
import requests
import threading
import time
import re
from collections import deque


lock = threading.Lock()
links = deque()
nb_article_crawled = 0
nb_links_found = 0
nb_links_found_lock = threading.Lock()

blacklist = [
	"wikimedia.org",
	"wikipedia.org",
	"www.minorplanetcenter.net",
]

headers = {
	'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1',
}


class LinkChecker (threading.Thread):
	def __init__(self, threadID):
		threading.Thread.__init__(self)
		self.threadID = threadID
	def run(self):
		global nb_links_found
		print "Starting " + str(self.threadID)
		while True:
			with lock:
				try:
					(title, link) = links.popleft()
					print str(nb_links_found)+"#"+str(nb_article_crawled)+"#\t"+str(len(links))
				except IndexError:
					print "pause"+str(self.threadID)
					link = None
			if link == None:
				time.sleep(2)
				continue

			found = False
			try:
				r = requests.get(link, headers=headers)
				if r.status_code != 200:
					print title+" --> ("+str(r.status_code)+") "+link
					found = True
			except requests.exceptions.ConnectionError:
				print title+" --> (DOM) "+link
				found = True
			except requests.exceptions.TooManyRedirects:
				print title+" --> (RED) "+link
				found = True

			if found:
				with nb_links_found_lock:
					nb_links_found += 1

		print "Exiting " + str(self.threadID)


# Main
def main():
	global links, nb_article_crawled
	pw = pywiki.Pywiki("frwiki-NeoBOT")
	pw.login()
	pw.limit = 500

	blacklist_r = "("
	for site in blacklist:
		blacklist_r += site + "|"
	blacklist_r = re.compile(blacklist_r[:-1]+")")

	threads = []
	for i in range(0,392):
		threads.append(LinkChecker(i))
		threads[i].start()

	gap_continue = ""
	le_offset = 0
	while gap_continue != None:
		need_more_links = False
		with lock:
			if len(links) < 700:
				need_more_links = True

		if need_more_links:
			print "Get more links"
			response = pw.request({
				"action":"query",
				"format":"json",
				"prop":"extlinks",
				"generator":"allpages",
				"elexpandurl":"1",
				"ellimit":pw.limit,
				"gaplimit":pw.limit,
				"eloffset":le_offset,
				"gapcontinue":gap_continue,
				"assert":pw.assertion,
			})

			with lock:
				for id in response["query"]["pages"]:
					if "extlinks" in response["query"]["pages"][id]:
						for extlink in response["query"]["pages"][id]["extlinks"]:
							if not blacklist_r.search(extlink["*"]):
								links.append((response["query"]["pages"][id]["title"], extlink["*"]))

			if "continue" in response:
				if response["continue"]["continue"] == "gapcontinue||":
					gap_continue = response["continue"]["gapcontinue"]
					le_offset = 0
					nb_article_crawled += pw.limit
				else:
					le_offset = response["continue"]["eloffset"]
			else:
				gap_continue = None
		else:
			time.sleep(10)

main()