from urllib parse import unquote quote from ioweb import request DataN

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
from urllib.parse import unquote, quote
from ioweb import request, DataNotValid
def main(**kwargs):
username = 'cal_89'
cursor_id = None
while True:
url = 'https://mobile.twitter.com/%s/followers' % (
quote(username),
)
if cursor_id:
url += '?cursor=%d' % cursor_id
res = request(url)
if not res.css('div.user-list').exists():
raise DataNotValid
usernames = set()
for elem in res.xpath(
'//a[strong[contains(@class, "fullname")]]'
):
usernames.add(
unquote(elem.attr('href').lstrip('/').split('?')[0])
.lower()
)
print(usernames)
# <div class="w-button-more"><a href="/dzuev/followers?cursor=1623574621321415001">
more = res.css('div.w-button-more a')
if more.exists():
cursor_id = int(
more.attr('href').split('cursor=')[1].split('&')[0]
)
else:
cursor_id = None
print('Cursor ID: %s' % cursor_id)
if not cursor_id:
print('Last page!')
break